From b58afbf849217c861c5a685214d958431ef3411f Mon Sep 17 00:00:00 2001 From: Peter Huene Date: Wed, 25 Nov 2020 16:10:09 -0800 Subject: [PATCH 01/33] Refactor module instantiation in the runtime. This commit refactors module instantiation in the runtime to allow for different instance allocation strategy implementations. It adds an `InstanceAllocator` trait with the current implementation put behind the `OnDemandInstanceAllocator` struct. The Wasmtime API has been updated to allow a `Config` to have an instance allocation strategy set which will determine how instances get allocated. This change is in preparation for an alternative *pooling* instance allocator that can reserve all needed host process address space in advance. This commit also makes changes to the `wasmtime_environ` crate to represent compiled modules in a way that reduces copying at instantiation time. --- cranelift/wasm/src/module_translator.rs | 2 + crates/environ/src/data_structures.rs | 2 +- crates/environ/src/module.rs | 53 +- crates/environ/src/module_environ.rs | 39 +- crates/jit/src/instantiate.rs | 134 ++-- crates/runtime/src/instance.rs | 594 ++---------------- crates/runtime/src/instance/allocator.rs | 536 ++++++++++++++++ crates/runtime/src/lib.rs | 5 +- crates/runtime/src/vmcontext.rs | 2 +- crates/wasmtime/src/config.rs | 52 +- crates/wasmtime/src/instance.rs | 37 +- crates/wasmtime/src/store.rs | 18 +- .../wasmtime/src/trampoline/create_handle.rs | 35 +- crates/wasmtime/src/trampoline/memory.rs | 6 +- 14 files changed, 829 insertions(+), 686 deletions(-) create mode 100644 crates/runtime/src/instance/allocator.rs diff --git a/cranelift/wasm/src/module_translator.rs b/cranelift/wasm/src/module_translator.rs index ed71d7ce1ca2..819834f20e0b 100644 --- a/cranelift/wasm/src/module_translator.rs +++ b/cranelift/wasm/src/module_translator.rs @@ -101,6 +101,8 @@ pub fn translate_module<'data>( Payload::DataCountSection { count, range } => { validator.data_count_section(count, &range)?; + + // NOTE: the count here is the total segment count, not the passive segment count environ.reserve_passive_data(count)?; } diff --git a/crates/environ/src/data_structures.rs b/crates/environ/src/data_structures.rs index 2fc9e9ecd05d..07f2aedaeca9 100644 --- a/crates/environ/src/data_structures.rs +++ b/crates/environ/src/data_structures.rs @@ -20,7 +20,7 @@ pub mod isa { } pub mod entity { - pub use cranelift_entity::{packed_option, BoxedSlice, EntityRef, PrimaryMap}; + pub use cranelift_entity::{packed_option, BoxedSlice, EntityRef, EntitySet, PrimaryMap}; } pub mod wasm { diff --git a/crates/environ/src/module.rs b/crates/environ/src/module.rs index b25682acf215..957feb55f0a7 100644 --- a/crates/environ/src/module.rs +++ b/crates/environ/src/module.rs @@ -158,13 +158,19 @@ pub struct Module { pub table_elements: Vec, /// WebAssembly passive elements. - pub passive_elements: HashMap>, + pub passive_elements: Vec>, + + /// The map from passive element index (element segment index space) to index in `passive_elements`. + pub passive_elements_map: HashMap, /// WebAssembly passive data segments. #[serde(with = "passive_data_serde")] - pub passive_data: HashMap>, + pub passive_data: Vec>, - /// WebAssembly table initializers. + /// The map from passive data index (data segment index space) to index in `passive_data`. + pub passive_data_map: HashMap, + + /// WebAssembly function names. pub func_names: HashMap, /// Types declared in the wasm module. @@ -272,7 +278,8 @@ impl Module { /// Get the given passive element, if it exists. pub fn get_passive_element(&self, index: ElemIndex) -> Option<&[FuncIndex]> { - self.passive_elements.get(&index).map(|es| &**es) + let index = *self.passive_elements_map.get(&index)?; + Some(self.passive_elements[index].as_ref()) } /// Convert a `DefinedFuncIndex` into a `FuncIndex`. @@ -419,47 +426,45 @@ pub struct InstanceSignature { } mod passive_data_serde { - use super::{Arc, DataIndex, HashMap}; - use serde::{de::MapAccess, de::Visitor, ser::SerializeMap, Deserializer, Serializer}; + use super::Arc; + use serde::{de::SeqAccess, de::Visitor, ser::SerializeSeq, Deserializer, Serializer}; use std::fmt; - pub(super) fn serialize( - data: &HashMap>, - ser: S, - ) -> Result + pub(super) fn serialize(data: &Vec>, ser: S) -> Result where S: Serializer, { - let mut map = ser.serialize_map(Some(data.len()))?; - for (k, v) in data { - map.serialize_entry(k, v.as_ref())?; + let mut seq = ser.serialize_seq(Some(data.len()))?; + for v in data { + seq.serialize_element(v.as_ref())?; } - map.end() + seq.end() } struct PassiveDataVisitor; impl<'de> Visitor<'de> for PassiveDataVisitor { - type Value = HashMap>; + type Value = Vec>; fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { - formatter.write_str("a passive_data map") + formatter.write_str("a passive data sequence") } - fn visit_map(self, mut access: M) -> Result + + fn visit_seq(self, mut access: M) -> Result where - M: MapAccess<'de>, + M: SeqAccess<'de>, { - let mut map = HashMap::with_capacity(access.size_hint().unwrap_or(0)); - while let Some((key, value)) = access.next_entry::<_, Vec>()? { - map.insert(key, value.into()); + let mut data = Vec::with_capacity(access.size_hint().unwrap_or(0)); + while let Some(value) = access.next_element::>()? { + data.push(value.into()); } - Ok(map) + Ok(data) } } - pub(super) fn deserialize<'de, D>(de: D) -> Result>, D::Error> + pub(super) fn deserialize<'de, D>(de: D) -> Result>, D::Error> where D: Deserializer<'de>, { - de.deserialize_map(PassiveDataVisitor) + de.deserialize_seq(PassiveDataVisitor) } } diff --git a/crates/environ/src/module_environ.rs b/crates/environ/src/module_environ.rs index 9d1f89cb1f82..800455358032 100644 --- a/crates/environ/src/module_environ.rs +++ b/crates/environ/src/module_environ.rs @@ -710,11 +710,13 @@ impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data elem_index: ElemIndex, segments: Box<[FuncIndex]>, ) -> WasmResult<()> { + let index = self.result.module.passive_elements.len(); + self.result.module.passive_elements.push(segments); let old = self .result .module - .passive_elements - .insert(elem_index, segments); + .passive_elements_map + .insert(elem_index, index); debug_assert!( old.is_none(), "should never get duplicate element indices, that would be a bug in `cranelift_wasm`'s \ @@ -782,17 +784,21 @@ impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data Ok(()) } - fn reserve_passive_data(&mut self, count: u32) -> WasmResult<()> { - self.result.module.passive_data.reserve(count as usize); + fn reserve_passive_data(&mut self, _count: u32) -> WasmResult<()> { + // Note: the count passed in here is the *total* segment count + // There is no way to reserve for just the passive segments as they are discovered when iterating the data section entries + // Given that the total segment count might be much larger than the passive count, do not reserve Ok(()) } fn declare_passive_data(&mut self, data_index: DataIndex, data: &'data [u8]) -> WasmResult<()> { + let index = self.result.module.passive_data.len(); + self.result.module.passive_data.push(Arc::from(data)); let old = self .result .module - .passive_data - .insert(data_index, Arc::from(data)); + .passive_data_map + .insert(data_index, index); debug_assert!( old.is_none(), "a module can't have duplicate indices, this would be a cranelift-wasm bug" @@ -1088,3 +1094,24 @@ pub struct DataInitializer<'data> { /// The initialization data. pub data: &'data [u8], } + +/// Similar to `DataInitializer`, but owns its own copy of the data rather +/// than holding a slice of the original module. +#[derive(Serialize, Deserialize)] +pub struct OwnedDataInitializer { + /// The location where the initialization is to be performed. + pub location: DataInitializerLocation, + + /// The initialization data. + pub data: Box<[u8]>, +} + +impl OwnedDataInitializer { + /// Creates a new owned data initializer from a borrowed data initializer. + pub fn new(borrowed: DataInitializer<'_>) -> Self { + Self { + location: borrowed.location.clone(), + data: borrowed.data.into(), + } + } +} diff --git a/crates/jit/src/instantiate.rs b/crates/jit/src/instantiate.rs index 655432e00f47..a1647cde5a9b 100644 --- a/crates/jit/src/instantiate.rs +++ b/crates/jit/src/instantiate.rs @@ -11,7 +11,6 @@ use object::File as ObjectFile; #[cfg(feature = "parallel-compilation")] use rayon::prelude::*; use serde::{Deserialize, Serialize}; -use std::any::Any; use std::ops::Range; use std::sync::Arc; use thiserror::Error; @@ -22,16 +21,11 @@ use wasmtime_environ::wasm::{ DefinedFuncIndex, InstanceTypeIndex, ModuleTypeIndex, SignatureIndex, WasmFuncType, }; use wasmtime_environ::{ - CompileError, DataInitializer, DataInitializerLocation, DebugInfoData, FunctionAddressMap, - InstanceSignature, Module, ModuleEnvironment, ModuleSignature, ModuleTranslation, - StackMapInformation, TrapInformation, + CompileError, DebugInfoData, FunctionAddressMap, InstanceSignature, Module, ModuleEnvironment, + ModuleSignature, ModuleTranslation, OwnedDataInitializer, StackMapInformation, TrapInformation, }; use wasmtime_profiling::ProfilingAgent; -use wasmtime_runtime::{ - GdbJitImageRegistration, Imports, InstanceHandle, InstantiationError, RuntimeMemoryCreator, - StackMapRegistry, VMExternRefActivationsTable, VMFunctionBody, VMInterrupts, - VMSharedSignatureIndex, VMTrampoline, -}; +use wasmtime_runtime::{GdbJitImageRegistration, InstantiationError, VMFunctionBody, VMTrampoline}; /// An error condition while setting up a wasm instance, be it validation, /// compilation, or instantiation. @@ -59,7 +53,8 @@ pub enum SetupError { #[derive(Serialize, Deserialize)] pub struct CompilationArtifacts { /// Module metadata. - module: Module, + #[serde(with = "arc_serde")] + module: Arc, /// ELF image with functions code. obj: Box<[u8]>, @@ -68,7 +63,8 @@ pub struct CompilationArtifacts { unwind_info: Box<[ObjectUnwindInfo]>, /// Data initiailizers. - data_initializers: Box<[OwnedDataInitializer]>, + #[serde(with = "arc_slice_serde")] + data_initializers: Arc<[OwnedDataInitializer]>, /// Descriptions of compiled functions funcs: PrimaryMap, @@ -134,7 +130,7 @@ impl CompilationArtifacts { .into_iter() .map(OwnedDataInitializer::new) .collect::>() - .into_boxed_slice(); + .into(); let obj = obj.write().map_err(|_| { SetupError::Instantiate(InstantiationError::Resource( @@ -143,7 +139,7 @@ impl CompilationArtifacts { })?; Ok(CompilationArtifacts { - module, + module: Arc::new(module), obj: obj.into_boxed_slice(), unwind_info: unwind_info.into_boxed_slice(), data_initializers, @@ -208,7 +204,6 @@ pub struct ModuleCode { /// A compiled wasm module, ready to be instantiated. pub struct CompiledModule { artifacts: CompilationArtifacts, - module: Arc, code: Arc, finished_functions: FinishedFunctions, trampolines: PrimaryMap, @@ -267,7 +262,6 @@ impl CompiledModule { let finished_functions = FinishedFunctions(finished_functions); Ok(Arc::new(Self { - module: Arc::new(artifacts.module.clone()), artifacts, code: Arc::new(ModuleCode { code_memory, @@ -278,62 +272,24 @@ impl CompiledModule { })) } - /// Crate an `Instance` from this `CompiledModule`. - /// - /// Note that if only one instance of this module is needed, it may be more - /// efficient to call the top-level `instantiate`, since that avoids copying - /// the data initializers. - /// - /// # Unsafety - /// - /// See `InstanceHandle::new` - pub unsafe fn instantiate( - &self, - imports: Imports<'_>, - lookup_shared_signature: &dyn Fn(SignatureIndex) -> VMSharedSignatureIndex, - mem_creator: Option<&dyn RuntimeMemoryCreator>, - interrupts: *const VMInterrupts, - host_state: Box, - externref_activations_table: *mut VMExternRefActivationsTable, - stack_map_registry: *mut StackMapRegistry, - ) -> Result { - InstanceHandle::new( - self.module.clone(), - &self.finished_functions.0, - imports, - mem_creator, - lookup_shared_signature, - host_state, - interrupts, - externref_activations_table, - stack_map_registry, - ) - } /// Extracts `CompilationArtifacts` from the compiled module. pub fn compilation_artifacts(&self) -> &CompilationArtifacts { &self.artifacts } - /// Returns data initializers to pass to `InstanceHandle::initialize` - pub fn data_initializers(&self) -> Vec> { - self.artifacts - .data_initializers - .iter() - .map(|init| DataInitializer { - location: init.location.clone(), - data: &*init.data, - }) - .collect() + /// Returns the data initializers from the compiled module. + pub fn data_initializers(&self) -> &Arc<[OwnedDataInitializer]> { + &self.artifacts.data_initializers } /// Return a reference-counting pointer to a module. pub fn module(&self) -> &Arc { - &self.module + &self.artifacts.module } /// Return a reference to a mutable module (if possible). pub fn module_mut(&mut self) -> Option<&mut Module> { - Arc::get_mut(&mut self.module) + Arc::get_mut(&mut self.artifacts.module) } /// Returns the map of all finished JIT functions compiled for this module @@ -470,26 +426,6 @@ impl SymbolizeContext { } } -/// Similar to `DataInitializer`, but owns its own copy of the data rather -/// than holding a slice of the original module. -#[derive(Clone, Serialize, Deserialize)] -pub struct OwnedDataInitializer { - /// The location where the initialization is to be performed. - location: DataInitializerLocation, - - /// The initialization data. - data: Box<[u8]>, -} - -impl OwnedDataInitializer { - fn new(borrowed: DataInitializer<'_>) -> Self { - Self { - location: borrowed.location.clone(), - data: borrowed.data.to_vec().into_boxed_slice(), - } - } -} - fn create_dbg_image( obj: Vec, code_range: (*const u8, usize), @@ -586,3 +522,45 @@ impl From> for DebugInfo { } } } + +mod arc_serde { + use super::Arc; + use serde::{de::Deserialize, ser::Serialize, Deserializer, Serializer}; + + pub(super) fn serialize(arc: &Arc, ser: S) -> Result + where + S: Serializer, + T: Serialize, + { + (**arc).serialize(ser) + } + + pub(super) fn deserialize<'de, D, T>(de: D) -> Result, D::Error> + where + D: Deserializer<'de>, + T: Deserialize<'de>, + { + Ok(Arc::new(T::deserialize(de)?)) + } +} + +mod arc_slice_serde { + use super::Arc; + use serde::{Deserialize, Deserializer, Serialize, Serializer}; + + pub(super) fn serialize(arc: &Arc<[T]>, ser: S) -> Result + where + S: Serializer, + T: Serialize, + { + (**arc).serialize(ser) + } + + pub(super) fn deserialize<'de, D, T>(de: D) -> Result, D::Error> + where + D: Deserializer<'de>, + T: Deserialize<'de>, + { + Ok(Vec::::deserialize(de)?.into()) + } +} diff --git a/crates/runtime/src/instance.rs b/crates/runtime/src/instance.rs index 1df1cfb64065..cdb892842f1a 100644 --- a/crates/runtime/src/instance.rs +++ b/crates/runtime/src/instance.rs @@ -4,12 +4,11 @@ use crate::export::Export; use crate::externref::{StackMapRegistry, VMExternRefActivationsTable}; -use crate::imports::Imports; -use crate::memory::{DefaultMemoryCreator, RuntimeLinearMemory, RuntimeMemoryCreator}; +use crate::memory::{RuntimeLinearMemory, RuntimeMemoryCreator}; use crate::table::{Table, TableElement}; use crate::traphandlers::Trap; use crate::vmcontext::{ - VMBuiltinFunctionsArray, VMCallerCheckedAnyfunc, VMContext, VMFunctionBody, VMFunctionImport, + VMBuiltinFunctionsArray, VMCallerCheckedAnyfunc, VMContext, VMFunctionImport, VMGlobalDefinition, VMGlobalImport, VMInterrupts, VMMemoryDefinition, VMMemoryImport, VMSharedSignatureIndex, VMTableDefinition, VMTableImport, }; @@ -17,23 +16,24 @@ use crate::{ExportFunction, ExportGlobal, ExportMemory, ExportTable}; use indexmap::IndexMap; use memoffset::offset_of; use more_asserts::assert_lt; -use std::alloc::{self, Layout}; +use std::alloc::Layout; use std::any::Any; use std::cell::RefCell; -use std::collections::HashMap; use std::convert::TryFrom; use std::ptr::NonNull; use std::rc::Rc; use std::sync::Arc; use std::{mem, ptr, slice}; -use thiserror::Error; -use wasmtime_environ::entity::{packed_option::ReservedValue, BoxedSlice, EntityRef, PrimaryMap}; +use wasmtime_environ::entity::{packed_option::ReservedValue, BoxedSlice, EntityRef, EntitySet}; use wasmtime_environ::wasm::{ - DataIndex, DefinedFuncIndex, DefinedGlobalIndex, DefinedMemoryIndex, DefinedTableIndex, - ElemIndex, EntityIndex, FuncIndex, GlobalIndex, GlobalInit, MemoryIndex, SignatureIndex, - TableElementType, TableIndex, WasmType, + DataIndex, DefinedGlobalIndex, DefinedMemoryIndex, DefinedTableIndex, ElemIndex, EntityIndex, + FuncIndex, GlobalIndex, MemoryIndex, TableElementType, TableIndex, }; -use wasmtime_environ::{ir, DataInitializer, Module, ModuleType, TableElements, VMOffsets}; +use wasmtime_environ::{ir, Module, VMOffsets}; + +mod allocator; + +pub use allocator::*; /// Runtime representation of an instance value, which erases all `Instance` /// information since instances are just a collection of values. @@ -56,14 +56,13 @@ pub(crate) struct Instance { /// WebAssembly table data. tables: BoxedSlice, - /// Passive elements in this instantiation. As `elem.drop`s happen, these - /// entries get removed. A missing entry is considered equivalent to an - /// empty slice. - passive_elements: RefCell>>, + /// Stores the dropped passive element segments in this instantiation by index. + /// If the index is present in the set, the segment has been dropped. + dropped_elements: RefCell>, - /// Passive data segments from our module. As `data.drop`s happen, entries - /// get removed. A missing entry is considered equivalent to an empty slice. - passive_data: RefCell>>, + /// Stores the dropped passive data segments in this instantiation by index. + /// If the index is present in the set, the segment has been dropped. + dropped_data: RefCell>, /// Hosts can store arbitrary per-instance information here. host_state: Box, @@ -551,11 +550,21 @@ impl Instance { // https://webassembly.github.io/bulk-memory-operations/core/exec/instructions.html#exec-table-init let table = self.get_table(table_index); - let passive_elements = self.passive_elements.borrow(); - let elem = passive_elements - .get(&elem_index) - .map(|e| &**e) - .unwrap_or_else(|| &[]); + let elem_index = self.module.passive_elements_map.get(&elem_index); + let elem = match elem_index { + Some(index) => { + if self + .dropped_elements + .borrow() + .contains(ElemIndex::new(*index)) + { + &[] + } else { + self.module.passive_elements[*index].as_ref() + } + } + None => &[], + }; if src .checked_add(len) @@ -567,8 +576,14 @@ impl Instance { // TODO(#983): investigate replacing this get/set loop with a `memcpy`. for (dst, src) in (dst..dst + len).zip(src..src + len) { + let elem = self + .get_caller_checked_anyfunc(elem[src as usize]) + .map_or(ptr::null_mut(), |f: &VMCallerCheckedAnyfunc| { + f as *const VMCallerCheckedAnyfunc as *mut _ + }); + table - .set(dst, TableElement::FuncRef(elem[src as usize])) + .set(dst, TableElement::FuncRef(elem)) .expect("should never panic because we already did the bounds check above"); } @@ -579,10 +594,14 @@ impl Instance { pub(crate) fn elem_drop(&self, elem_index: ElemIndex) { // https://webassembly.github.io/reference-types/core/exec/instructions.html#exec-elem-drop - let mut passive_elements = self.passive_elements.borrow_mut(); - passive_elements.remove(&elem_index); - // Note that we don't check that we actually removed an element because - // dropping a non-passive element is a no-op (not a trap). + if let Some(index) = self.module.passive_elements_map.get(&elem_index) { + self.dropped_elements + .borrow_mut() + .insert(ElemIndex::new(*index)); + } + + // Note that we don't check that we actually removed a segment because + // dropping a non-passive segment is a no-op (not a trap). } /// Do a `memory.copy` @@ -701,10 +720,17 @@ impl Instance { // https://webassembly.github.io/bulk-memory-operations/core/exec/instructions.html#exec-memory-init let memory = self.get_memory(memory_index); - let passive_data = self.passive_data.borrow(); - let data = passive_data - .get(&data_index) - .map_or(&[][..], |data| &**data); + let data_index = self.module.passive_data_map.get(&data_index); + let data = match data_index { + Some(index) => { + if self.dropped_data.borrow().contains(DataIndex::new(*index)) { + &[] + } else { + self.module.passive_data[*index].as_ref() + } + } + None => &[], + }; if src .checked_add(len) @@ -729,8 +755,14 @@ impl Instance { /// Drop the given data segment, truncating its length to zero. pub(crate) fn data_drop(&self, data_index: DataIndex) { - let mut passive_data = self.passive_data.borrow_mut(); - passive_data.remove(&data_index); + if let Some(index) = self.module.passive_data_map.get(&data_index) { + self.dropped_data + .borrow_mut() + .insert(DataIndex::new(*index)); + } + + // Note that we don't check that we actually removed a segment because + // dropping a non-passive segment is a no-op (not a trap). } /// Get a table by index regardless of whether it is locally-defined or an @@ -780,197 +812,8 @@ pub struct InstanceHandle { } impl InstanceHandle { - /// Create a new `InstanceHandle` pointing at a new `Instance`. - /// - /// # Unsafety - /// - /// This method is not necessarily inherently unsafe to call, but in general - /// the APIs of an `Instance` are quite unsafe and have not been really - /// audited for safety that much. As a result the unsafety here on this - /// method is a low-overhead way of saying "this is an extremely unsafe type - /// to work with". - /// - /// Extreme care must be taken when working with `InstanceHandle` and it's - /// recommended to have relatively intimate knowledge of how it works - /// internally if you'd like to do so. If possible it's recommended to use - /// the `wasmtime` crate API rather than this type since that is vetted for - /// safety. - /// - /// It is your responsibility to ensure that the given raw - /// `externref_activations_table` and `stack_map_registry` outlive this - /// instance. - pub unsafe fn new( - module: Arc, - finished_functions: &PrimaryMap, - imports: Imports, - mem_creator: Option<&dyn RuntimeMemoryCreator>, - lookup_shared_signature: &dyn Fn(SignatureIndex) -> VMSharedSignatureIndex, - host_state: Box, - interrupts: *const VMInterrupts, - externref_activations_table: *mut VMExternRefActivationsTable, - stack_map_registry: *mut StackMapRegistry, - ) -> Result { - debug_assert!(!externref_activations_table.is_null()); - debug_assert!(!stack_map_registry.is_null()); - - let tables = create_tables(&module); - let memories = create_memories(&module, mem_creator.unwrap_or(&DefaultMemoryCreator {}))?; - - let vmctx_tables = tables - .values() - .map(Table::vmtable) - .collect::>() - .into_boxed_slice(); - - let vmctx_memories = memories - .values() - .map(|a| a.vmmemory()) - .collect::>() - .into_boxed_slice(); - - let vmctx_globals = create_globals(&module); - - let offsets = VMOffsets::new(mem::size_of::<*const u8>() as u8, &module); - - let passive_data = RefCell::new(module.passive_data.clone()); - - let handle = { - let instance = Instance { - module, - offsets, - memories, - tables, - passive_elements: Default::default(), - passive_data, - host_state, - vmctx: VMContext {}, - }; - let layout = instance.alloc_layout(); - let instance_ptr = alloc::alloc(layout) as *mut Instance; - if instance_ptr.is_null() { - alloc::handle_alloc_error(layout); - } - ptr::write(instance_ptr, instance); - InstanceHandle { - instance: instance_ptr, - } - }; - let instance = handle.instance(); - - let mut ptr = instance.signature_ids_ptr(); - for sig in handle.module().types.values() { - *ptr = match sig { - ModuleType::Function(sig) => lookup_shared_signature(*sig), - _ => VMSharedSignatureIndex::new(u32::max_value()), - }; - ptr = ptr.add(1); - } - - debug_assert_eq!(imports.functions.len(), handle.module().num_imported_funcs); - ptr::copy( - imports.functions.as_ptr(), - instance.imported_functions_ptr() as *mut VMFunctionImport, - imports.functions.len(), - ); - debug_assert_eq!(imports.tables.len(), handle.module().num_imported_tables); - ptr::copy( - imports.tables.as_ptr(), - instance.imported_tables_ptr() as *mut VMTableImport, - imports.tables.len(), - ); - debug_assert_eq!( - imports.memories.len(), - handle.module().num_imported_memories - ); - ptr::copy( - imports.memories.as_ptr(), - instance.imported_memories_ptr() as *mut VMMemoryImport, - imports.memories.len(), - ); - debug_assert_eq!(imports.globals.len(), handle.module().num_imported_globals); - ptr::copy( - imports.globals.as_ptr(), - instance.imported_globals_ptr() as *mut VMGlobalImport, - imports.globals.len(), - ); - ptr::copy( - vmctx_tables.values().as_slice().as_ptr(), - instance.tables_ptr() as *mut VMTableDefinition, - vmctx_tables.len(), - ); - ptr::copy( - vmctx_memories.values().as_slice().as_ptr(), - instance.memories_ptr() as *mut VMMemoryDefinition, - vmctx_memories.len(), - ); - ptr::copy( - vmctx_globals.values().as_slice().as_ptr(), - instance.globals_ptr() as *mut VMGlobalDefinition, - vmctx_globals.len(), - ); - ptr::write( - instance.builtin_functions_ptr() as *mut VMBuiltinFunctionsArray, - VMBuiltinFunctionsArray::initialized(), - ); - *instance.interrupts() = interrupts; - *instance.externref_activations_table() = externref_activations_table; - *instance.stack_map_registry() = stack_map_registry; - - for (index, sig) in instance.module.functions.iter() { - let type_index = lookup_shared_signature(*sig); - - let (func_ptr, vmctx) = - if let Some(def_index) = instance.module.defined_func_index(index) { - ( - NonNull::new(finished_functions[def_index] as *mut _).unwrap(), - instance.vmctx_ptr(), - ) - } else { - let import = instance.imported_function(index); - (import.body, import.vmctx) - }; - - ptr::write( - instance.anyfunc_ptr(index), - VMCallerCheckedAnyfunc { - func_ptr, - type_index, - vmctx, - }, - ); - } - - // Perform infallible initialization in this constructor, while fallible - // initialization is deferred to the `initialize` method. - initialize_passive_elements(instance); - initialize_globals(instance); - - Ok(handle) - } - - /// Finishes the instantiation process started by `Instance::new`. - /// - /// Only safe to call immediately after instantiation. - pub unsafe fn initialize( - &self, - is_bulk_memory: bool, - data_initializers: &[DataInitializer<'_>], - ) -> Result<(), InstantiationError> { - // Check initializer bounds before initializing anything. Only do this - // when bulk memory is disabled, since the bulk memory proposal changes - // instantiation such that the intermediate results of failed - // initializations are visible. - if !is_bulk_memory { - check_table_init_bounds(self.instance())?; - check_memory_init_bounds(self.instance(), data_initializers)?; - } - - // Apply fallible initializers. Note that this can "leak" state even if - // it fails. - initialize_tables(self.instance())?; - initialize_memories(self.instance(), data_initializers)?; - - Ok(()) + pub(crate) unsafe fn new(instance: *mut Instance) -> Self { + Self { instance } } /// Create a new `InstanceHandle` pointing at the instance @@ -1126,305 +969,4 @@ impl InstanceHandle { instance: self.instance, } } - - /// Deallocates memory associated with this instance. - /// - /// Note that this is unsafe because there might be other handles to this - /// `InstanceHandle` elsewhere, and there's nothing preventing usage of - /// this handle after this function is called. - pub unsafe fn dealloc(&self) { - let instance = self.instance(); - let layout = instance.alloc_layout(); - ptr::drop_in_place(self.instance); - alloc::dealloc(self.instance.cast(), layout); - } -} - -fn check_table_init_bounds(instance: &Instance) -> Result<(), InstantiationError> { - for init in &instance.module().table_elements { - let start = get_table_init_start(init, instance); - let table = instance.get_table(init.table_index); - - let size = usize::try_from(table.size()).unwrap(); - if size < start + init.elements.len() { - return Err(InstantiationError::Link(LinkError( - "table out of bounds: elements segment does not fit".to_owned(), - ))); - } - } - - Ok(()) -} - -/// Compute the offset for a memory data initializer. -fn get_memory_init_start(init: &DataInitializer<'_>, instance: &Instance) -> usize { - let mut start = init.location.offset; - - if let Some(base) = init.location.base { - let val = unsafe { - if let Some(def_index) = instance.module.defined_global_index(base) { - *instance.global(def_index).as_u32() - } else { - *(*instance.imported_global(base).from).as_u32() - } - }; - start += usize::try_from(val).unwrap(); - } - - start -} - -/// Return a byte-slice view of a memory's data. -unsafe fn get_memory_slice<'instance>( - init: &DataInitializer<'_>, - instance: &'instance Instance, -) -> &'instance mut [u8] { - let memory = if let Some(defined_memory_index) = instance - .module - .defined_memory_index(init.location.memory_index) - { - instance.memory(defined_memory_index) - } else { - let import = instance.imported_memory(init.location.memory_index); - let foreign_instance = (&mut *(import).vmctx).instance(); - let foreign_memory = &mut *(import).from; - let foreign_index = foreign_instance.memory_index(foreign_memory); - foreign_instance.memory(foreign_index) - }; - slice::from_raw_parts_mut(memory.base, memory.current_length) -} - -fn check_memory_init_bounds( - instance: &Instance, - data_initializers: &[DataInitializer<'_>], -) -> Result<(), InstantiationError> { - for init in data_initializers { - let start = get_memory_init_start(init, instance); - unsafe { - let mem_slice = get_memory_slice(init, instance); - if mem_slice.get_mut(start..start + init.data.len()).is_none() { - return Err(InstantiationError::Link(LinkError( - "memory out of bounds: data segment does not fit".into(), - ))); - } - } - } - - Ok(()) -} - -/// Allocate memory for just the tables of the current module. -fn create_tables(module: &Module) -> BoxedSlice { - let num_imports = module.num_imported_tables; - let mut tables: PrimaryMap = - PrimaryMap::with_capacity(module.table_plans.len() - num_imports); - for table in &module.table_plans.values().as_slice()[num_imports..] { - tables.push(Table::new(table)); - } - tables.into_boxed_slice() -} - -/// Compute the offset for a table element initializer. -fn get_table_init_start(init: &TableElements, instance: &Instance) -> usize { - let mut start = init.offset; - - if let Some(base) = init.base { - let val = unsafe { - if let Some(def_index) = instance.module.defined_global_index(base) { - *instance.global(def_index).as_u32() - } else { - *(*instance.imported_global(base).from).as_u32() - } - }; - start += usize::try_from(val).unwrap(); - } - - start -} - -/// Initialize the table memory from the provided initializers. -fn initialize_tables(instance: &Instance) -> Result<(), InstantiationError> { - for init in &instance.module().table_elements { - let start = get_table_init_start(init, instance); - let table = instance.get_table(init.table_index); - - if start - .checked_add(init.elements.len()) - .map_or(true, |end| end > table.size() as usize) - { - return Err(InstantiationError::Trap(Trap::wasm( - ir::TrapCode::TableOutOfBounds, - ))); - } - - for (i, func_idx) in init.elements.iter().enumerate() { - let item = match table.element_type() { - TableElementType::Func => instance - .get_caller_checked_anyfunc(*func_idx) - .map_or(ptr::null_mut(), |f: &VMCallerCheckedAnyfunc| { - f as *const VMCallerCheckedAnyfunc as *mut VMCallerCheckedAnyfunc - }) - .into(), - TableElementType::Val(_) => { - assert!(*func_idx == FuncIndex::reserved_value()); - TableElement::ExternRef(None) - } - }; - table.set(u32::try_from(start + i).unwrap(), item).unwrap(); - } - } - - Ok(()) -} - -/// Initialize the `Instance::passive_elements` map by resolving the -/// `Module::passive_elements`'s `FuncIndex`s into `VMCallerCheckedAnyfunc`s for -/// this instance. -fn initialize_passive_elements(instance: &Instance) { - let mut passive_elements = instance.passive_elements.borrow_mut(); - debug_assert!( - passive_elements.is_empty(), - "should only be called once, at initialization time" - ); - - passive_elements.extend( - instance - .module - .passive_elements - .iter() - .filter(|(_, segments)| !segments.is_empty()) - .map(|(idx, segments)| { - ( - *idx, - segments - .iter() - .map(|s| { - instance.get_caller_checked_anyfunc(*s).map_or( - ptr::null_mut(), - |f: &VMCallerCheckedAnyfunc| { - f as *const VMCallerCheckedAnyfunc as *mut _ - }, - ) - }) - .collect(), - ) - }), - ); -} - -/// Allocate memory for just the memories of the current module. -fn create_memories( - module: &Module, - mem_creator: &dyn RuntimeMemoryCreator, -) -> Result>, InstantiationError> { - let num_imports = module.num_imported_memories; - let mut memories: PrimaryMap = - PrimaryMap::with_capacity(module.memory_plans.len() - num_imports); - for plan in &module.memory_plans.values().as_slice()[num_imports..] { - memories.push( - mem_creator - .new_memory(plan) - .map_err(InstantiationError::Resource)?, - ); - } - Ok(memories.into_boxed_slice()) -} - -/// Initialize the table memory from the provided initializers. -fn initialize_memories( - instance: &Instance, - data_initializers: &[DataInitializer<'_>], -) -> Result<(), InstantiationError> { - for init in data_initializers { - let memory = instance.get_memory(init.location.memory_index); - - let start = get_memory_init_start(init, instance); - if start - .checked_add(init.data.len()) - .map_or(true, |end| end > memory.current_length) - { - return Err(InstantiationError::Trap(Trap::wasm( - ir::TrapCode::HeapOutOfBounds, - ))); - } - - unsafe { - let mem_slice = get_memory_slice(init, instance); - let end = start + init.data.len(); - let to_init = &mut mem_slice[start..end]; - to_init.copy_from_slice(init.data); - } - } - - Ok(()) -} - -/// Allocate memory for just the globals of the current module, -/// with initializers applied. -fn create_globals(module: &Module) -> BoxedSlice { - let num_imports = module.num_imported_globals; - let mut vmctx_globals = PrimaryMap::with_capacity(module.globals.len() - num_imports); - - for _ in &module.globals.values().as_slice()[num_imports..] { - vmctx_globals.push(VMGlobalDefinition::new()); - } - - vmctx_globals.into_boxed_slice() -} - -fn initialize_globals(instance: &Instance) { - let module = instance.module(); - let num_imports = module.num_imported_globals; - for (index, global) in module.globals.iter().skip(num_imports) { - let def_index = module.defined_global_index(index).unwrap(); - unsafe { - let to = instance.global_ptr(def_index); - match global.initializer { - GlobalInit::I32Const(x) => *(*to).as_i32_mut() = x, - GlobalInit::I64Const(x) => *(*to).as_i64_mut() = x, - GlobalInit::F32Const(x) => *(*to).as_f32_bits_mut() = x, - GlobalInit::F64Const(x) => *(*to).as_f64_bits_mut() = x, - GlobalInit::V128Const(x) => *(*to).as_u128_bits_mut() = x.0, - GlobalInit::GetGlobal(x) => { - let from = if let Some(def_x) = module.defined_global_index(x) { - instance.global(def_x) - } else { - *instance.imported_global(x).from - }; - *to = from; - } - GlobalInit::RefFunc(f) => { - *(*to).as_anyfunc_mut() = instance.get_caller_checked_anyfunc(f).unwrap() - as *const VMCallerCheckedAnyfunc; - } - GlobalInit::RefNullConst => match global.wasm_ty { - WasmType::FuncRef => *(*to).as_anyfunc_mut() = ptr::null(), - WasmType::ExternRef => *(*to).as_externref_mut() = None, - ty => panic!("unsupported reference type for global: {:?}", ty), - }, - GlobalInit::Import => panic!("locally-defined global initialized as import"), - } - } - } -} - -/// An link error while instantiating a module. -#[derive(Error, Debug)] -#[error("Link error: {0}")] -pub struct LinkError(pub String); - -/// An error while instantiating a module. -#[derive(Error, Debug)] -pub enum InstantiationError { - /// Insufficient resources available for execution. - #[error("Insufficient resources: {0}")] - Resource(String), - - /// A wasm link error occured. - #[error("Failed to link module")] - Link(#[from] LinkError), - - /// A trap ocurred during instantiation, after linking. - #[error("Trap occurred during instantiation")] - Trap(Trap), } diff --git a/crates/runtime/src/instance/allocator.rs b/crates/runtime/src/instance/allocator.rs new file mode 100644 index 000000000000..395b7c120fd0 --- /dev/null +++ b/crates/runtime/src/instance/allocator.rs @@ -0,0 +1,536 @@ +use crate::externref::{StackMapRegistry, VMExternRefActivationsTable}; +use crate::imports::Imports; +use crate::instance::{Instance, InstanceHandle, RuntimeMemoryCreator}; +use crate::memory::{DefaultMemoryCreator, RuntimeLinearMemory}; +use crate::table::{Table, TableElement}; +use crate::traphandlers::Trap; +use crate::vmcontext::{ + VMBuiltinFunctionsArray, VMCallerCheckedAnyfunc, VMContext, VMFunctionBody, VMFunctionImport, + VMGlobalDefinition, VMGlobalImport, VMInterrupts, VMMemoryDefinition, VMMemoryImport, + VMSharedSignatureIndex, VMTableDefinition, VMTableImport, +}; +use std::alloc; +use std::any::Any; +use std::cell::RefCell; +use std::convert::TryFrom; +use std::ptr::{self, NonNull}; +use std::slice; +use std::sync::Arc; +use thiserror::Error; +use wasmtime_environ::entity::{ + packed_option::ReservedValue, BoxedSlice, EntityRef, EntitySet, PrimaryMap, +}; +use wasmtime_environ::wasm::{ + DefinedFuncIndex, DefinedMemoryIndex, DefinedTableIndex, FuncIndex, GlobalInit, SignatureIndex, + TableElementType, WasmType, +}; +use wasmtime_environ::{ir, Module, ModuleType, OwnedDataInitializer, TableElements, VMOffsets}; + +/// Represents a request for a new runtime instance. +pub struct InstanceAllocationRequest<'a> { + /// The module being instantiated. + pub module: Arc, + + /// The finished (JIT) functions for the module. + pub finished_functions: &'a PrimaryMap, + + /// The imports to use for the instantiation. + pub imports: Imports<'a>, + + /// A callback for looking up shared signature indexes. + pub lookup_shared_signature: &'a dyn Fn(SignatureIndex) -> VMSharedSignatureIndex, + + /// The host state to associate with the instance. + pub host_state: Box, + + /// The pointer to the VM interrupts structure to use for the instance. + pub interrupts: *const VMInterrupts, + + /// The pointer to the reference activations table to use for the instance. + pub externref_activations_table: *mut VMExternRefActivationsTable, + + /// The pointer to the stack map registry to use for the instance. + pub stack_map_registry: *mut StackMapRegistry, +} + +/// An link error while instantiating a module. +#[derive(Error, Debug)] +#[error("Link error: {0}")] +pub struct LinkError(pub String); + +/// An error while instantiating a module. +#[derive(Error, Debug)] +pub enum InstantiationError { + /// Insufficient resources available for execution. + #[error("Insufficient resources: {0}")] + Resource(String), + + /// A wasm link error occured. + #[error("Failed to link module")] + Link(#[from] LinkError), + + /// A trap ocurred during instantiation, after linking. + #[error("Trap occurred during instantiation")] + Trap(Trap), +} + +/// Represents a runtime instance allocator. +/// +/// # Safety +/// +/// This trait is unsafe as it requires knowledge of Wasmtime's runtime internals to implement correctly. +pub unsafe trait InstanceAllocator: Send + Sync { + /// Allocates an instance for the given allocation request. + /// + /// # Safety + /// + /// This method is not inherently unsafe, but care must be made to ensure + /// pointers passed in the allocation request outlive the returned instance. + unsafe fn allocate( + &self, + req: InstanceAllocationRequest, + ) -> Result; + + /// Finishes the instantiation process started by an instance allocator. + /// + /// # Safety + /// + /// This method is only safe to call immediately after an instance has been allocated. + unsafe fn initialize( + &self, + handle: &InstanceHandle, + is_bulk_memory: bool, + data_initializers: &Arc<[OwnedDataInitializer]>, + ) -> Result<(), InstantiationError>; + + /// Deallocates a previously allocated instance. + /// + /// # Safety + /// + /// This function is unsafe because there are no guarantees that the given handle + /// is the only owner of the underlying instance to deallocate. + /// + /// Use extreme care when deallocating an instance so that there are no dangling instance pointers. + unsafe fn deallocate(&self, handle: &InstanceHandle); +} + +unsafe fn initialize_vmcontext( + instance: &Instance, + functions: &[VMFunctionImport], + tables: &[VMTableImport], + memories: &[VMMemoryImport], + globals: &[VMGlobalImport], + finished_functions: &PrimaryMap, + lookup_shared_signature: &dyn Fn(SignatureIndex) -> VMSharedSignatureIndex, + interrupts: *const VMInterrupts, + externref_activations_table: *mut VMExternRefActivationsTable, + stack_map_registry: *mut StackMapRegistry, + get_mem_def: impl Fn(DefinedMemoryIndex) -> VMMemoryDefinition, + get_table_def: impl Fn(DefinedTableIndex) -> VMTableDefinition, +) { + let module = &instance.module; + + *instance.interrupts() = interrupts; + *instance.externref_activations_table() = externref_activations_table; + *instance.stack_map_registry() = stack_map_registry; + + // Initialize shared signatures + let mut ptr = instance.signature_ids_ptr(); + for sig in module.types.values() { + *ptr = match sig { + ModuleType::Function(sig) => lookup_shared_signature(*sig), + _ => VMSharedSignatureIndex::new(u32::max_value()), + }; + ptr = ptr.add(1); + } + + // Initialize the built-in functions + ptr::write( + instance.builtin_functions_ptr() as *mut VMBuiltinFunctionsArray, + VMBuiltinFunctionsArray::initialized(), + ); + + // Initialize the imports + debug_assert_eq!(functions.len(), module.num_imported_funcs); + ptr::copy( + functions.as_ptr(), + instance.imported_functions_ptr() as *mut VMFunctionImport, + functions.len(), + ); + debug_assert_eq!(tables.len(), module.num_imported_tables); + ptr::copy( + tables.as_ptr(), + instance.imported_tables_ptr() as *mut VMTableImport, + tables.len(), + ); + debug_assert_eq!(memories.len(), module.num_imported_memories); + ptr::copy( + memories.as_ptr(), + instance.imported_memories_ptr() as *mut VMMemoryImport, + memories.len(), + ); + debug_assert_eq!(globals.len(), module.num_imported_globals); + ptr::copy( + globals.as_ptr(), + instance.imported_globals_ptr() as *mut VMGlobalImport, + globals.len(), + ); + + // Initialize the defined functions + for (index, sig) in instance.module.functions.iter() { + let type_index = lookup_shared_signature(*sig); + + let (func_ptr, vmctx) = if let Some(def_index) = instance.module.defined_func_index(index) { + ( + NonNull::new(finished_functions[def_index] as *mut _).unwrap(), + instance.vmctx_ptr(), + ) + } else { + let import = instance.imported_function(index); + (import.body, import.vmctx) + }; + + ptr::write( + instance.anyfunc_ptr(index), + VMCallerCheckedAnyfunc { + func_ptr, + type_index, + vmctx, + }, + ); + } + + // Initialize the defined tables + let mut ptr = instance.tables_ptr(); + for i in 0..module.table_plans.len() - module.num_imported_tables { + ptr::write(ptr, get_table_def(DefinedTableIndex::new(i))); + ptr = ptr.add(1); + } + + // Initialize the defined memories + let mut ptr = instance.memories_ptr(); + for i in 0..module.memory_plans.len() - module.num_imported_memories { + ptr::write(ptr, get_mem_def(DefinedMemoryIndex::new(i))); + ptr = ptr.add(1); + } + + // Initialize the defined globals + initialize_vmcontext_globals(instance); +} + +unsafe fn initialize_vmcontext_globals(instance: &Instance) { + let module = &instance.module; + let num_imports = module.num_imported_globals; + for (index, global) in module.globals.iter().skip(num_imports) { + let def_index = module.defined_global_index(index).unwrap(); + let to = instance.global_ptr(def_index); + + // Initialize the global before writing to it + ptr::write(to, VMGlobalDefinition::new()); + + match global.initializer { + GlobalInit::I32Const(x) => *(*to).as_i32_mut() = x, + GlobalInit::I64Const(x) => *(*to).as_i64_mut() = x, + GlobalInit::F32Const(x) => *(*to).as_f32_bits_mut() = x, + GlobalInit::F64Const(x) => *(*to).as_f64_bits_mut() = x, + GlobalInit::V128Const(x) => *(*to).as_u128_bits_mut() = x.0, + GlobalInit::GetGlobal(x) => { + let from = if let Some(def_x) = module.defined_global_index(x) { + instance.global(def_x) + } else { + *instance.imported_global(x).from + }; + *to = from; + } + GlobalInit::RefFunc(f) => { + *(*to).as_anyfunc_mut() = instance.get_caller_checked_anyfunc(f).unwrap() + as *const VMCallerCheckedAnyfunc; + } + GlobalInit::RefNullConst => match global.wasm_ty { + WasmType::FuncRef => *(*to).as_anyfunc_mut() = ptr::null(), + WasmType::ExternRef => *(*to).as_externref_mut() = None, + ty => panic!("unsupported reference type for global: {:?}", ty), + }, + GlobalInit::Import => panic!("locally-defined global initialized as import"), + } + } +} + +/// Represents the on-demand instance allocator. +#[derive(Clone)] +pub struct OnDemandInstanceAllocator { + mem_creator: Option>, +} + +impl OnDemandInstanceAllocator { + /// Creates a new on-demand instance allocator. + pub fn new(mem_creator: Option>) -> Self { + Self { mem_creator } + } + + fn create_tables(module: &Module) -> BoxedSlice { + let num_imports = module.num_imported_tables; + let mut tables: PrimaryMap = + PrimaryMap::with_capacity(module.table_plans.len() - num_imports); + for table in &module.table_plans.values().as_slice()[num_imports..] { + tables.push(Table::new(table)); + } + tables.into_boxed_slice() + } + + fn create_memories( + &self, + module: &Module, + ) -> Result>, InstantiationError> + { + let creator = self + .mem_creator + .as_deref() + .unwrap_or_else(|| &DefaultMemoryCreator); + let num_imports = module.num_imported_memories; + let mut memories: PrimaryMap = + PrimaryMap::with_capacity(module.memory_plans.len() - num_imports); + for plan in &module.memory_plans.values().as_slice()[num_imports..] { + memories.push( + creator + .new_memory(plan) + .map_err(InstantiationError::Resource)?, + ); + } + Ok(memories.into_boxed_slice()) + } + + fn check_table_init_bounds(instance: &Instance) -> Result<(), InstantiationError> { + for init in &instance.module.table_elements { + let start = Self::get_table_init_start(init, instance); + let table = instance.get_table(init.table_index); + + let size = usize::try_from(table.size()).unwrap(); + if size < start + init.elements.len() { + return Err(InstantiationError::Link(LinkError( + "table out of bounds: elements segment does not fit".to_owned(), + ))); + } + } + + Ok(()) + } + + fn get_memory_init_start(init: &OwnedDataInitializer, instance: &Instance) -> usize { + let mut start = init.location.offset; + + if let Some(base) = init.location.base { + let val = unsafe { + if let Some(def_index) = instance.module.defined_global_index(base) { + *instance.global(def_index).as_u32() + } else { + *(*instance.imported_global(base).from).as_u32() + } + }; + start += usize::try_from(val).unwrap(); + } + + start + } + + unsafe fn get_memory_slice<'instance>( + init: &OwnedDataInitializer, + instance: &'instance Instance, + ) -> &'instance mut [u8] { + let memory = if let Some(defined_memory_index) = instance + .module + .defined_memory_index(init.location.memory_index) + { + instance.memory(defined_memory_index) + } else { + let import = instance.imported_memory(init.location.memory_index); + let foreign_instance = (&mut *(import).vmctx).instance(); + let foreign_memory = &mut *(import).from; + let foreign_index = foreign_instance.memory_index(foreign_memory); + foreign_instance.memory(foreign_index) + }; + slice::from_raw_parts_mut(memory.base, memory.current_length) + } + + fn check_memory_init_bounds( + instance: &Instance, + data_initializers: &[OwnedDataInitializer], + ) -> Result<(), InstantiationError> { + for init in data_initializers { + let start = Self::get_memory_init_start(init, instance); + unsafe { + let mem_slice = Self::get_memory_slice(init, instance); + if mem_slice.get_mut(start..start + init.data.len()).is_none() { + return Err(InstantiationError::Link(LinkError( + "memory out of bounds: data segment does not fit".into(), + ))); + } + } + } + + Ok(()) + } + + fn get_table_init_start(init: &TableElements, instance: &Instance) -> usize { + let mut start = init.offset; + + if let Some(base) = init.base { + let val = unsafe { + if let Some(def_index) = instance.module.defined_global_index(base) { + *instance.global(def_index).as_u32() + } else { + *(*instance.imported_global(base).from).as_u32() + } + }; + start += usize::try_from(val).unwrap(); + } + + start + } + + fn initialize_tables(instance: &Instance) -> Result<(), InstantiationError> { + for init in &instance.module.table_elements { + let start = Self::get_table_init_start(init, instance); + let table = instance.get_table(init.table_index); + + if start + .checked_add(init.elements.len()) + .map_or(true, |end| end > table.size() as usize) + { + return Err(InstantiationError::Trap(Trap::wasm( + ir::TrapCode::TableOutOfBounds, + ))); + } + + for (i, func_idx) in init.elements.iter().enumerate() { + let item = match table.element_type() { + TableElementType::Func => instance + .get_caller_checked_anyfunc(*func_idx) + .map_or(ptr::null_mut(), |f: &VMCallerCheckedAnyfunc| { + f as *const VMCallerCheckedAnyfunc as *mut VMCallerCheckedAnyfunc + }) + .into(), + TableElementType::Val(_) => { + assert!(*func_idx == FuncIndex::reserved_value()); + TableElement::ExternRef(None) + } + }; + table.set(u32::try_from(start + i).unwrap(), item).unwrap(); + } + } + + Ok(()) + } + + /// Initialize the table memory from the provided initializers. + fn initialize_memories( + instance: &Instance, + data_initializers: &[OwnedDataInitializer], + ) -> Result<(), InstantiationError> { + for init in data_initializers { + let memory = instance.get_memory(init.location.memory_index); + + let start = Self::get_memory_init_start(init, instance); + if start + .checked_add(init.data.len()) + .map_or(true, |end| end > memory.current_length) + { + return Err(InstantiationError::Trap(Trap::wasm( + ir::TrapCode::HeapOutOfBounds, + ))); + } + + unsafe { + let mem_slice = Self::get_memory_slice(init, instance); + let end = start + init.data.len(); + let to_init = &mut mem_slice[start..end]; + to_init.copy_from_slice(&init.data); + } + } + + Ok(()) + } +} + +unsafe impl InstanceAllocator for OnDemandInstanceAllocator { + unsafe fn allocate( + &self, + req: InstanceAllocationRequest, + ) -> Result { + debug_assert!(!req.externref_activations_table.is_null()); + debug_assert!(!req.stack_map_registry.is_null()); + + let memories = self.create_memories(&req.module)?; + let tables = Self::create_tables(&req.module); + + let handle = { + let instance = Instance { + module: req.module.clone(), + offsets: VMOffsets::new(std::mem::size_of::<*const u8>() as u8, &req.module), + memories, + tables, + dropped_elements: RefCell::new(EntitySet::with_capacity( + req.module.passive_elements.len(), + )), + dropped_data: RefCell::new(EntitySet::with_capacity(req.module.passive_data.len())), + host_state: req.host_state, + vmctx: VMContext {}, + }; + let layout = instance.alloc_layout(); + let instance_ptr = alloc::alloc(layout) as *mut Instance; + if instance_ptr.is_null() { + alloc::handle_alloc_error(layout); + } + ptr::write(instance_ptr, instance); + InstanceHandle::new(instance_ptr) + }; + + let instance = handle.instance(); + initialize_vmcontext( + instance, + req.imports.functions, + req.imports.tables, + req.imports.memories, + req.imports.globals, + req.finished_functions, + req.lookup_shared_signature, + req.interrupts, + req.externref_activations_table, + req.stack_map_registry, + &|index| instance.memories[index].vmmemory(), + &|index| instance.tables[index].vmtable(), + ); + + Ok(handle) + } + + unsafe fn initialize( + &self, + handle: &InstanceHandle, + is_bulk_memory: bool, + data_initializers: &Arc<[OwnedDataInitializer]>, + ) -> Result<(), InstantiationError> { + // Check initializer bounds before initializing anything. Only do this + // when bulk memory is disabled, since the bulk memory proposal changes + // instantiation such that the intermediate results of failed + // initializations are visible. + if !is_bulk_memory { + Self::check_table_init_bounds(handle.instance())?; + Self::check_memory_init_bounds(handle.instance(), data_initializers.as_ref())?; + } + + // Apply fallible initializers. Note that this can "leak" state even if + // it fails. + Self::initialize_tables(handle.instance())?; + Self::initialize_memories(handle.instance(), data_initializers.as_ref())?; + + Ok(()) + } + + unsafe fn deallocate(&self, handle: &InstanceHandle) { + let instance = handle.instance(); + let layout = instance.alloc_layout(); + ptr::drop_in_place(instance as *const Instance as *mut Instance); + alloc::dealloc(instance as *const Instance as *mut _, layout); + } +} diff --git a/crates/runtime/src/lib.rs b/crates/runtime/src/lib.rs index 00539c36b4c5..eff31fdeead5 100644 --- a/crates/runtime/src/lib.rs +++ b/crates/runtime/src/lib.rs @@ -37,7 +37,10 @@ pub mod libcalls; pub use crate::export::*; pub use crate::externref::*; pub use crate::imports::Imports; -pub use crate::instance::{InstanceHandle, InstantiationError, LinkError, RuntimeInstance}; +pub use crate::instance::{ + InstanceAllocationRequest, InstanceAllocator, InstanceHandle, InstantiationError, LinkError, + OnDemandInstanceAllocator, RuntimeInstance, +}; pub use crate::jit_int::GdbJitImageRegistration; pub use crate::memory::{RuntimeLinearMemory, RuntimeMemoryCreator}; pub use crate::mmap::Mmap; diff --git a/crates/runtime/src/vmcontext.rs b/crates/runtime/src/vmcontext.rs index c20a42b5ed4a..f4dffeee2c3b 100644 --- a/crates/runtime/src/vmcontext.rs +++ b/crates/runtime/src/vmcontext.rs @@ -750,7 +750,7 @@ impl VMContext { } } -/// +/// Trampoline function pointer type. pub type VMTrampoline = unsafe extern "C" fn( *mut VMContext, // callee vmctx *mut VMContext, // caller vmctx diff --git a/crates/wasmtime/src/config.rs b/crates/wasmtime/src/config.rs index 6ddaa022c4d7..60adfb3005f7 100644 --- a/crates/wasmtime/src/config.rs +++ b/crates/wasmtime/src/config.rs @@ -14,6 +14,25 @@ use wasmtime_environ::settings::{self, Configurable, SetError}; use wasmtime_environ::{isa, isa::TargetIsa, Tunables}; use wasmtime_jit::{native, CompilationStrategy, Compiler}; use wasmtime_profiling::{JitDumpAgent, NullProfilerAgent, ProfilingAgent, VTuneAgent}; +use wasmtime_runtime::{InstanceAllocator, OnDemandInstanceAllocator}; + +/// Represents the module instance allocation strategy to use. +#[derive(Clone)] +pub enum InstanceAllocationStrategy { + /// The on-demand instance allocation strategy. + /// + /// Resources related to a module instance are allocated at instantiation time and + /// immediately deallocated when the `Store` referencing the instance is dropped. + /// + /// This is the default allocation strategy for Wasmtime. + OnDemand, +} + +impl Default for InstanceAllocationStrategy { + fn default() -> Self { + Self::OnDemand + } +} /// Global configuration options used to create an [`Engine`](crate::Engine) /// and customize its behavior. @@ -29,7 +48,10 @@ pub struct Config { #[cfg(feature = "cache")] pub(crate) cache_config: CacheConfig, pub(crate) profiler: Arc, - pub(crate) memory_creator: Option, + pub(crate) instance_allocator: Option>, + // The default instance allocator is used for instantiating host objects + // and for module instatiation when `instance_allocator` is None + pub(crate) default_instance_allocator: OnDemandInstanceAllocator, pub(crate) max_wasm_stack: usize, pub(crate) features: WasmFeatures, pub(crate) wasm_backtrace_details_env_used: bool, @@ -73,7 +95,8 @@ impl Config { #[cfg(feature = "cache")] cache_config: CacheConfig::new_cache_disabled(), profiler: Arc::new(NullProfilerAgent), - memory_creator: None, + instance_allocator: None, + default_instance_allocator: OnDemandInstanceAllocator::new(None), max_wasm_stack: 1 << 20, wasm_backtrace_details_env_used: false, features: WasmFeatures { @@ -504,9 +527,24 @@ impl Config { Ok(self) } - /// Sets a custom memory creator + /// Sets a custom memory creator. + /// + /// Custom memory creators are used when creating host `Memory` objects or when + /// creating instance linear memories for the on-demand instance allocation strategy. pub fn with_host_memory(&mut self, mem_creator: Arc) -> &mut Self { - self.memory_creator = Some(MemoryCreatorProxy { mem_creator }); + self.default_instance_allocator = + OnDemandInstanceAllocator::new(Some(Arc::new(MemoryCreatorProxy(mem_creator)))); + self + } + + /// Sets the instance allocation strategy to use. + pub fn with_instance_allocation_strategy( + &mut self, + strategy: InstanceAllocationStrategy, + ) -> &mut Self { + self.instance_allocator = match strategy { + InstanceAllocationStrategy::OnDemand => None, + }; self } @@ -728,6 +766,12 @@ impl Config { let isa = self.target_isa(); Compiler::new(isa, self.strategy, self.tunables.clone(), self.features) } + + pub(crate) fn instance_allocator(&self) -> &dyn InstanceAllocator { + self.instance_allocator + .as_deref() + .unwrap_or(&self.default_instance_allocator) + } } fn round_up_to_pages(val: u64) -> u64 { diff --git a/crates/wasmtime/src/instance.rs b/crates/wasmtime/src/instance.rs index 2ddaee2a6834..87936fd535ee 100644 --- a/crates/wasmtime/src/instance.rs +++ b/crates/wasmtime/src/instance.rs @@ -12,9 +12,9 @@ use wasmtime_environ::wasm::{ }; use wasmtime_environ::Initializer; use wasmtime_runtime::{ - Imports, InstantiationError, RuntimeInstance, StackMapRegistry, VMContext, - VMExternRefActivationsTable, VMFunctionBody, VMFunctionImport, VMGlobalImport, VMMemoryImport, - VMTableImport, + Imports, InstanceAllocationRequest, InstantiationError, RuntimeInstance, StackMapRegistry, + VMContext, VMExternRefActivationsTable, VMFunctionBody, VMFunctionImport, VMGlobalImport, + VMMemoryImport, VMTableImport, }; /// An instantiated WebAssembly module. @@ -492,18 +492,26 @@ impl<'a> Instantiator<'a> { // compiled JIT code within the `Store`. self.store.register_module(&self.cur.module); - let config = self.store.engine().config(); unsafe { - let instance = compiled_module.instantiate( - self.cur.build(), - &self.store.lookup_shared_signature(self.cur.module.types()), - config.memory_creator.as_ref().map(|a| a as _), - self.store.interrupts(), - Box::new(()), - self.store.externref_activations_table() as *const VMExternRefActivationsTable + let config = self.store.engine().config(); + + let allocator = config.instance_allocator(); + + let instance = allocator.allocate(InstanceAllocationRequest { + module: compiled_module.module().clone(), + finished_functions: compiled_module.finished_functions(), + imports: self.cur.build(), + lookup_shared_signature: &self + .store + .lookup_shared_signature(self.cur.module.types()), + host_state: Box::new(()), + interrupts: self.store.interrupts(), + externref_activations_table: self.store.externref_activations_table() + as *const VMExternRefActivationsTable as *mut _, - self.store.stack_map_registry() as *const StackMapRegistry as *mut _, - )?; + stack_map_registry: self.store.stack_map_registry() as *const StackMapRegistry + as *mut _, + })?; // After we've created the `InstanceHandle` we still need to run // initialization to set up data/elements/etc. We do this after adding @@ -513,8 +521,9 @@ impl<'a> Instantiator<'a> { // tables. This means that from this point on, regardless of whether // initialization is successful, we need to keep the instance alive. let instance = self.store.add_instance(instance); - instance + allocator .initialize( + &instance.handle, config.features.bulk_memory, &compiled_module.data_initializers(), ) diff --git a/crates/wasmtime/src/store.rs b/crates/wasmtime/src/store.rs index 7945270a3519..dbaa5eb44c05 100644 --- a/crates/wasmtime/src/store.rs +++ b/crates/wasmtime/src/store.rs @@ -18,8 +18,8 @@ use std::task::{Context, Poll}; use wasmtime_environ::wasm; use wasmtime_jit::{CompiledModule, ModuleCode, TypeTables}; use wasmtime_runtime::{ - InstanceHandle, RuntimeMemoryCreator, SignalHandler, StackMapRegistry, TrapInfo, VMContext, - VMExternRef, VMExternRefActivationsTable, VMInterrupts, VMSharedSignatureIndex, + InstanceHandle, SignalHandler, StackMapRegistry, TrapInfo, VMContext, VMExternRef, + VMExternRefActivationsTable, VMInterrupts, VMSharedSignatureIndex, }; /// A `Store` is a collection of WebAssembly instances and host-defined items. @@ -254,15 +254,6 @@ impl Store { &self.inner.engine } - /// Returns an optional reference to a ['RuntimeMemoryCreator'] - pub(crate) fn memory_creator(&self) -> Option<&dyn RuntimeMemoryCreator> { - self.engine() - .config() - .memory_creator - .as_ref() - .map(|x| x as _) - } - pub(crate) fn signatures(&self) -> &RefCell { &self.inner.signatures } @@ -969,9 +960,10 @@ impl fmt::Debug for Store { impl Drop for StoreInner { fn drop(&mut self) { - for instance in self.instances.get_mut().iter() { + let allocator = self.engine.config().instance_allocator(); + for instance in self.instances.borrow().iter() { unsafe { - instance.dealloc(); + allocator.deallocate(instance); } } } diff --git a/crates/wasmtime/src/trampoline/create_handle.rs b/crates/wasmtime/src/trampoline/create_handle.rs index f597987e1b7b..ff088b9aa17d 100644 --- a/crates/wasmtime/src/trampoline/create_handle.rs +++ b/crates/wasmtime/src/trampoline/create_handle.rs @@ -9,15 +9,15 @@ use wasmtime_environ::entity::PrimaryMap; use wasmtime_environ::wasm::DefinedFuncIndex; use wasmtime_environ::Module; use wasmtime_runtime::{ - Imports, InstanceHandle, StackMapRegistry, VMExternRefActivationsTable, VMFunctionBody, - VMFunctionImport, VMSharedSignatureIndex, + Imports, InstanceAllocationRequest, InstanceAllocator, StackMapRegistry, + VMExternRefActivationsTable, VMFunctionBody, VMFunctionImport, VMSharedSignatureIndex, }; pub(crate) fn create_handle( module: Module, store: &Store, finished_functions: PrimaryMap, - state: Box, + host_state: Box, func_imports: &[VMFunctionImport], shared_signature_id: Option, ) -> Result { @@ -26,17 +26,24 @@ pub(crate) fn create_handle( let module = Arc::new(module); unsafe { - let handle = InstanceHandle::new( - module, - &finished_functions, - imports, - store.memory_creator(), - &|_| shared_signature_id.unwrap(), - state, - store.interrupts(), - store.externref_activations_table() as *const VMExternRefActivationsTable as *mut _, - store.stack_map_registry() as *const StackMapRegistry as *mut _, - )?; + // Use the default allocator when creating handles associated with host objects + let handle = store + .engine() + .config() + .default_instance_allocator + .allocate(InstanceAllocationRequest { + module: module.clone(), + finished_functions: &finished_functions, + imports, + lookup_shared_signature: &|_| shared_signature_id.unwrap(), + host_state, + interrupts: store.interrupts(), + externref_activations_table: store.externref_activations_table() + as *const VMExternRefActivationsTable + as *mut _, + stack_map_registry: store.stack_map_registry() as *const StackMapRegistry as *mut _, + })?; + Ok(store.add_instance(handle)) } } diff --git a/crates/wasmtime/src/trampoline/memory.rs b/crates/wasmtime/src/trampoline/memory.rs index 0bfb2bfff9b8..af2b9547920b 100644 --- a/crates/wasmtime/src/trampoline/memory.rs +++ b/crates/wasmtime/src/trampoline/memory.rs @@ -54,9 +54,7 @@ impl RuntimeLinearMemory for LinearMemoryProxy { } #[derive(Clone)] -pub(crate) struct MemoryCreatorProxy { - pub(crate) mem_creator: Arc, -} +pub(crate) struct MemoryCreatorProxy(pub Arc); impl RuntimeMemoryCreator for MemoryCreatorProxy { fn new_memory(&self, plan: &MemoryPlan) -> Result, String> { @@ -65,7 +63,7 @@ impl RuntimeMemoryCreator for MemoryCreatorProxy { MemoryStyle::Static { bound } => Some(bound as u64 * WASM_PAGE_SIZE as u64), MemoryStyle::Dynamic => None, }; - self.mem_creator + self.0 .new_memory(ty, reserved_size_in_bytes, plan.offset_guard_size) .map(|mem| Box::new(LinearMemoryProxy { mem }) as Box) } From c8871ee1e6b10d635ee2f36f397c72153f1d4543 Mon Sep 17 00:00:00 2001 From: Peter Huene Date: Mon, 7 Dec 2020 22:12:33 -0800 Subject: [PATCH 02/33] Allow instance allocators control over module compilation. This commit introduces two new methods on `InstanceAllocator`: * `validate_module` - this method is used to validate a module after translation but before compilation. It will be used for the upcoming pooling allocator to ensure a module being compiled adheres to the limits of the allocator. * `adjust_tunables` - this method is used to adjust the `Tunables` given the JIT compiler. The pooling allocator will use this to force all memories to be static during compilation. --- crates/cache/src/lib.rs | 2 +- crates/cache/src/tests.rs | 80 ++++++++++++++++++------ crates/environ/src/module.rs | 12 +++- crates/environ/src/tunables.rs | 4 ++ crates/jit/src/instantiate.rs | 3 + crates/runtime/src/instance/allocator.rs | 19 +++++- crates/runtime/src/memory.rs | 14 ----- crates/wasmtime/src/config.rs | 4 +- crates/wasmtime/src/module.rs | 19 ++++-- 9 files changed, 112 insertions(+), 45 deletions(-) diff --git a/crates/cache/src/lib.rs b/crates/cache/src/lib.rs index 9fe69c5bafd2..fc50ff4dad7c 100644 --- a/crates/cache/src/lib.rs +++ b/crates/cache/src/lib.rs @@ -43,7 +43,7 @@ impl<'config> ModuleCacheEntry<'config> { } /// Gets cached data if state matches, otherwise calls the `compute`. - pub fn get_data(&self, state: T, compute: fn(T) -> Result) -> Result + pub fn get_data(&self, state: T, compute: impl Fn(T) -> Result) -> Result where T: Hash, U: Serialize + for<'a> Deserialize<'a>, diff --git a/crates/cache/src/tests.rs b/crates/cache/src/tests.rs index 4362aaba2222..857a1f0ab76b 100644 --- a/crates/cache/src/tests.rs +++ b/crates/cache/src/tests.rs @@ -65,28 +65,68 @@ fn test_write_read_cache() { let entry1 = ModuleCacheEntry::from_inner(ModuleCacheEntryInner::new(compiler1, &cache_config)); let entry2 = ModuleCacheEntry::from_inner(ModuleCacheEntryInner::new(compiler2, &cache_config)); - entry1.get_data::<_, i32, i32>(1, |_| Ok(100)).unwrap(); - entry1.get_data::<_, i32, i32>(1, |_| panic!()).unwrap(); + entry1 + .get_data(1, |_| -> Result { Ok(100) }) + .unwrap(); + entry1 + .get_data(1, |_| -> Result { panic!() }) + .unwrap(); - entry1.get_data::<_, i32, i32>(2, |_| Ok(100)).unwrap(); - entry1.get_data::<_, i32, i32>(1, |_| panic!()).unwrap(); - entry1.get_data::<_, i32, i32>(2, |_| panic!()).unwrap(); + entry1 + .get_data(2, |_| -> Result { Ok(100) }) + .unwrap(); + entry1 + .get_data(1, |_| -> Result { panic!() }) + .unwrap(); + entry1 + .get_data(2, |_| -> Result { panic!() }) + .unwrap(); - entry1.get_data::<_, i32, i32>(3, |_| Ok(100)).unwrap(); - entry1.get_data::<_, i32, i32>(1, |_| panic!()).unwrap(); - entry1.get_data::<_, i32, i32>(2, |_| panic!()).unwrap(); - entry1.get_data::<_, i32, i32>(3, |_| panic!()).unwrap(); + entry1 + .get_data(3, |_| -> Result { Ok(100) }) + .unwrap(); + entry1 + .get_data(1, |_| -> Result { panic!() }) + .unwrap(); + entry1 + .get_data(2, |_| -> Result { panic!() }) + .unwrap(); + entry1 + .get_data(3, |_| -> Result { panic!() }) + .unwrap(); - entry1.get_data::<_, i32, i32>(4, |_| Ok(100)).unwrap(); - entry1.get_data::<_, i32, i32>(1, |_| panic!()).unwrap(); - entry1.get_data::<_, i32, i32>(2, |_| panic!()).unwrap(); - entry1.get_data::<_, i32, i32>(3, |_| panic!()).unwrap(); - entry1.get_data::<_, i32, i32>(4, |_| panic!()).unwrap(); + entry1 + .get_data(4, |_| -> Result { Ok(100) }) + .unwrap(); + entry1 + .get_data(1, |_| -> Result { panic!() }) + .unwrap(); + entry1 + .get_data(2, |_| -> Result { panic!() }) + .unwrap(); + entry1 + .get_data(3, |_| -> Result { panic!() }) + .unwrap(); + entry1 + .get_data(4, |_| -> Result { panic!() }) + .unwrap(); - entry2.get_data::<_, i32, i32>(1, |_| Ok(100)).unwrap(); - entry1.get_data::<_, i32, i32>(1, |_| panic!()).unwrap(); - entry1.get_data::<_, i32, i32>(2, |_| panic!()).unwrap(); - entry1.get_data::<_, i32, i32>(3, |_| panic!()).unwrap(); - entry1.get_data::<_, i32, i32>(4, |_| panic!()).unwrap(); - entry2.get_data::<_, i32, i32>(1, |_| panic!()).unwrap(); + entry2 + .get_data(1, |_| -> Result { Ok(100) }) + .unwrap(); + entry1 + .get_data(1, |_| -> Result { panic!() }) + .unwrap(); + entry1 + .get_data(2, |_| -> Result { panic!() }) + .unwrap(); + entry1 + .get_data(3, |_| -> Result { panic!() }) + .unwrap(); + entry1 + .get_data(4, |_| -> Result { panic!() }) + .unwrap(); + entry2 + .get_data(1, |_| -> Result { panic!() }) + .unwrap(); } diff --git a/crates/environ/src/module.rs b/crates/environ/src/module.rs index 957feb55f0a7..7282501e4a59 100644 --- a/crates/environ/src/module.rs +++ b/crates/environ/src/module.rs @@ -42,8 +42,16 @@ impl MemoryStyle { // A heap with a maximum that doesn't exceed the static memory bound specified by the // tunables make it static. // - // If the module doesn't declare an explicit maximum treat it as 4GiB. - let maximum = memory.maximum.unwrap_or(WASM_MAX_PAGES); + // If the module doesn't declare an explicit maximum treat it as 4GiB when not + // requested to use the static memory bound itself as the maximum. + let maximum = memory + .maximum + .unwrap_or(if tunables.static_memory_bound_is_maximum { + tunables.static_memory_bound + } else { + WASM_MAX_PAGES + }); + if maximum <= tunables.static_memory_bound { assert_ge!(tunables.static_memory_bound, memory.minimum); return ( diff --git a/crates/environ/src/tunables.rs b/crates/environ/src/tunables.rs index bd86fef237f9..4e5aba91450e 100644 --- a/crates/environ/src/tunables.rs +++ b/crates/environ/src/tunables.rs @@ -27,6 +27,9 @@ pub struct Tunables { /// Whether or not fuel is enabled for generated code, meaning that fuel /// will be consumed every time a wasm instruction is executed. pub consume_fuel: bool, + + /// Whether or not to treat the static memory bound as the maximum for unbounded heaps. + pub static_memory_bound_is_maximum: bool, } impl Default for Tunables { @@ -62,6 +65,7 @@ impl Default for Tunables { parse_wasm_debuginfo: true, interruptable: false, consume_fuel: false, + static_memory_bound_is_maximum: false, } } } diff --git a/crates/jit/src/instantiate.rs b/crates/jit/src/instantiate.rs index a1647cde5a9b..4d43f43ba12b 100644 --- a/crates/jit/src/instantiate.rs +++ b/crates/jit/src/instantiate.rs @@ -101,6 +101,7 @@ impl CompilationArtifacts { pub fn build( compiler: &Compiler, data: &[u8], + validate: impl Fn(&ModuleTranslation) -> Result<(), String> + Sync, ) -> Result<(usize, Vec, TypeTables), SetupError> { let (main_module, translations, types) = ModuleEnvironment::new( compiler.frontend_config(), @@ -112,6 +113,8 @@ impl CompilationArtifacts { let list = maybe_parallel!(translations.(into_iter | into_par_iter)) .map(|mut translation| { + validate(&translation).map_err(|e| SetupError::Validate(e))?; + let Compilation { obj, unwind_info, diff --git a/crates/runtime/src/instance/allocator.rs b/crates/runtime/src/instance/allocator.rs index 395b7c120fd0..bda83832b6af 100644 --- a/crates/runtime/src/instance/allocator.rs +++ b/crates/runtime/src/instance/allocator.rs @@ -24,7 +24,9 @@ use wasmtime_environ::wasm::{ DefinedFuncIndex, DefinedMemoryIndex, DefinedTableIndex, FuncIndex, GlobalInit, SignatureIndex, TableElementType, WasmType, }; -use wasmtime_environ::{ir, Module, ModuleType, OwnedDataInitializer, TableElements, VMOffsets}; +use wasmtime_environ::{ + ir, Module, ModuleTranslation, ModuleType, OwnedDataInitializer, TableElements, VMOffsets, +}; /// Represents a request for a new runtime instance. pub struct InstanceAllocationRequest<'a> { @@ -80,6 +82,21 @@ pub enum InstantiationError { /// /// This trait is unsafe as it requires knowledge of Wasmtime's runtime internals to implement correctly. pub unsafe trait InstanceAllocator: Send + Sync { + /// Validates a module translation. + /// + /// This is used to ensure a module being compiled is supported by the instance allocator. + fn validate_module(&self, translation: &ModuleTranslation) -> Result<(), String> { + drop(translation); + Ok(()) + } + + /// Adjusts the tunables prior to creation of any JIT compiler. + /// + /// This method allows the instance allocator control over tunables passed to a `wasmtime_jit::Compiler`. + fn adjust_tunables(&self, tunables: &mut wasmtime_environ::Tunables) { + drop(tunables); + } + /// Allocates an instance for the given allocation request. /// /// # Safety diff --git a/crates/runtime/src/memory.rs b/crates/runtime/src/memory.rs index 340feb31304d..a2c03e1d31d3 100644 --- a/crates/runtime/src/memory.rs +++ b/crates/runtime/src/memory.rs @@ -52,10 +52,6 @@ pub struct MmapMemory { // Size in bytes of extra guard pages after the end to optimize loads and stores with // constant offsets. offset_guard_size: usize, - - // Records whether we're using a bounds-checking strategy which requires - // handlers to catch trapping accesses. - pub(crate) needs_signal_handlers: bool, } #[derive(Debug)] @@ -75,15 +71,6 @@ impl MmapMemory { let offset_guard_bytes = plan.offset_guard_size as usize; - // If we have an offset guard, or if we're doing the static memory - // allocation strategy, we need signal handlers to catch out of bounds - // acceses. - let needs_signal_handlers = offset_guard_bytes > 0 - || match plan.style { - MemoryStyle::Dynamic => false, - MemoryStyle::Static { .. } => true, - }; - let minimum_pages = match plan.style { MemoryStyle::Dynamic => plan.memory.minimum, MemoryStyle::Static { bound } => { @@ -105,7 +92,6 @@ impl MmapMemory { mmap: mmap.into(), maximum: plan.memory.maximum, offset_guard_size: offset_guard_bytes, - needs_signal_handlers, }) } } diff --git a/crates/wasmtime/src/config.rs b/crates/wasmtime/src/config.rs index 60adfb3005f7..324b03501d13 100644 --- a/crates/wasmtime/src/config.rs +++ b/crates/wasmtime/src/config.rs @@ -764,7 +764,9 @@ impl Config { pub(crate) fn build_compiler(&self) -> Compiler { let isa = self.target_isa(); - Compiler::new(isa, self.strategy, self.tunables.clone(), self.features) + let mut tunables = self.tunables.clone(); + self.instance_allocator().adjust_tunables(&mut tunables); + Compiler::new(isa, self.strategy, tunables, self.features) } pub(crate) fn instance_allocator(&self) -> &dyn InstanceAllocator { diff --git a/crates/wasmtime/src/module.rs b/crates/wasmtime/src/module.rs index 0bcf8f568657..a9d32ee1d17d 100644 --- a/crates/wasmtime/src/module.rs +++ b/crates/wasmtime/src/module.rs @@ -307,15 +307,22 @@ impl Module { /// # } /// ``` pub fn from_binary(engine: &Engine, binary: &[u8]) -> Result { + // Check with the instance allocator to see if the given module is supported + let allocator = engine.config().instance_allocator(); + #[cfg(feature = "cache")] - let (main_module, artifacts, types) = - ModuleCacheEntry::new("wasmtime", engine.cache_config()) - .get_data((engine.compiler(), binary), |(compiler, binary)| { - CompilationArtifacts::build(compiler, binary) - })?; + let (main_module, artifacts, types) = ModuleCacheEntry::new( + "wasmtime", + engine.cache_config(), + ) + .get_data((engine.compiler(), binary), |(compiler, binary)| { + CompilationArtifacts::build(compiler, binary, |m| allocator.validate_module(m)) + })?; #[cfg(not(feature = "cache"))] let (main_module, artifacts, types) = - CompilationArtifacts::build(engine.compiler(), binary)?; + CompilationArtifacts::build(engine.compiler(), binary, |m| { + allocator.validate_module(m) + })?; let mut modules = CompiledModule::from_artifacts_list( artifacts, From f0d93d102ccf56c465b0dc5fc603faab20b357fd Mon Sep 17 00:00:00 2001 From: Peter Huene Date: Mon, 7 Dec 2020 22:26:38 -0800 Subject: [PATCH 03/33] Refactor runtime `Table` to support static storage. This commit refactors `Table` in the runtime such that it can be created from a pointer to existing table data. The current `Vec` backing of the `Table` is considered to be "dynamic" storage. This will be used for the upcoming pooling allocator where table memory is managed externally to the instance. The `table.copy` implementation was improved to use slice primitives for doing the copying. Fixes #983. --- crates/runtime/src/instance/allocator.rs | 2 +- crates/runtime/src/table.rs | 388 +++++++++++++++++------ 2 files changed, 291 insertions(+), 99 deletions(-) diff --git a/crates/runtime/src/instance/allocator.rs b/crates/runtime/src/instance/allocator.rs index bda83832b6af..ba00aedaf91c 100644 --- a/crates/runtime/src/instance/allocator.rs +++ b/crates/runtime/src/instance/allocator.rs @@ -290,7 +290,7 @@ impl OnDemandInstanceAllocator { let mut tables: PrimaryMap = PrimaryMap::with_capacity(module.table_plans.len() - num_imports); for table in &module.table_plans.values().as_slice()[num_imports..] { - tables.push(Table::new(table)); + tables.push(Table::new_dynamic(table)); } tables.into_boxed_slice() } diff --git a/crates/runtime/src/table.rs b/crates/runtime/src/table.rs index 462480bc7672..4a2d0bd4de0e 100644 --- a/crates/runtime/src/table.rs +++ b/crates/runtime/src/table.rs @@ -4,19 +4,13 @@ use crate::vmcontext::{VMCallerCheckedAnyfunc, VMTableDefinition}; use crate::{Trap, VMExternRef}; -use std::cell::RefCell; +use std::cell::{Cell, RefCell}; +use std::cmp::min; use std::convert::{TryFrom, TryInto}; use std::ptr; use wasmtime_environ::wasm::TableElementType; use wasmtime_environ::{ir, TablePlan, TableStyle}; -/// A table instance. -#[derive(Debug)] -pub struct Table { - elements: RefCell, - maximum: Option, -} - /// An element going into or coming out of a table. #[derive(Clone, Debug)] pub enum TableElement { @@ -26,15 +20,75 @@ pub enum TableElement { ExternRef(Option), } +impl TryFrom for *mut VMCallerCheckedAnyfunc { + type Error = (); + + fn try_from(e: TableElement) -> Result { + match e { + TableElement::FuncRef(f) => Ok(f), + _ => Err(()), + } + } +} + +impl TryFrom for Option { + type Error = (); + + fn try_from(e: TableElement) -> Result { + match e { + TableElement::ExternRef(x) => Ok(x), + _ => Err(()), + } + } +} + +impl From<*mut VMCallerCheckedAnyfunc> for TableElement { + fn from(f: *mut VMCallerCheckedAnyfunc) -> TableElement { + TableElement::FuncRef(f) + } +} + +impl From> for TableElement { + fn from(x: Option) -> TableElement { + TableElement::ExternRef(x) + } +} + +impl From for TableElement { + fn from(x: VMExternRef) -> TableElement { + TableElement::ExternRef(Some(x)) + } +} + #[derive(Debug)] enum TableElements { FuncRefs(Vec<*mut VMCallerCheckedAnyfunc>), ExternRefs(Vec>), } +#[derive(Debug)] +enum TableStorage { + Static { + data: *mut u8, + size: Cell, + ty: TableElementType, + maximum: u32, + }, + Dynamic { + elements: RefCell, + maximum: Option, + }, +} + +/// Represents an instance's table. +#[derive(Debug)] +pub struct Table { + storage: TableStorage, +} + impl Table { - /// Create a new table instance with specified minimum and maximum number of elements. - pub fn new(plan: &TablePlan) -> Self { + /// Create a new dynamic (movable) table instance for the specified table plan. + pub fn new_dynamic(plan: &TablePlan) -> Self { let min = usize::try_from(plan.table.minimum).unwrap(); let elements = RefCell::new(match plan.table.ty { TableElementType::Func => TableElements::FuncRefs(vec![ptr::null_mut(); min]), @@ -43,27 +97,58 @@ impl Table { TableElements::ExternRefs(vec![None; min]) } }); + + match plan.style { + TableStyle::CallerChecksSignature => Self { + storage: TableStorage::Dynamic { + elements, + maximum: plan.table.maximum, + }, + }, + } + } + + /// Create a new static (immovable) table instance for the specified table plan. + pub fn new_static(plan: &TablePlan, data: *mut u8, maximum: u32) -> Self { match plan.style { TableStyle::CallerChecksSignature => Self { - elements, - maximum: plan.table.maximum, + storage: TableStorage::Static { + data, + size: Cell::new(plan.table.minimum), + ty: plan.table.ty.clone(), + maximum: min(plan.table.maximum.unwrap_or(maximum), maximum), + }, }, } } /// Returns the type of the elements in this table. pub fn element_type(&self) -> TableElementType { - match &*self.elements.borrow() { - TableElements::FuncRefs(_) => TableElementType::Func, - TableElements::ExternRefs(_) => TableElementType::Val(crate::ref_type()), + match &self.storage { + TableStorage::Static { ty, .. } => *ty, + TableStorage::Dynamic { elements, .. } => match &*elements.borrow() { + TableElements::FuncRefs(_) => TableElementType::Func, + TableElements::ExternRefs(_) => TableElementType::Val(crate::ref_type()), + }, } } /// Returns the number of allocated elements. pub fn size(&self) -> u32 { - match &*self.elements.borrow() { - TableElements::FuncRefs(x) => x.len().try_into().unwrap(), - TableElements::ExternRefs(x) => x.len().try_into().unwrap(), + match &self.storage { + TableStorage::Static { size, .. } => size.get(), + TableStorage::Dynamic { elements, .. } => match &*elements.borrow() { + TableElements::FuncRefs(x) => x.len().try_into().unwrap(), + TableElements::ExternRefs(x) => x.len().try_into().unwrap(), + }, + } + } + + /// Returns the maximum number of elements. + pub fn maximum(&self) -> Option { + match &self.storage { + TableStorage::Static { maximum, .. } => Some(*maximum), + TableStorage::Dynamic { maximum, .. } => maximum.clone(), } } @@ -80,8 +165,31 @@ impl Table { return Err(Trap::wasm(ir::TrapCode::TableOutOfBounds)); } - for i in start..end { - self.set(i, val.clone()).unwrap(); + match val { + TableElement::FuncRef(r) => { + unsafe { + self.with_funcrefs_mut(move |elements| { + let elements = elements.unwrap(); + + // TODO: replace this with slice::fill (https://github.com/rust-lang/rust/issues/70758) when stabilized + for e in &mut elements[start as usize..end as usize] { + *e = r; + } + }); + } + } + TableElement::ExternRef(r) => { + unsafe { + self.with_externrefs_mut(move |elements| { + let elements = elements.unwrap(); + + // TODO: replace this with slice::fill (https://github.com/rust-lang/rust/issues/70758) when stabilized + for e in &mut elements[start as usize..end as usize] { + *e = r.clone(); + } + }); + } + } } Ok(()) @@ -104,38 +212,48 @@ impl Table { /// Generally, prefer using `InstanceHandle::table_grow`, which encapsulates /// this unsafety. pub unsafe fn grow(&self, delta: u32, init_value: TableElement) -> Option { - let size = self.size(); + let old_size = self.size(); - let new_len = size.checked_add(delta)?; - if let Some(max) = self.maximum { - if new_len > max { + let new_size = old_size.checked_add(delta)?; + if let Some(max) = self.maximum() { + if new_size > max { return None; } } - let new_len = usize::try_from(new_len).unwrap(); - match &mut *self.elements.borrow_mut() { - TableElements::FuncRefs(x) => { - let init_value = init_value.try_into().ok()?; - x.resize(new_len, init_value) + match &self.storage { + TableStorage::Static { size, .. } => { + size.set(new_size); + self.fill(old_size, init_value, delta) + .ok() + .map(|_| old_size) } - TableElements::ExternRefs(x) => { - let init_value = init_value.try_into().ok()?; - x.resize(new_len, init_value) + TableStorage::Dynamic { elements, .. } => { + let new_len = usize::try_from(new_size).unwrap(); + + match &mut *elements.borrow_mut() { + TableElements::FuncRefs(x) => x.resize(new_len, init_value.try_into().ok()?), + TableElements::ExternRefs(x) => x.resize(new_len, init_value.try_into().ok()?), + } + + Some(old_size) } } - - Some(size) } /// Get reference to the specified element. /// /// Returns `None` if the index is out of bounds. pub fn get(&self, index: u32) -> Option { - match &*self.elements.borrow() { - TableElements::FuncRefs(x) => x.get(index as usize).cloned().map(TableElement::FuncRef), - TableElements::ExternRefs(x) => { - x.get(index as usize).cloned().map(TableElement::ExternRef) + unsafe { + match self.element_type() { + TableElementType::Func => self.with_funcrefs(|elements| { + elements.and_then(|e| e.get(index as usize).cloned().map(TableElement::FuncRef)) + }), + TableElementType::Val(_) => self.with_externrefs(|elements| { + elements + .and_then(|e| e.get(index as usize).cloned().map(TableElement::ExternRef)) + }), } } } @@ -147,18 +265,22 @@ impl Table { /// Returns an error if `index` is out of bounds or if this table type does /// not match the element type. pub fn set(&self, index: u32, elem: TableElement) -> Result<(), ()> { - let mut elems = self.elements.borrow_mut(); - match &mut *elems { - TableElements::FuncRefs(x) => { - let slot = x.get_mut(index as usize).ok_or(())?; - *slot = elem.try_into().or(Err(()))?; - } - TableElements::ExternRefs(x) => { - let slot = x.get_mut(index as usize).ok_or(())?; - *slot = elem.try_into().or(Err(()))?; + unsafe { + match self.element_type() { + TableElementType::Func => self.with_funcrefs_mut(move |elements| { + let elements = elements.ok_or(())?; + let e = elements.get_mut(index as usize).ok_or(())?; + *e = elem.try_into()?; + Ok(()) + }), + TableElementType::Val(_) => self.with_externrefs_mut(move |elements| { + let elements = elements.ok_or(())?; + let e = elements.get_mut(index as usize).ok_or(())?; + *e = elem.try_into()?; + Ok(()) + }), } } - Ok(()) } /// Copy `len` elements from `src_table[src_index..]` into `dst_table[dst_index..]`. @@ -186,20 +308,48 @@ impl Table { return Err(Trap::wasm(ir::TrapCode::TableOutOfBounds)); } - let srcs = src_index..src_index + len; - let dsts = dst_index..dst_index + len; + // Check if the source and destination are the same table + // This ensures we don't `borrow` and `borrow_mut` the same underlying RefCell + let same_table = ptr::eq(dst_table, src_table); - // Note on the unwraps: the bounds check above means that these will - // never panic. - // - // TODO(#983): investigate replacing this get/set loop with a `memcpy`. - if dst_index <= src_index { - for (s, d) in (srcs).zip(dsts) { - dst_table.set(d, src_table.get(s).unwrap()).unwrap(); - } - } else { - for (s, d) in srcs.rev().zip(dsts.rev()) { - dst_table.set(d, src_table.get(s).unwrap()).unwrap(); + let src_range = src_index as usize..src_index as usize + len as usize; + let dst_range = dst_index as usize..dst_index as usize + len as usize; + + unsafe { + match dst_table.element_type() { + TableElementType::Func => dst_table.with_funcrefs_mut(|dst| { + let dst = dst.unwrap(); + + if same_table { + dst.copy_within(src_range, dst_index as usize); + } else { + src_table.with_funcrefs(|src| { + let src = src.unwrap(); + dst[dst_range].copy_from_slice(&src[src_range]); + }) + } + }), + TableElementType::Val(_) => dst_table.with_externrefs_mut(|dst| { + let dst = dst.unwrap(); + + if same_table { + // As there's no `slice::clone_within` because cloning can't be done with memmove, use a loop + if dst_index <= src_index { + for (s, d) in (src_range).zip(dst_range) { + dst[d] = dst[s].clone(); + } + } else { + for (s, d) in src_range.rev().zip(dst_range.rev()) { + dst[d] = dst[s].clone(); + } + } + } else { + src_table.with_externrefs(|src| { + let src = src.unwrap(); + dst[dst_range].clone_from_slice(&src[src_range]); + }) + } + }), } } @@ -208,55 +358,97 @@ impl Table { /// Return a `VMTableDefinition` for exposing the table to compiled wasm code. pub fn vmtable(&self) -> VMTableDefinition { - match &*self.elements.borrow() { - TableElements::FuncRefs(x) => VMTableDefinition { - base: x.as_ptr() as *const u8 as *mut u8, - current_elements: x.len().try_into().unwrap(), + match &self.storage { + TableStorage::Static { data, size, .. } => VMTableDefinition { + base: *data, + current_elements: size.get(), }, - TableElements::ExternRefs(x) => VMTableDefinition { - base: x.as_ptr() as *const u8 as *mut u8, - current_elements: x.len().try_into().unwrap(), + TableStorage::Dynamic { elements, .. } => match &*elements.borrow() { + TableElements::FuncRefs(x) => VMTableDefinition { + base: x.as_ptr() as *const u8 as _, + current_elements: x.len().try_into().unwrap(), + }, + TableElements::ExternRefs(x) => VMTableDefinition { + base: x.as_ptr() as *const u8 as _, + current_elements: x.len().try_into().unwrap(), + }, }, } } -} - -impl TryFrom for *mut VMCallerCheckedAnyfunc { - type Error = TableElement; - fn try_from(e: TableElement) -> Result { - match e { - TableElement::FuncRef(f) => Ok(f), - _ => Err(e), + unsafe fn with_funcrefs(&self, with: F) -> R + where + F: FnOnce(Option<&[*mut VMCallerCheckedAnyfunc]>) -> R, + { + match &self.storage { + TableStorage::Static { data, size, ty, .. } => match ty { + TableElementType::Func => with(Some(std::slice::from_raw_parts( + *data as *const _, + size.get() as usize, + ))), + _ => with(None), + }, + TableStorage::Dynamic { elements, .. } => match &*elements.borrow() { + TableElements::FuncRefs(x) => with(Some(x.as_slice())), + _ => with(None), + }, } } -} -impl TryFrom for Option { - type Error = TableElement; - - fn try_from(e: TableElement) -> Result { - match e { - TableElement::ExternRef(x) => Ok(x), - _ => Err(e), + unsafe fn with_funcrefs_mut(&self, with: F) -> R + where + F: FnOnce(Option<&mut [*mut VMCallerCheckedAnyfunc]>) -> R, + { + match &self.storage { + TableStorage::Static { data, size, ty, .. } => match ty { + TableElementType::Func => with(Some(std::slice::from_raw_parts_mut( + *data as *mut _, + size.get() as usize, + ))), + _ => with(None), + }, + TableStorage::Dynamic { elements, .. } => match &mut *elements.borrow_mut() { + TableElements::FuncRefs(x) => with(Some(x.as_mut_slice())), + _ => with(None), + }, } } -} - -impl From<*mut VMCallerCheckedAnyfunc> for TableElement { - fn from(f: *mut VMCallerCheckedAnyfunc) -> TableElement { - TableElement::FuncRef(f) - } -} -impl From> for TableElement { - fn from(x: Option) -> TableElement { - TableElement::ExternRef(x) + unsafe fn with_externrefs(&self, with: F) -> R + where + F: FnOnce(Option<&[Option]>) -> R, + { + match &self.storage { + TableStorage::Static { data, size, ty, .. } => match ty { + TableElementType::Val(_) => with(Some(std::slice::from_raw_parts( + *data as *const _, + size.get() as usize, + ))), + _ => with(None), + }, + TableStorage::Dynamic { elements, .. } => match &*elements.borrow() { + TableElements::ExternRefs(x) => with(Some(x.as_slice())), + _ => with(None), + }, + } } -} -impl From for TableElement { - fn from(x: VMExternRef) -> TableElement { - TableElement::ExternRef(Some(x)) + unsafe fn with_externrefs_mut(&self, with: F) -> R + where + F: FnOnce(Option<&mut [Option]>) -> R, + { + match &self.storage { + TableStorage::Static { data, size, ty, .. } => match ty { + TableElementType::Val(_) => with(Some(std::slice::from_raw_parts_mut( + *data as *mut _, + size.get() as usize, + ))), + _ => with(None), + }, + TableStorage::Dynamic { elements, .. } => match &mut *elements.borrow_mut() { + TableElements::ExternRefs(x) => with(Some(x.as_mut_slice())), + _ => with(None), + }, + } } } From dd284ac218ffb29fc4ee27df919649eff2c786b0 Mon Sep 17 00:00:00 2001 From: Peter Huene Date: Tue, 8 Dec 2020 12:13:15 -0800 Subject: [PATCH 04/33] Store memories and tables on Instance as PrimaryMap. This commit changes how memories and tables are stored in `Instance`. Previously, the memories and tables were stored as a `BoxedSlice`. Storing it this way requires an allocation to change the length of the memories and tables, which is desirable for a pooling instance allocator that is reusing an `Instance` structure for a new instantiation. By storing it instead as `PrimaryMap`, the memories and tables can be resized without any allocations (the capacity of these maps will always be the configured limits of the pooling allocator). --- crates/runtime/src/instance.rs | 6 +++--- crates/runtime/src/instance/allocator.rs | 12 +++++------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/crates/runtime/src/instance.rs b/crates/runtime/src/instance.rs index cdb892842f1a..396d60f9ad39 100644 --- a/crates/runtime/src/instance.rs +++ b/crates/runtime/src/instance.rs @@ -24,7 +24,7 @@ use std::ptr::NonNull; use std::rc::Rc; use std::sync::Arc; use std::{mem, ptr, slice}; -use wasmtime_environ::entity::{packed_option::ReservedValue, BoxedSlice, EntityRef, EntitySet}; +use wasmtime_environ::entity::{packed_option::ReservedValue, EntityRef, EntitySet, PrimaryMap}; use wasmtime_environ::wasm::{ DataIndex, DefinedGlobalIndex, DefinedMemoryIndex, DefinedTableIndex, ElemIndex, EntityIndex, FuncIndex, GlobalIndex, MemoryIndex, TableElementType, TableIndex, @@ -51,10 +51,10 @@ pub(crate) struct Instance { offsets: VMOffsets, /// WebAssembly linear memory data. - memories: BoxedSlice>, + memories: PrimaryMap>, /// WebAssembly table data. - tables: BoxedSlice, + tables: PrimaryMap, /// Stores the dropped passive element segments in this instantiation by index. /// If the index is present in the set, the segment has been dropped. diff --git a/crates/runtime/src/instance/allocator.rs b/crates/runtime/src/instance/allocator.rs index ba00aedaf91c..590b532caeca 100644 --- a/crates/runtime/src/instance/allocator.rs +++ b/crates/runtime/src/instance/allocator.rs @@ -17,9 +17,7 @@ use std::ptr::{self, NonNull}; use std::slice; use std::sync::Arc; use thiserror::Error; -use wasmtime_environ::entity::{ - packed_option::ReservedValue, BoxedSlice, EntityRef, EntitySet, PrimaryMap, -}; +use wasmtime_environ::entity::{packed_option::ReservedValue, EntityRef, EntitySet, PrimaryMap}; use wasmtime_environ::wasm::{ DefinedFuncIndex, DefinedMemoryIndex, DefinedTableIndex, FuncIndex, GlobalInit, SignatureIndex, TableElementType, WasmType, @@ -285,20 +283,20 @@ impl OnDemandInstanceAllocator { Self { mem_creator } } - fn create_tables(module: &Module) -> BoxedSlice { + fn create_tables(module: &Module) -> PrimaryMap { let num_imports = module.num_imported_tables; let mut tables: PrimaryMap = PrimaryMap::with_capacity(module.table_plans.len() - num_imports); for table in &module.table_plans.values().as_slice()[num_imports..] { tables.push(Table::new_dynamic(table)); } - tables.into_boxed_slice() + tables } fn create_memories( &self, module: &Module, - ) -> Result>, InstantiationError> + ) -> Result>, InstantiationError> { let creator = self .mem_creator @@ -314,7 +312,7 @@ impl OnDemandInstanceAllocator { .map_err(InstantiationError::Resource)?, ); } - Ok(memories.into_boxed_slice()) + Ok(memories) } fn check_table_init_bounds(instance: &Instance) -> Result<(), InstantiationError> { From 5beb81d02a2d82d26132d2a289f750dd5f1320b3 Mon Sep 17 00:00:00 2001 From: Peter Huene Date: Wed, 9 Dec 2020 10:15:32 -0800 Subject: [PATCH 05/33] Change how Instance stores instantiated memories in the runtime. This commit changes `Instance` such that memories can be stored statically, with just a base pointer, size, maximum, and a callback to make memory accessible. Previously the memories were being stored as boxed trait objects, which would require the pooling allocator to do some unpleasant things to avoid allocations. With this change, the pooling allocator can simply define a memory for the instance without using a trait object. --- crates/runtime/src/instance.rs | 4 +- crates/runtime/src/instance/allocator.rs | 12 +-- crates/runtime/src/lib.rs | 2 +- crates/runtime/src/memory.rs | 115 ++++++++++++++++++++++- 4 files changed, 121 insertions(+), 12 deletions(-) diff --git a/crates/runtime/src/instance.rs b/crates/runtime/src/instance.rs index 396d60f9ad39..91a0dd5c46eb 100644 --- a/crates/runtime/src/instance.rs +++ b/crates/runtime/src/instance.rs @@ -4,7 +4,7 @@ use crate::export::Export; use crate::externref::{StackMapRegistry, VMExternRefActivationsTable}; -use crate::memory::{RuntimeLinearMemory, RuntimeMemoryCreator}; +use crate::memory::{Memory, RuntimeMemoryCreator}; use crate::table::{Table, TableElement}; use crate::traphandlers::Trap; use crate::vmcontext::{ @@ -51,7 +51,7 @@ pub(crate) struct Instance { offsets: VMOffsets, /// WebAssembly linear memory data. - memories: PrimaryMap>, + memories: PrimaryMap, /// WebAssembly table data. tables: PrimaryMap, diff --git a/crates/runtime/src/instance/allocator.rs b/crates/runtime/src/instance/allocator.rs index 590b532caeca..6c16604b0764 100644 --- a/crates/runtime/src/instance/allocator.rs +++ b/crates/runtime/src/instance/allocator.rs @@ -1,7 +1,7 @@ use crate::externref::{StackMapRegistry, VMExternRefActivationsTable}; use crate::imports::Imports; use crate::instance::{Instance, InstanceHandle, RuntimeMemoryCreator}; -use crate::memory::{DefaultMemoryCreator, RuntimeLinearMemory}; +use crate::memory::{DefaultMemoryCreator, Memory}; use crate::table::{Table, TableElement}; use crate::traphandlers::Trap; use crate::vmcontext::{ @@ -296,8 +296,7 @@ impl OnDemandInstanceAllocator { fn create_memories( &self, module: &Module, - ) -> Result>, InstantiationError> - { + ) -> Result, InstantiationError> { let creator = self .mem_creator .as_deref() @@ -306,11 +305,8 @@ impl OnDemandInstanceAllocator { let mut memories: PrimaryMap = PrimaryMap::with_capacity(module.memory_plans.len() - num_imports); for plan in &module.memory_plans.values().as_slice()[num_imports..] { - memories.push( - creator - .new_memory(plan) - .map_err(InstantiationError::Resource)?, - ); + memories + .push(Memory::new_dynamic(plan, creator).map_err(InstantiationError::Resource)?); } Ok(memories) } diff --git a/crates/runtime/src/lib.rs b/crates/runtime/src/lib.rs index eff31fdeead5..04ef0026ce2c 100644 --- a/crates/runtime/src/lib.rs +++ b/crates/runtime/src/lib.rs @@ -42,7 +42,7 @@ pub use crate::instance::{ OnDemandInstanceAllocator, RuntimeInstance, }; pub use crate::jit_int::GdbJitImageRegistration; -pub use crate::memory::{RuntimeLinearMemory, RuntimeMemoryCreator}; +pub use crate::memory::{Memory, RuntimeLinearMemory, RuntimeMemoryCreator}; pub use crate::mmap::Mmap; pub use crate::table::{Table, TableElement}; pub use crate::traphandlers::{ diff --git a/crates/runtime/src/memory.rs b/crates/runtime/src/memory.rs index a2c03e1d31d3..1b865c0eecda 100644 --- a/crates/runtime/src/memory.rs +++ b/crates/runtime/src/memory.rs @@ -5,7 +5,8 @@ use crate::mmap::Mmap; use crate::vmcontext::VMMemoryDefinition; use more_asserts::{assert_ge, assert_le}; -use std::cell::RefCell; +use std::cell::{Cell, RefCell}; +use std::cmp::min; use std::convert::TryFrom; use wasmtime_environ::{MemoryPlan, MemoryStyle, WASM_MAX_PAGES, WASM_PAGE_SIZE}; @@ -170,3 +171,115 @@ impl RuntimeLinearMemory for MmapMemory { } } } + +enum MemoryStorage { + Static { + base: *mut u8, + size: Cell, + maximum: u32, + make_accessible: Option bool>, + }, + Dynamic(Box), +} + +/// Represents an instantiation of a WebAssembly memory. +pub struct Memory { + storage: MemoryStorage, +} + +impl Memory { + /// Create a new dynamic (movable) memory instance for the specified plan. + pub fn new_dynamic( + plan: &MemoryPlan, + creator: &dyn RuntimeMemoryCreator, + ) -> Result { + Ok(Self { + storage: MemoryStorage::Dynamic(creator.new_memory(plan)?), + }) + } + + /// Create a new static (immovable) memory instance for the specified plan. + pub fn new_static( + plan: &MemoryPlan, + base: *mut u8, + maximum: u32, + make_accessible: Option bool>, + ) -> Result { + if plan.memory.minimum > 0 { + if let Some(make_accessible) = &make_accessible { + if !make_accessible(base, plan.memory.minimum as usize * WASM_PAGE_SIZE as usize) { + return Err("memory cannot be made accessible".into()); + } + } + } + + Ok(Self { + storage: MemoryStorage::Static { + base, + size: Cell::new(plan.memory.minimum), + maximum: min(plan.memory.maximum.unwrap_or(maximum), maximum), + make_accessible, + }, + }) + } + + /// Returns the number of allocated wasm pages. + pub fn size(&self) -> u32 { + match &self.storage { + MemoryStorage::Static { size, .. } => size.get(), + MemoryStorage::Dynamic(mem) => mem.size(), + } + } + + /// Grow memory by the specified amount of wasm pages. + /// + /// Returns `None` if memory can't be grown by the specified amount + /// of wasm pages. + pub fn grow(&self, delta: u32) -> Option { + match &self.storage { + MemoryStorage::Static { + base, + size, + maximum, + make_accessible, + .. + } => { + let old_size = size.get(); + if delta == 0 { + return Some(old_size); + } + + let new_size = old_size.checked_add(delta)?; + + if new_size > *maximum || new_size >= WASM_MAX_PAGES { + return None; + } + + let start = usize::try_from(old_size).unwrap() * WASM_PAGE_SIZE as usize; + let len = usize::try_from(delta).unwrap() * WASM_PAGE_SIZE as usize; + + if let Some(make_accessible) = make_accessible { + if !make_accessible(unsafe { base.add(start) }, len) { + return None; + } + } + + size.set(new_size); + + Some(old_size) + } + MemoryStorage::Dynamic(mem) => mem.grow(delta), + } + } + + /// Return a `VMMemoryDefinition` for exposing the memory to compiled wasm code. + pub fn vmmemory(&self) -> VMMemoryDefinition { + match &self.storage { + MemoryStorage::Static { base, size, .. } => VMMemoryDefinition { + base: *base, + current_length: size.get() as usize * WASM_PAGE_SIZE as usize, + }, + MemoryStorage::Dynamic(mem) => mem.vmmemory(), + } + } +} From 8457261cfe6fffb80049b025bb259909afab98d8 Mon Sep 17 00:00:00 2001 From: Peter Huene Date: Mon, 14 Dec 2020 17:12:54 -0800 Subject: [PATCH 06/33] Ensure default allocator is used for instance deallocation. Handles created with `create_handle` need to be deallocated with the default (on-demand) instance allocator. This commit changes Store such that handles can be added with a flag that is used to force deallocation via the default instance allocator when the Store is dropped. --- crates/wasmtime/src/instance.rs | 2 +- crates/wasmtime/src/store.rs | 37 +++++++++++++++---- .../wasmtime/src/trampoline/create_handle.rs | 2 +- 3 files changed, 32 insertions(+), 9 deletions(-) diff --git a/crates/wasmtime/src/instance.rs b/crates/wasmtime/src/instance.rs index 87936fd535ee..7048d9333813 100644 --- a/crates/wasmtime/src/instance.rs +++ b/crates/wasmtime/src/instance.rs @@ -520,7 +520,7 @@ impl<'a> Instantiator<'a> { // initializers may have run which placed elements into other instance's // tables. This means that from this point on, regardless of whether // initialization is successful, we need to keep the instance alive. - let instance = self.store.add_instance(instance); + let instance = self.store.add_instance(instance, false); allocator .initialize( &instance.handle, diff --git a/crates/wasmtime/src/store.rs b/crates/wasmtime/src/store.rs index dbaa5eb44c05..9445f852a2c4 100644 --- a/crates/wasmtime/src/store.rs +++ b/crates/wasmtime/src/store.rs @@ -18,10 +18,19 @@ use std::task::{Context, Poll}; use wasmtime_environ::wasm; use wasmtime_jit::{CompiledModule, ModuleCode, TypeTables}; use wasmtime_runtime::{ - InstanceHandle, SignalHandler, StackMapRegistry, TrapInfo, VMContext, VMExternRef, - VMExternRefActivationsTable, VMInterrupts, VMSharedSignatureIndex, + InstanceAllocator, InstanceHandle, SignalHandler, StackMapRegistry, TrapInfo, VMContext, + VMExternRef, VMExternRefActivationsTable, VMInterrupts, VMSharedSignatureIndex, }; +/// Used to associate instances with the store. +/// +/// This is needed to track if the instance was allocated expliclty with the default +/// instance allocator. +struct StoreInstance { + handle: InstanceHandle, + use_default_allocator: bool, +} + /// A `Store` is a collection of WebAssembly instances and host-defined items. /// /// All WebAssembly instances and items will be attached to and refer to a @@ -63,7 +72,7 @@ pub(crate) struct StoreInner { engine: Engine, interrupts: Arc, signatures: RefCell, - instances: RefCell>, + instances: RefCell>, signal_handler: RefCell>>>, externref_activations_table: VMExternRefActivationsTable, stack_map_registry: StackMapRegistry, @@ -374,8 +383,15 @@ impl Store { Ok(()) } - pub(crate) unsafe fn add_instance(&self, handle: InstanceHandle) -> StoreInstanceHandle { - self.inner.instances.borrow_mut().push(handle.clone()); + pub(crate) unsafe fn add_instance( + &self, + handle: InstanceHandle, + use_default_allocator: bool, + ) -> StoreInstanceHandle { + self.inner.instances.borrow_mut().push(StoreInstance { + handle: handle.clone(), + use_default_allocator, + }); StoreInstanceHandle { store: self.clone(), handle, @@ -388,7 +404,7 @@ impl Store { .instances .borrow() .iter() - .any(|i| i.vmctx_ptr() == handle.vmctx_ptr())); + .any(|i| i.handle.vmctx_ptr() == handle.vmctx_ptr())); StoreInstanceHandle { store: self.clone(), handle, @@ -963,7 +979,14 @@ impl Drop for StoreInner { let allocator = self.engine.config().instance_allocator(); for instance in self.instances.borrow().iter() { unsafe { - allocator.deallocate(instance); + if instance.use_default_allocator { + self.engine + .config() + .default_instance_allocator + .deallocate(&instance.handle); + } else { + allocator.deallocate(&instance.handle); + } } } } diff --git a/crates/wasmtime/src/trampoline/create_handle.rs b/crates/wasmtime/src/trampoline/create_handle.rs index ff088b9aa17d..71595ff7296a 100644 --- a/crates/wasmtime/src/trampoline/create_handle.rs +++ b/crates/wasmtime/src/trampoline/create_handle.rs @@ -44,6 +44,6 @@ pub(crate) fn create_handle( stack_map_registry: store.stack_map_registry() as *const StackMapRegistry as *mut _, })?; - Ok(store.add_instance(handle)) + Ok(store.add_instance(handle, true)) } } From 3bb145f65c4a3b840d02bd7704c2affe51539d1b Mon Sep 17 00:00:00 2001 From: Peter Huene Date: Wed, 16 Dec 2020 13:00:35 -0800 Subject: [PATCH 07/33] Only treat a memory as static when the minimum is also within bounds. With the change to artificially limit unbounded memories based on Tunables, it's possible to hit the assert where the minimum might exceed the static memory bound. This commit removes the assert in favor of a check to see if the minimum also fits within the static memory bound. It also corrects the maximum bounding to ensure the minimum between the memory's maximum and the configured maximum is used. If it does not fit, the memory will be treated as dynamic. In the case of the pooling instance allocator, the bounds will be checked again during translation and an appropriate error will be returned as dynamic memories are not supported for that allocator. --- crates/environ/src/module.rs | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/crates/environ/src/module.rs b/crates/environ/src/module.rs index 7282501e4a59..8daefaf0790b 100644 --- a/crates/environ/src/module.rs +++ b/crates/environ/src/module.rs @@ -6,7 +6,6 @@ use cranelift_codegen::ir; use cranelift_entity::{EntityRef, PrimaryMap}; use cranelift_wasm::*; use indexmap::IndexMap; -use more_asserts::assert_ge; use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::sync::Arc; @@ -44,16 +43,18 @@ impl MemoryStyle { // // If the module doesn't declare an explicit maximum treat it as 4GiB when not // requested to use the static memory bound itself as the maximum. - let maximum = memory - .maximum - .unwrap_or(if tunables.static_memory_bound_is_maximum { + let maximum = std::cmp::min( + memory.maximum.unwrap_or(WASM_MAX_PAGES), + if tunables.static_memory_bound_is_maximum { tunables.static_memory_bound } else { WASM_MAX_PAGES - }); + }, + ); - if maximum <= tunables.static_memory_bound { - assert_ge!(tunables.static_memory_bound, memory.minimum); + // Ensure the minimum is less than the maximum; the minimum might exceed the maximum + // when the memory is artificially bounded via `static_memory_bound_is_maximum` above + if memory.minimum <= maximum && maximum <= tunables.static_memory_bound { return ( Self::Static { bound: tunables.static_memory_bound, From 16ca5e16d940eb1ecd2d5e11eaee92c1ad29df3e Mon Sep 17 00:00:00 2001 From: Peter Huene Date: Thu, 4 Feb 2021 13:05:01 -0800 Subject: [PATCH 08/33] Implement allocating fiber stacks for an instance allocator. This commit implements allocating fiber stacks in an instance allocator. The on-demand instance allocator doesn't support custom stacks, so the implementation will use the allocation from `wasmtime-fiber` for the fiber stacks. In the future, the pooling instance allocator will return custom stacks to use on Linux and macOS. On Windows, the native fiber implementation will always be used. --- crates/c-api/src/config.rs | 4 +- crates/fiber/src/lib.rs | 21 ++++++++ crates/fiber/src/unix.rs | 61 ++++++++++++++++-------- crates/fiber/src/windows.rs | 12 ++++- crates/runtime/src/instance/allocator.rs | 37 ++++++++++++++ crates/runtime/src/lib.rs | 4 +- crates/wasmtime/src/config.rs | 59 +++++++++++++++++++---- 7 files changed, 161 insertions(+), 37 deletions(-) diff --git a/crates/c-api/src/config.rs b/crates/c-api/src/config.rs index 3bb7f0c5c534..f84bc320cd8d 100644 --- a/crates/c-api/src/config.rs +++ b/crates/c-api/src/config.rs @@ -61,8 +61,8 @@ pub extern "C" fn wasmtime_config_consume_fuel_set(c: &mut wasm_config_t, enable } #[no_mangle] -pub extern "C" fn wasmtime_config_max_wasm_stack_set(c: &mut wasm_config_t, size: usize) { - c.config.max_wasm_stack(size); +pub extern "C" fn wasmtime_config_max_wasm_stack_set(c: &mut wasm_config_t, size: usize) -> bool { + c.config.max_wasm_stack(size).is_ok() } #[no_mangle] diff --git a/crates/fiber/src/lib.rs b/crates/fiber/src/lib.rs index bafae2f01c13..6c835c511f81 100644 --- a/crates/fiber/src/lib.rs +++ b/crates/fiber/src/lib.rs @@ -51,6 +51,27 @@ impl<'a, Resume, Yield, Return> Fiber<'a, Resume, Yield, Return> { }) } + /// Creates a new fiber with existing stack space that will execute `func`. + /// + /// This function returns a `Fiber` which, when resumed, will execute `func` + /// to completion. When desired the `func` can suspend itself via + /// `Fiber::suspend`. + /// + /// # Safety + /// + /// The caller must properly allocate the stack space with a guard page and + /// make the pages accessible for correct behavior. + pub unsafe fn new_with_stack( + top_of_stack: *mut u8, + func: impl FnOnce(Resume, &Suspend) -> Return + 'a, + ) -> io::Result> { + Ok(Fiber { + inner: imp::Fiber::new_with_stack(top_of_stack, func), + done: Cell::new(false), + _phantom: PhantomData, + }) + } + /// Resumes execution of this fiber. /// /// This function will transfer execution to the fiber and resume from where diff --git a/crates/fiber/src/unix.rs b/crates/fiber/src/unix.rs index 2c1069041ebe..c14a188f50f5 100644 --- a/crates/fiber/src/unix.rs +++ b/crates/fiber/src/unix.rs @@ -35,10 +35,10 @@ use std::io; use std::ptr; pub struct Fiber { - // Description of the mmap region we own. This should be abstracted - // eventually so we aren't personally mmap-ing this region. - mmap: *mut libc::c_void, - mmap_len: usize, + // The top of the stack; for stacks allocated by the fiber implementation itself, + // the base address of the allocation will be `top_of_stack.sub(alloc_len.unwrap())` + top_of_stack: *mut u8, + alloc_len: Option, } pub struct Suspend { @@ -66,21 +66,40 @@ where } impl Fiber { - pub fn new(stack_size: usize, func: F) -> io::Result + pub fn new(stack_size: usize, func: F) -> io::Result + where + F: FnOnce(A, &super::Suspend) -> C, + { + let fiber = Self::alloc_with_stack(stack_size)?; + fiber.init(func); + Ok(fiber) + } + + pub fn new_with_stack(top_of_stack: *mut u8, func: F) -> Self + where + F: FnOnce(A, &super::Suspend) -> C, + { + let fiber = Self { + top_of_stack, + alloc_len: None, + }; + + fiber.init(func); + + fiber + } + + fn init(&self, func: F) where F: FnOnce(A, &super::Suspend) -> C, { - let fiber = Fiber::alloc_with_stack(stack_size)?; unsafe { - // Initialize the top of the stack to be resumed from - let top_of_stack = fiber.top_of_stack(); let data = Box::into_raw(Box::new(func)).cast(); - wasmtime_fiber_init(top_of_stack, fiber_start::, data); - Ok(fiber) + wasmtime_fiber_init(self.top_of_stack, fiber_start::, data); } } - fn alloc_with_stack(stack_size: usize) -> io::Result { + fn alloc_with_stack(stack_size: usize) -> io::Result { unsafe { // Round up our stack size request to the nearest multiple of the // page size. @@ -104,7 +123,10 @@ impl Fiber { if mmap == libc::MAP_FAILED { return Err(io::Error::last_os_error()); } - let ret = Fiber { mmap, mmap_len }; + let ret = Self { + top_of_stack: mmap.cast::().add(mmap_len), + alloc_len: Some(mmap_len), + }; let res = libc::mprotect( mmap.cast::().add(page_size).cast(), stack_size, @@ -124,27 +146,24 @@ impl Fiber { // stack, otherwise known as our reserved slot for this information. // // In the diagram above this is updating address 0xAff8 - let top_of_stack = self.top_of_stack(); - let addr = top_of_stack.cast::().offset(-1); + let addr = self.top_of_stack.cast::().offset(-1); addr.write(result as *const _ as usize); - wasmtime_fiber_switch(top_of_stack); + wasmtime_fiber_switch(self.top_of_stack); // null this out to help catch use-after-free addr.write(0); } } - - unsafe fn top_of_stack(&self) -> *mut u8 { - self.mmap.cast::().add(self.mmap_len) - } } impl Drop for Fiber { fn drop(&mut self) { unsafe { - let ret = libc::munmap(self.mmap, self.mmap_len); - debug_assert!(ret == 0); + if let Some(alloc_len) = self.alloc_len { + let ret = libc::munmap(self.top_of_stack.sub(alloc_len) as _, alloc_len); + debug_assert!(ret == 0); + } } } } diff --git a/crates/fiber/src/windows.rs b/crates/fiber/src/windows.rs index 69a5f161e6f0..c40fb8aeb031 100644 --- a/crates/fiber/src/windows.rs +++ b/crates/fiber/src/windows.rs @@ -40,7 +40,7 @@ where } impl Fiber { - pub fn new(stack_size: usize, func: F) -> io::Result + pub fn new(stack_size: usize, func: F) -> io::Result where F: FnOnce(A, &super::Suspend) -> C, { @@ -61,11 +61,19 @@ impl Fiber { drop(Box::from_raw(state.initial_closure.get().cast::())); Err(io::Error::last_os_error()) } else { - Ok(Fiber { fiber, state }) + Ok(Self { fiber, state }) } } } + pub fn new_with_stack(_top_of_stack: *mut u8, _func: F) -> Self + where + F: FnOnce(A, &super::Suspend) -> C, + { + // Windows fibers have no support for custom stacks + unimplemented!() + } + pub(crate) fn resume(&self, result: &Cell>) { unsafe { let is_fiber = IsThreadAFiber() != 0; diff --git a/crates/runtime/src/instance/allocator.rs b/crates/runtime/src/instance/allocator.rs index 6c16604b0764..45d6a14445ef 100644 --- a/crates/runtime/src/instance/allocator.rs +++ b/crates/runtime/src/instance/allocator.rs @@ -74,6 +74,17 @@ pub enum InstantiationError { Trap(Trap), } +/// An error while creating a fiber stack. +#[derive(Error, Debug)] +pub enum FiberStackError { + /// An error for when the allocator doesn't support custom fiber stacks. + #[error("Custom fiber stacks are not supported by the allocator")] + NotSupported, + /// A limit on how many fibers are supported has been reached. + #[error("Limit of {0} concurrent fibers has been reached")] + Limit(u32), +} + /// Represents a runtime instance allocator. /// /// # Safety @@ -127,6 +138,22 @@ pub unsafe trait InstanceAllocator: Send + Sync { /// /// Use extreme care when deallocating an instance so that there are no dangling instance pointers. unsafe fn deallocate(&self, handle: &InstanceHandle); + + /// Allocates a fiber stack for calling async functions on. + /// + /// Returns the top of the fiber stack if successfully allocated. + fn allocate_fiber_stack(&self) -> Result<*mut u8, FiberStackError>; + + /// Deallocates a fiber stack that was previously allocated. + /// + /// # Safety + /// + /// This function is unsafe because there are no guarantees that the given stack + /// is no longer in use. + /// + /// Additionally, passing a stack pointer that was not returned from `allocate_fiber_stack` + /// will lead to undefined behavior. + unsafe fn deallocate_fiber_stack(&self, stack: *mut u8); } unsafe fn initialize_vmcontext( @@ -544,4 +571,14 @@ unsafe impl InstanceAllocator for OnDemandInstanceAllocator { ptr::drop_in_place(instance as *const Instance as *mut Instance); alloc::dealloc(instance as *const Instance as *mut _, layout); } + + fn allocate_fiber_stack(&self) -> Result<*mut u8, FiberStackError> { + // The on-demand allocator does not support allocating fiber stacks + Err(FiberStackError::NotSupported) + } + + unsafe fn deallocate_fiber_stack(&self, _stack: *mut u8) { + // This should never be called as `allocate_fiber_stack` never returns success + unreachable!() + } } diff --git a/crates/runtime/src/lib.rs b/crates/runtime/src/lib.rs index 04ef0026ce2c..f33cfd8f381c 100644 --- a/crates/runtime/src/lib.rs +++ b/crates/runtime/src/lib.rs @@ -38,8 +38,8 @@ pub use crate::export::*; pub use crate::externref::*; pub use crate::imports::Imports; pub use crate::instance::{ - InstanceAllocationRequest, InstanceAllocator, InstanceHandle, InstantiationError, LinkError, - OnDemandInstanceAllocator, RuntimeInstance, + FiberStackError, InstanceAllocationRequest, InstanceAllocator, InstanceHandle, + InstantiationError, LinkError, OnDemandInstanceAllocator, RuntimeInstance, }; pub use crate::jit_int::GdbJitImageRegistration; pub use crate::memory::{Memory, RuntimeLinearMemory, RuntimeMemoryCreator}; diff --git a/crates/wasmtime/src/config.rs b/crates/wasmtime/src/config.rs index 324b03501d13..19151600b54f 100644 --- a/crates/wasmtime/src/config.rs +++ b/crates/wasmtime/src/config.rs @@ -58,6 +58,8 @@ pub struct Config { pub(crate) max_instances: usize, pub(crate) max_tables: usize, pub(crate) max_memories: usize, + #[cfg(feature = "async")] + pub(crate) async_stack_size: usize, } impl Config { @@ -108,6 +110,8 @@ impl Config { max_instances: 10_000, max_tables: 10_000, max_memories: 10_000, + #[cfg(feature = "async")] + async_stack_size: 2 << 20, }; ret.wasm_backtrace_details(WasmBacktraceDetails::Environment); return ret; @@ -182,23 +186,58 @@ impl Config { self } - /// Configures the maximum amount of native stack space available to + /// Configures the maximum amount of stack space available for /// executing WebAssembly code. /// - /// WebAssembly code currently executes on the native call stack for its own - /// call frames. WebAssembly, however, also has well-defined semantics on - /// stack overflow. This is intended to be a knob which can help configure - /// how much native stack space a wasm module is allowed to consume. Note - /// that the number here is not super-precise, but rather wasm will take at - /// most "pretty close to this much" stack space. + /// WebAssembly has well-defined semantics on stack overflow. This is + /// intended to be a knob which can help configure how much stack space + /// wasm execution is allowed to consume. Note that the number here is not + /// super-precise, but rather wasm will take at most "pretty close to this + /// much" stack space. /// /// If a wasm call (or series of nested wasm calls) take more stack space /// than the `size` specified then a stack overflow trap will be raised. /// - /// By default this option is 1 MB. - pub fn max_wasm_stack(&mut self, size: usize) -> &mut Self { + /// When the `async` feature is enabled, this value cannot exceed the + /// `async_stack_size` option. Be careful not to set this value too close + /// to `async_stack_size` as doing so may limit how much stack space + /// is available for host functions. Unlike wasm functions that trap + /// on stack overflow, a host function that overflows the stack will + /// abort the process. + /// + /// By default this option is 1 MiB. + pub fn max_wasm_stack(&mut self, size: usize) -> Result<&mut Self> { + #[cfg(feature = "async")] + if size > self.async_stack_size { + bail!("wasm stack size cannot exceed the async stack size"); + } + + if size == 0 { + bail!("wasm stack size cannot be zero"); + } + self.max_wasm_stack = size; - self + Ok(self) + } + + /// Configures the size of the stacks used for asynchronous execution. + /// + /// This setting configures the size of the stacks that are allocated for + /// asynchronous execution. The value cannot be less than `max_wasm_stack`. + /// + /// The amount of stack space guaranteed for host functions is + /// `async_stack_size - max_wasm_stack`, so take care not to set these two values + /// close to one another; doing so may cause host functions to overflow the + /// stack and abort the process. + /// + /// By default this option is 2 MiB. + #[cfg(feature = "async")] + pub fn async_stack_size(&mut self, size: usize) -> Result<&mut Self> { + if size < self.max_wasm_stack { + bail!("async stack size cannot be less than the maximum wasm stack size"); + } + self.async_stack_size = size; + Ok(self) } /// Configures whether the WebAssembly threads proposal will be enabled for From e71ccbf9bc3c8cacba13b25bd22a67b7ab044af5 Mon Sep 17 00:00:00 2001 From: Peter Huene Date: Tue, 8 Dec 2020 16:00:48 -0800 Subject: [PATCH 09/33] Implement the pooling instance allocator. This commit implements the pooling instance allocator. The allocation strategy can be set with `Config::with_allocation_strategy`. The pooling strategy uses the pooling instance allocator to preallocate a contiguous region of memory for instantiating modules that adhere to various limits. The intention of the pooling instance allocator is to reserve as much of the host address space needed for instantiating modules ahead of time and to reuse committed memory pages wherever possible. --- Cargo.lock | 66 +- crates/environ/src/vmoffsets.rs | 1 + crates/runtime/Cargo.toml | 4 + crates/runtime/src/instance/allocator.rs | 15 +- .../runtime/src/instance/allocator/pooling.rs | 1666 +++++++++++++++++ .../src/instance/allocator/pooling/linux.rs | 22 + .../src/instance/allocator/pooling/unix.rs | 26 + .../src/instance/allocator/pooling/windows.rs | 21 + crates/runtime/src/lib.rs | 5 +- crates/runtime/src/memory.rs | 20 +- crates/runtime/src/mmap.rs | 7 +- crates/runtime/src/table.rs | 14 + crates/wasmtime/src/config.rs | 62 +- tests/all/async_functions.rs | 34 + tests/all/main.rs | 1 + tests/all/pooling_allocator.rs | 430 +++++ 16 files changed, 2374 insertions(+), 20 deletions(-) create mode 100644 crates/runtime/src/instance/allocator/pooling.rs create mode 100644 crates/runtime/src/instance/allocator/pooling/linux.rs create mode 100644 crates/runtime/src/instance/allocator/pooling/unix.rs create mode 100644 crates/runtime/src/instance/allocator/pooling/windows.rs create mode 100644 tests/all/pooling_allocator.rs diff --git a/Cargo.lock b/Cargo.lock index 87e030f372ec..3b22f19a5515 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -202,6 +202,25 @@ dependencies = [ "which", ] +[[package]] +name = "bindgen" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd4865004a46a0aafb2a0a5eb19d3c9fc46ee5f063a6cfc605c69ac9ecf5263d" +dependencies = [ + "bitflags", + "cexpr", + "clang-sys", + "lazy_static", + "lazycell", + "peeking_take_while", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex", +] + [[package]] name = "bit-set" version = "0.5.2" @@ -1556,6 +1575,19 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0debeb9fcf88823ea64d64e4a815ab1643f33127d995978e099942ce38f25238" +[[package]] +name = "nix" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50e4785f2c3b7589a0d0c1dd60285e1188adac4006e8abd6dd578e1567027363" +dependencies = [ + "bitflags", + "cc", + "cfg-if 0.1.10", + "libc", + "void", +] + [[package]] name = "nom" version = "5.1.2" @@ -1703,7 +1735,7 @@ version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8fb64bef270a1ff665b0b2e28ebfa213e6205a007ce88223d020730225d6008f" dependencies = [ - "bindgen", + "bindgen 0.55.1", "cmake", ] @@ -2918,6 +2950,30 @@ dependencies = [ "winapi", ] +[[package]] +name = "userfaultfd" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18d8164d4a8198fa546e7553b529f53e82907214a25fafda4a6f90d978b30a5c" +dependencies = [ + "bitflags", + "libc", + "nix", + "thiserror", + "userfaultfd-sys", +] + +[[package]] +name = "userfaultfd-sys" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ada4f4ae167325015f52cc65f9fb6c251b868d8fb3b6dd0ce2d60e497c4870a" +dependencies = [ + "bindgen 0.57.0", + "cc", + "cfg-if 0.1.10", +] + [[package]] name = "vec_map" version = "0.8.2" @@ -2930,6 +2986,12 @@ version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5a972e5669d67ba988ce3dc826706fb0a8b01471c088cb0b6110b805cc36aed" +[[package]] +name = "void" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" + [[package]] name = "wait-timeout" version = "0.2.0" @@ -3397,8 +3459,10 @@ dependencies = [ "memoffset", "more-asserts", "psm", + "rand 0.7.3", "region", "thiserror", + "userfaultfd", "wasmtime-environ", "winapi", ] diff --git a/crates/environ/src/vmoffsets.rs b/crates/environ/src/vmoffsets.rs index 7f74f46754c1..042f0b9dcfbc 100644 --- a/crates/environ/src/vmoffsets.rs +++ b/crates/environ/src/vmoffsets.rs @@ -50,6 +50,7 @@ fn align(offset: u32, width: u32) -> u32 { /// This class computes offsets to fields within `VMContext` and other /// related structs that JIT code accesses directly. +#[derive(Debug, Clone, Copy)] pub struct VMOffsets { /// The size in bytes of a pointer on the target. pub pointer_size: u8, diff --git a/crates/runtime/Cargo.toml b/crates/runtime/Cargo.toml index 7cea17df8659..bdcb99755aac 100644 --- a/crates/runtime/Cargo.toml +++ b/crates/runtime/Cargo.toml @@ -24,10 +24,14 @@ cfg-if = "1.0" backtrace = "0.3.55" lazy_static = "1.3.0" psm = "0.1.11" +rand = "0.7.3" [target.'cfg(target_os = "windows")'.dependencies] winapi = { version = "0.3.7", features = ["winbase", "memoryapi", "errhandlingapi"] } +[target.'cfg(target_os = "linux")'.dependencies] +userfaultfd = { version = "0.3.0", optional = true } + [build-dependencies] cc = "1.0" diff --git a/crates/runtime/src/instance/allocator.rs b/crates/runtime/src/instance/allocator.rs index 45d6a14445ef..a1d806e4a17e 100644 --- a/crates/runtime/src/instance/allocator.rs +++ b/crates/runtime/src/instance/allocator.rs @@ -26,6 +26,12 @@ use wasmtime_environ::{ ir, Module, ModuleTranslation, ModuleType, OwnedDataInitializer, TableElements, VMOffsets, }; +mod pooling; + +pub use self::pooling::{ + InstanceLimits, ModuleLimits, PoolingAllocationStrategy, PoolingInstanceAllocator, +}; + /// Represents a request for a new runtime instance. pub struct InstanceAllocationRequest<'a> { /// The module being instantiated. @@ -72,11 +78,18 @@ pub enum InstantiationError { /// A trap ocurred during instantiation, after linking. #[error("Trap occurred during instantiation")] Trap(Trap), + + /// A limit on how many instances are supported has been reached. + #[error("Limit of {0} concurrent instances has been reached")] + Limit(u32), } /// An error while creating a fiber stack. #[derive(Error, Debug)] pub enum FiberStackError { + /// Insufficient resources available for the request. + #[error("Insufficient resources: {0}")] + Resource(String), /// An error for when the allocator doesn't support custom fiber stacks. #[error("Custom fiber stacks are not supported by the allocator")] NotSupported, @@ -218,7 +231,7 @@ unsafe fn initialize_vmcontext( globals.len(), ); - // Initialize the defined functions + // Initialize the functions for (index, sig) in instance.module.functions.iter() { let type_index = lookup_shared_signature(*sig); diff --git a/crates/runtime/src/instance/allocator/pooling.rs b/crates/runtime/src/instance/allocator/pooling.rs new file mode 100644 index 000000000000..5bf139555a8f --- /dev/null +++ b/crates/runtime/src/instance/allocator/pooling.rs @@ -0,0 +1,1666 @@ +//! Implements the pooling instance allocator. +//! +//! The pooling instance allocator maps memory in advance +//! and allocates instances, memories, tables, and stacks from +//! a pool of available resources. +//! +//! Using the pooling instance allocator can speed up module instantiation +//! when modules can be constrained based on configurable limits. + +use super::{ + initialize_vmcontext, FiberStackError, InstanceAllocationRequest, InstanceAllocator, + InstanceHandle, InstantiationError, +}; +use crate::{ + instance::Instance, table::max_table_element_size, Memory, Mmap, OnDemandInstanceAllocator, + Table, VMContext, +}; +use rand::Rng; +use std::cell::RefCell; +use std::cmp::min; +use std::convert::TryFrom; +use std::mem; +use std::sync::{Arc, Mutex}; +use wasmtime_environ::{ + entity::{EntitySet, PrimaryMap}, + MemoryStyle, Module, ModuleTranslation, OwnedDataInitializer, Tunables, VMOffsets, + WASM_PAGE_SIZE, +}; + +cfg_if::cfg_if! { + if #[cfg(windows)] { + mod windows; + use windows as imp; + } else if #[cfg(target_os = "linux")] { + mod linux; + use linux as imp; + } else { + mod unix; + use unix as imp; + } +} + +use imp::{create_memory_map, decommit, make_accessible}; + +fn round_up_to_pow2(n: usize, to: usize) -> usize { + debug_assert!(to > 0); + debug_assert!(to.is_power_of_two()); + (n + to - 1) & !(to - 1) +} + +/// Represents the limits placed on a module for compiling with the pooling instance allocator. +#[derive(Debug, Copy, Clone)] +pub struct ModuleLimits { + /// The maximum number of imported functions for a module (default is 1000). + pub imported_functions: u32, + + /// The maximum number of imported tables for a module (default is 0). + pub imported_tables: u32, + + /// The maximum number of imported memories for a module (default is 0). + pub imported_memories: u32, + + /// The maximum number of imported globals for a module (default is 0). + pub imported_globals: u32, + + /// The maximum number of defined types for a module (default is 100). + pub types: u32, + + /// The maximum number of defined functions for a module (default is 10000). + pub functions: u32, + + /// The maximum number of defined tables for a module (default is 1). + pub tables: u32, + + /// The maximum number of defined memories for a module (default is 1). + pub memories: u32, + + /// The maximum number of defined globals for a module (default is 10). + pub globals: u32, + + /// The maximum table elements for any table defined in a module (default is 10000). + /// + /// If a table's minimum element limit is greater than this value, the module will + /// fail to compile. + /// + /// If a table's maximum element limit is unbounded or greater than this value, + /// the maximum will be `table_elements` for the purpose of any `table.grow` instruction. + pub table_elements: u32, + + /// The maximum number of pages for any memory defined in a module (default is 160). + /// + /// The default of 160 means at most 10 MiB of host memory may be committed for each instance. + /// + /// If a memory's minimum page limit is greater than this value, the module will + /// fail to compile. + /// + /// If a memory's maximum page limit is unbounded or greater than this value, + /// the maximum will be `memory_pages` for the purpose of any `memory.grow` instruction. + /// + /// This value cannot exceed any address space limits placed on instances. + pub memory_pages: u32, +} + +impl ModuleLimits { + fn validate_module(&self, module: &Module) -> Result<(), String> { + if module.num_imported_funcs > self.imported_functions as usize { + return Err(format!( + "imported function count of {} exceeds the limit of {}", + module.num_imported_funcs, self.imported_functions + )); + } + + if module.num_imported_tables > self.imported_tables as usize { + return Err(format!( + "imported tables count of {} exceeds the limit of {}", + module.num_imported_tables, self.imported_tables + )); + } + + if module.num_imported_memories > self.imported_memories as usize { + return Err(format!( + "imported memories count of {} exceeds the limit of {}", + module.num_imported_memories, self.imported_memories + )); + } + + if module.num_imported_globals > self.imported_globals as usize { + return Err(format!( + "imported globals count of {} exceeds the limit of {}", + module.num_imported_globals, self.imported_globals + )); + } + + if module.types.len() > self.types as usize { + return Err(format!( + "defined types count of {} exceeds the limit of {}", + module.types.len(), + self.types + )); + } + + let functions = module.functions.len() - module.num_imported_funcs; + if functions > self.functions as usize { + return Err(format!( + "defined functions count of {} exceeds the limit of {}", + functions, self.functions + )); + } + + let tables = module.table_plans.len() - module.num_imported_tables; + if tables > self.tables as usize { + return Err(format!( + "defined tables count of {} exceeds the limit of {}", + tables, self.tables + )); + } + + let memories = module.memory_plans.len() - module.num_imported_memories; + if memories > self.memories as usize { + return Err(format!( + "defined memories count of {} exceeds the limit of {}", + memories, self.memories + )); + } + + let globals = module.globals.len() - module.num_imported_globals; + if globals > self.globals as usize { + return Err(format!( + "defined globals count of {} exceeds the limit of {}", + globals, self.globals + )); + } + + for (i, plan) in module.table_plans.values().as_slice()[module.num_imported_tables..] + .iter() + .enumerate() + { + if plan.table.minimum > self.table_elements { + return Err(format!( + "table index {} has a minimum element size of {} which exceeds the limit of {}", + i, plan.table.minimum, self.table_elements + )); + } + } + + for (i, plan) in module.memory_plans.values().as_slice()[module.num_imported_memories..] + .iter() + .enumerate() + { + if plan.memory.minimum > self.memory_pages { + return Err(format!( + "memory index {} has a minimum page size of {} which exceeds the limit of {}", + i, plan.memory.minimum, self.memory_pages + )); + } + + if let MemoryStyle::Dynamic = plan.style { + return Err(format!( + "memory index {} has an unsupported dynamic memory plan style", + i, + )); + } + } + + Ok(()) + } +} + +impl Default for ModuleLimits { + fn default() -> Self { + // See doc comments for `ModuleLimits` for these default values + Self { + imported_functions: 1000, + imported_tables: 0, + imported_memories: 0, + imported_globals: 0, + types: 100, + functions: 10000, + tables: 1, + memories: 1, + globals: 10, + table_elements: 10000, + memory_pages: 160, + } + } +} + +/// Represents the limits placed on instances by the pooling instance allocator. +#[derive(Debug, Copy, Clone)] +pub struct InstanceLimits { + /// The maximum number of concurrent instances supported (default is 1000). + pub count: u32, + + /// The maximum reserved host address space size to use for each instance in bytes. + /// + /// Note: this value has important performance ramifications. + /// + /// On 64-bit platforms, the default for this value will be 6 GiB. A value of less than 4 GiB will + /// force runtime bounds checking for memory accesses and thus will negatively impact performance. + /// Any value above 4 GiB will start eliding bounds checks provided the `offset` of the memory access is + /// less than (`address_space_size` - 4 GiB). A value of 8 GiB will completely elide *all* bounds + /// checks; consequently, 8 GiB will be the maximum supported value. The default of 6 GiB reserves + /// less host address space for each instance, but a memory access with an offet above 2 GiB will incur + /// runtime bounds checks. + /// + /// On 32-bit platforms, the default for this value will be 10 MiB. A 32-bit host has very limited address + /// space to reserve for a lot of concurrent instances. As a result, runtime bounds checking will be used + /// for all memory accesses. For better runtime performance, a 64-bit host is recommended. + /// + /// This value will be rounded up by the WebAssembly page size (64 KiB). + pub address_space_size: u64, +} + +impl Default for InstanceLimits { + fn default() -> Self { + // See doc comments for `InstanceLimits` for these default values + Self { + count: 1000, + #[cfg(target_pointer_width = "32")] + address_space_size: 0xA00000, + #[cfg(target_pointer_width = "64")] + address_space_size: 0x180000000, + } + } +} + +/// The allocation strategy to use for the pooling instance allocator. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PoolingAllocationStrategy { + /// Allocate from the next available instance. + NextAvailable, + /// Allocate from a random available instance. + Random, +} + +impl PoolingAllocationStrategy { + fn next(&self, free_count: usize) -> usize { + debug_assert!(free_count > 0); + + match self { + Self::NextAvailable => free_count - 1, + Self::Random => rand::thread_rng().gen_range(0, free_count), + } + } +} + +impl Default for PoolingAllocationStrategy { + fn default() -> Self { + Self::NextAvailable + } +} + +// Used to iterate the base address of instance memories and tables. +struct BasePointerIterator { + base: *mut u8, + current: usize, + num: usize, + size: usize, +} + +impl BasePointerIterator { + fn new(base: *mut u8, num: usize, size: usize) -> Self { + Self { + base, + current: 0, + num, + size, + } + } +} + +impl Iterator for BasePointerIterator { + type Item = *mut u8; + + fn next(&mut self) -> Option { + let current = self.current; + if current == self.num { + return None; + } + + self.current += 1; + + Some(unsafe { self.base.add(current * self.size) }) + } + + fn size_hint(&self) -> (usize, Option) { + let remaining = self.num - self.current; + (remaining, Some(remaining)) + } +} + +/// Represents a pool of maximal `Instance` structures. +/// +/// Each index in the pool provides enough space for a maximal `Instance` +/// structure depending on the limits used to create the pool. +/// +/// The pool maintains a free list for fast instance allocation. +#[derive(Debug)] +struct InstancePool { + mapping: Mmap, + offsets: VMOffsets, + instance_size: usize, + max_instances: usize, + free_list: Mutex>, + memories: MemoryPool, + tables: TablePool, +} + +impl InstancePool { + fn new(module_limits: &ModuleLimits, instance_limits: &InstanceLimits) -> Result { + let page_size = region::page::size(); + + // Calculate the maximum size of an Instance structure given the limits + let offsets = VMOffsets { + pointer_size: std::mem::size_of::<*const u8>() as u8, + num_signature_ids: module_limits.types, + num_imported_functions: module_limits.imported_functions, + num_imported_tables: module_limits.imported_tables, + num_imported_memories: module_limits.imported_memories, + num_imported_globals: module_limits.imported_globals, + num_defined_functions: module_limits.functions, + num_defined_tables: module_limits.tables, + num_defined_memories: module_limits.memories, + num_defined_globals: module_limits.globals, + }; + + let instance_size = round_up_to_pow2( + mem::size_of::() + .checked_add(offsets.size_of_vmctx() as usize) + .ok_or_else(|| "instance size exceeds addressable memory".to_string())?, + page_size, + ); + + let max_instances = instance_limits.count as usize; + + let allocation_size = instance_size + .checked_mul(max_instances) + .ok_or_else(|| "total size of instance data exceeds addressable memory".to_string())?; + + let pool = Self { + mapping: create_memory_map(allocation_size, allocation_size)?, + offsets, + instance_size, + max_instances, + free_list: Mutex::new((0..max_instances).collect()), + memories: MemoryPool::new(module_limits, instance_limits)?, + tables: TablePool::new(module_limits, instance_limits)?, + }; + + // Use a default module to initialize the instances to start + let module = Arc::new(Module::default()); + for i in 0..instance_limits.count as usize { + pool.initialize(i, &module); + } + + Ok(pool) + } + + fn initialize(&self, index: usize, module: &Arc) { + debug_assert!(index < self.max_instances); + + unsafe { + let instance_ptr = self.mapping.as_mut_ptr().add(index * self.instance_size); + + // Write a default instance with preallocated memory/table map storage to the ptr + std::ptr::write( + instance_ptr as _, + Instance { + module: module.clone(), + offsets: self.offsets, + memories: PrimaryMap::with_capacity(self.offsets.num_defined_memories as usize), + tables: PrimaryMap::with_capacity(self.offsets.num_defined_tables as usize), + dropped_elements: RefCell::new(EntitySet::new()), + dropped_data: RefCell::new(EntitySet::new()), + host_state: Box::new(()), + vmctx: VMContext {}, + }, + ); + } + } + + fn allocate( + &self, + strategy: PoolingAllocationStrategy, + req: InstanceAllocationRequest, + ) -> Result { + let index = { + let mut free_list = self.free_list.lock().unwrap(); + if free_list.is_empty() { + return Err(InstantiationError::Limit(self.max_instances as u32)); + } + let free_index = strategy.next(free_list.len()); + free_list.swap_remove(free_index) + }; + + unsafe { + debug_assert!(index < self.max_instances); + let instance = + &mut *(self.mapping.as_mut_ptr().add(index * self.instance_size) as *mut Instance); + + instance.module = req.module; + instance.offsets = VMOffsets::new( + std::mem::size_of::<*const u8>() as u8, + instance.module.as_ref(), + ); + instance.host_state = req.host_state; + + Self::set_instance_memories( + instance, + self.memories.get(index), + self.memories.max_wasm_pages, + )?; + Self::set_instance_tables(instance, self.tables.get(index), self.tables.max_elements)?; + + initialize_vmcontext( + instance, + req.imports.functions, + req.imports.tables, + req.imports.memories, + req.imports.globals, + req.finished_functions, + req.lookup_shared_signature, + req.interrupts, + req.externref_activations_table, + req.stack_map_registry, + &|index| instance.memories[index].vmmemory(), + &|index| instance.tables[index].vmtable(), + ); + + Ok(InstanceHandle::new(instance as _)) + } + } + + fn deallocate(&self, handle: &InstanceHandle) { + let addr = handle.instance as usize; + let base = self.mapping.as_ptr() as usize; + + debug_assert!(addr >= base && addr < base + self.mapping.len()); + debug_assert!((addr - base) % self.instance_size == 0); + + let index = (addr - base) / self.instance_size; + debug_assert!(index < self.max_instances); + + unsafe { + // Decommit any linear memories that were used + for (mem, base) in (*handle.instance) + .memories + .values() + .zip(self.memories.get(index)) + { + let size = (mem.size() * WASM_PAGE_SIZE) as usize; + if size > 0 { + decommit(base, size); + } + } + + // Decommit any tables that were used + let table_element_size = max_table_element_size(); + for (table, base) in (*handle.instance) + .tables + .values() + .zip(self.tables.get(index)) + { + let size = round_up_to_pow2( + table.size() as usize * table_element_size, + self.tables.page_size, + ); + if size > 0 { + decommit(base, size); + } + } + } + + { + self.free_list.lock().unwrap().push(index); + } + } + + fn set_instance_memories( + instance: &mut Instance, + mut memories: BasePointerIterator, + max_pages: u32, + ) -> Result<(), InstantiationError> { + let module = instance.module.as_ref(); + + instance.memories.clear(); + + for plan in + (&module.memory_plans.values().as_slice()[module.num_imported_memories..]).iter() + { + instance.memories.push( + Memory::new_static(plan, memories.next().unwrap(), max_pages, make_accessible) + .map_err(InstantiationError::Resource)?, + ); + } + + let mut dropped_data = instance.dropped_data.borrow_mut(); + dropped_data.clear(); + dropped_data.resize(module.passive_data.len()); + + Ok(()) + } + + fn set_instance_tables( + instance: &mut Instance, + mut tables: BasePointerIterator, + max_elements: u32, + ) -> Result<(), InstantiationError> { + let module = instance.module.as_ref(); + + instance.tables.clear(); + + for plan in (&module.table_plans.values().as_slice()[module.num_imported_tables..]).iter() { + let base = tables.next().unwrap(); + + // Make the table data accessible + if unsafe { !make_accessible(base, max_elements as usize * max_table_element_size()) } { + return Err(InstantiationError::Resource( + "failed to make instance memory accessible".into(), + )); + } + + instance + .tables + .push(Table::new_static(plan, base, max_elements)); + } + + let mut dropped_elements = instance.dropped_elements.borrow_mut(); + dropped_elements.clear(); + dropped_elements.resize(module.passive_elements.len()); + + Ok(()) + } +} + +impl Drop for InstancePool { + fn drop(&mut self) { + unsafe { + for i in 0..self.max_instances { + let ptr = self.mapping.as_mut_ptr().add(i * self.instance_size) as *mut Instance; + std::ptr::drop_in_place(ptr); + } + } + } +} + +/// Represents a pool of WebAssembly linear memories. +/// +/// A linear memory is divided into accessible pages and guard pages. +/// +/// Each instance index into the pool returns an iterator over the base addresses +/// of the instance's linear memories. +#[derive(Debug)] +struct MemoryPool { + mapping: Mmap, + memory_size: usize, + max_memories: usize, + max_instances: usize, + max_wasm_pages: u32, +} + +impl MemoryPool { + fn new(module_limits: &ModuleLimits, instance_limits: &InstanceLimits) -> Result { + let memory_size = usize::try_from(instance_limits.address_space_size) + .map_err(|_| "address space size exceeds addressable memory".to_string())?; + + debug_assert!( + memory_size % region::page::size() == 0, + "memory size {} is not a multiple of system page size", + memory_size + ); + + let max_instances = instance_limits.count as usize; + let max_memories = module_limits.memories as usize; + + let allocation_size = memory_size + .checked_mul(max_memories) + .and_then(|c| c.checked_mul(max_instances)) + .ok_or_else(|| { + "total size of instance address space exceeds addressable memory".to_string() + })?; + + Ok(Self { + mapping: create_memory_map(0, allocation_size)?, + memory_size, + max_memories, + max_instances, + max_wasm_pages: module_limits.memory_pages, + }) + } + + fn get(&self, instance_index: usize) -> BasePointerIterator { + debug_assert!(instance_index < self.max_instances); + + let base = unsafe { + self.mapping + .as_mut_ptr() + .add(instance_index * self.memory_size * self.max_memories) as _ + }; + + BasePointerIterator::new(base, self.max_memories, self.memory_size) + } +} + +/// Represents a pool of WebAssembly tables. +/// +/// Each instance index into the pool returns an iterator over the base addresses +/// of the instance's tables. +#[derive(Debug)] +struct TablePool { + mapping: Mmap, + table_size: usize, + max_tables: usize, + max_instances: usize, + page_size: usize, + max_elements: u32, +} + +impl TablePool { + fn new(module_limits: &ModuleLimits, instance_limits: &InstanceLimits) -> Result { + let page_size = region::page::size(); + + let table_size = round_up_to_pow2( + max_table_element_size() + .checked_mul(module_limits.table_elements as usize) + .ok_or_else(|| "table size exceeds addressable memory".to_string())?, + page_size, + ); + + let max_instances = instance_limits.count as usize; + let max_tables = module_limits.tables as usize; + + let allocation_size = table_size + .checked_mul(max_tables) + .and_then(|c| c.checked_mul(max_instances)) + .ok_or_else(|| { + "total size of instance tables exceeds addressable memory".to_string() + })?; + + Ok(Self { + mapping: create_memory_map(0, allocation_size)?, + table_size, + max_tables, + max_instances, + page_size: region::page::size(), + max_elements: module_limits.table_elements, + }) + } + + fn get(&self, instance_index: usize) -> BasePointerIterator { + debug_assert!(instance_index < self.max_instances); + + let base = unsafe { + self.mapping + .as_mut_ptr() + .add(instance_index * self.table_size * self.max_tables) as _ + }; + + BasePointerIterator::new(base, self.max_tables, self.table_size) + } +} + +/// Represents a pool of execution stacks (used for the async fiber implementation). +/// +/// Each index into the pool represents a single execution stack. The maximum number of +/// stacks is the same as the maximum number of instances. +/// +/// As stacks grow downwards, each stack starts (lowest address) with a guard page +/// that can be used to detect stack overflow. +/// +/// The top of the stack (starting stack pointer) is returned when a stack is allocated +/// from the pool. +#[derive(Debug)] +struct StackPool { + mapping: Mmap, + stack_size: usize, + max_instances: usize, + page_size: usize, + free_list: Mutex>, +} + +impl StackPool { + fn new(instance_limits: &InstanceLimits, stack_size: usize) -> Result { + let page_size = region::page::size(); + + // On Windows, don't allocate any fiber stacks as native fibers are always used + // Add a page to the stack size for the guard page when using fiber stacks + let stack_size = if cfg!(windows) || stack_size == 0 { + 0 + } else { + round_up_to_pow2(stack_size, page_size) + .checked_add(page_size) + .ok_or_else(|| "stack size exceeds addressable memory".to_string())? + }; + + let max_instances = instance_limits.count as usize; + + let allocation_size = stack_size.checked_mul(max_instances).ok_or_else(|| { + "total size of execution stacks exceeds addressable memory".to_string() + })?; + + Ok(Self { + mapping: create_memory_map(0, allocation_size)?, + stack_size, + max_instances, + page_size, + free_list: Mutex::new((0..max_instances).collect()), + }) + } + + fn allocate(&self, strategy: PoolingAllocationStrategy) -> Result<*mut u8, FiberStackError> { + // Stacks are not supported if nothing was allocated + if self.stack_size == 0 { + return Err(FiberStackError::NotSupported); + } + + let index = { + let mut free_list = self.free_list.lock().unwrap(); + if free_list.is_empty() { + return Err(FiberStackError::Limit(self.max_instances as u32)); + } + let free_index = strategy.next(free_list.len()); + free_list.swap_remove(free_index) + }; + + debug_assert!(index < self.max_instances); + + unsafe { + // Remove the guard page from the size + let size_without_guard = self.stack_size - self.page_size; + + let bottom_of_stack = self + .mapping + .as_mut_ptr() + .add((index * self.stack_size) + self.page_size); + + // Make the stack accessible (excluding the guard page) + if !make_accessible(bottom_of_stack, size_without_guard) { + return Err(FiberStackError::Resource( + "failed to make instance memory accessible".into(), + )); + } + + // The top of the stack should be returned + Ok(bottom_of_stack.add(size_without_guard)) + } + } + + fn deallocate(&self, top_of_stack: *mut u8) { + debug_assert!(!top_of_stack.is_null()); + + unsafe { + // Remove the guard page from the size + let stack_size = self.stack_size - self.page_size; + let bottom_of_stack = top_of_stack.sub(stack_size); + + let base = self.mapping.as_ptr() as usize; + let start_of_stack = (bottom_of_stack as usize) - self.page_size; + + debug_assert!(start_of_stack >= base && start_of_stack < (base + self.mapping.len())); + debug_assert!((start_of_stack - base) % self.stack_size == 0); + + let index = (start_of_stack - base) / self.stack_size; + debug_assert!(index < self.max_instances); + + decommit(bottom_of_stack, stack_size); + + { + self.free_list.lock().unwrap().push(index); + } + } + } +} + +/// Implements the pooling instance allocator. +/// +/// This allocator interinally maintains pools of instances, memories, tables, and stacks. +/// +/// Note: the resource pools are manually dropped so that the fault handler terminates correctly. +#[derive(Debug)] +pub struct PoolingInstanceAllocator { + strategy: PoolingAllocationStrategy, + module_limits: ModuleLimits, + instance_limits: InstanceLimits, + instances: mem::ManuallyDrop, + stacks: mem::ManuallyDrop, +} + +impl PoolingInstanceAllocator { + /// Creates a new pooling instance allocator with the given strategy and limits. + pub fn new( + strategy: PoolingAllocationStrategy, + module_limits: ModuleLimits, + mut instance_limits: InstanceLimits, + stack_size: usize, + ) -> Result { + if instance_limits.count == 0 { + return Err("the instance count limit cannot be zero".into()); + } + + // Round the instance address space size to the nearest Wasm page size + instance_limits.address_space_size = u64::try_from(round_up_to_pow2( + usize::try_from(instance_limits.address_space_size).unwrap(), + WASM_PAGE_SIZE as usize, + )) + .unwrap(); + + // Cap the instance address space size to 8 GiB (maximum 4 GiB address space + 4 GiB of guard region) + instance_limits.address_space_size = min(instance_limits.address_space_size, 0x200000000); + + // The maximum module memory page count cannot exceed 65536 pages + if module_limits.memory_pages > 0x10000 { + return Err(format!( + "module memory page limit of {} exceeds the maximum of 65536", + module_limits.memory_pages + )); + } + + // The maximum module memory page count cannot exceed the instance address space size + if (module_limits.memory_pages * WASM_PAGE_SIZE) as u64 > instance_limits.address_space_size + { + return Err(format!( + "module memory page limit of {} pages exeeds the instance address space size limit of {} bytes", + module_limits.memory_pages, + instance_limits.address_space_size + )); + } + + Ok(Self { + strategy, + module_limits, + instance_limits, + instances: mem::ManuallyDrop::new(InstancePool::new(&module_limits, &instance_limits)?), + stacks: mem::ManuallyDrop::new(StackPool::new(&instance_limits, stack_size)?), + }) + } +} + +impl Drop for PoolingInstanceAllocator { + fn drop(&mut self) { + // There are manually dropped for the future uffd implementation + unsafe { + mem::ManuallyDrop::drop(&mut self.instances); + mem::ManuallyDrop::drop(&mut self.stacks); + } + } +} + +unsafe impl InstanceAllocator for PoolingInstanceAllocator { + fn validate_module(&self, translation: &ModuleTranslation) -> Result<(), String> { + self.module_limits.validate_module(&translation.module) + } + + fn adjust_tunables(&self, tunables: &mut Tunables) { + let address_space_size = self.instance_limits.address_space_size; + + // For address spaces larger than 4 GiB, use a guard region to elide + if address_space_size >= 0x100000000 { + tunables.static_memory_bound = 0x10000; // in Wasm pages + tunables.static_memory_offset_guard_size = address_space_size - 0x100000000; + } else { + tunables.static_memory_bound = + u32::try_from(address_space_size).unwrap() / WASM_PAGE_SIZE; + tunables.static_memory_offset_guard_size = 0; + } + + // Treat the static memory bound as the maximum for unbounded Wasm memories + // Because we guarantee a module cannot compile unless it fits in the limits of + // the pool allocator, this ensures all memories are treated as static (i.e. immovable). + tunables.static_memory_bound_is_maximum = true; + } + + unsafe fn allocate( + &self, + req: InstanceAllocationRequest, + ) -> Result { + self.instances.allocate(self.strategy, req) + } + + unsafe fn initialize( + &self, + handle: &InstanceHandle, + is_bulk_memory: bool, + data_initializers: &Arc<[OwnedDataInitializer]>, + ) -> Result<(), InstantiationError> { + // TODO: refactor this implementation + + // Check initializer bounds before initializing anything. Only do this + // when bulk memory is disabled, since the bulk memory proposal changes + // instantiation such that the intermediate results of failed + // initializations are visible. + if !is_bulk_memory { + OnDemandInstanceAllocator::check_table_init_bounds(handle.instance())?; + OnDemandInstanceAllocator::check_memory_init_bounds( + handle.instance(), + data_initializers.as_ref(), + )?; + } + + // Apply fallible initializers. Note that this can "leak" state even if + // it fails. + OnDemandInstanceAllocator::initialize_tables(handle.instance())?; + OnDemandInstanceAllocator::initialize_memories( + handle.instance(), + data_initializers.as_ref(), + )?; + + Ok(()) + } + + unsafe fn deallocate(&self, handle: &InstanceHandle) { + self.instances.deallocate(handle); + } + + fn allocate_fiber_stack(&self) -> Result<*mut u8, FiberStackError> { + self.stacks.allocate(self.strategy) + } + + unsafe fn deallocate_fiber_stack(&self, stack: *mut u8) { + self.stacks.deallocate(stack); + } +} + +#[cfg(test)] +mod test { + use super::*; + use crate::{Imports, VMSharedSignatureIndex}; + use wasmtime_environ::{ + entity::EntityRef, + ir::Type, + wasm::{Global, GlobalInit, Memory, SignatureIndex, Table, TableElementType, WasmType}, + MemoryPlan, ModuleType, TablePlan, TableStyle, + }; + + #[test] + fn test_module_imported_functions_limit() { + let limits = ModuleLimits { + imported_functions: 0, + ..Default::default() + }; + + let mut module = Module::default(); + + module.functions.push(SignatureIndex::new(0)); + assert_eq!(limits.validate_module(&module), Ok(())); + + module.num_imported_funcs = 1; + assert_eq!( + limits.validate_module(&module), + Err("imported function count of 1 exceeds the limit of 0".into()) + ); + } + + #[test] + fn test_module_imported_tables_limit() { + let limits = ModuleLimits { + imported_tables: 0, + ..Default::default() + }; + + let mut module = Module::default(); + + module.table_plans.push(TablePlan { + style: TableStyle::CallerChecksSignature, + table: Table { + wasm_ty: WasmType::FuncRef, + ty: TableElementType::Func, + minimum: 0, + maximum: None, + }, + }); + + assert_eq!(limits.validate_module(&module), Ok(())); + + module.num_imported_tables = 1; + assert_eq!( + limits.validate_module(&module), + Err("imported tables count of 1 exceeds the limit of 0".into()) + ); + } + + #[test] + fn test_module_imported_memories_limit() { + let limits = ModuleLimits { + imported_memories: 0, + ..Default::default() + }; + + let mut module = Module::default(); + + module.memory_plans.push(MemoryPlan { + style: MemoryStyle::Static { bound: 0 }, + memory: Memory { + minimum: 0, + maximum: None, + shared: false, + }, + offset_guard_size: 0, + }); + + assert_eq!(limits.validate_module(&module), Ok(())); + + module.num_imported_memories = 1; + assert_eq!( + limits.validate_module(&module), + Err("imported memories count of 1 exceeds the limit of 0".into()) + ); + } + + #[test] + fn test_module_imported_globals_limit() { + let limits = ModuleLimits { + imported_globals: 0, + ..Default::default() + }; + + let mut module = Module::default(); + + module.globals.push(Global { + wasm_ty: WasmType::I32, + ty: Type::int(32).unwrap(), + mutability: false, + initializer: GlobalInit::I32Const(0), + }); + + assert_eq!(limits.validate_module(&module), Ok(())); + + module.num_imported_globals = 1; + assert_eq!( + limits.validate_module(&module), + Err("imported globals count of 1 exceeds the limit of 0".into()) + ); + } + + #[test] + fn test_module_defined_types_limit() { + let limits = ModuleLimits { + types: 0, + ..Default::default() + }; + + let mut module = Module::default(); + assert_eq!(limits.validate_module(&module), Ok(())); + + module + .types + .push(ModuleType::Function(SignatureIndex::new(0))); + assert_eq!( + limits.validate_module(&module), + Err("defined types count of 1 exceeds the limit of 0".into()) + ); + } + + #[test] + fn test_module_defined_functions_limit() { + let limits = ModuleLimits { + functions: 0, + ..Default::default() + }; + + let mut module = Module::default(); + assert_eq!(limits.validate_module(&module), Ok(())); + + module.functions.push(SignatureIndex::new(0)); + assert_eq!( + limits.validate_module(&module), + Err("defined functions count of 1 exceeds the limit of 0".into()) + ); + } + + #[test] + fn test_module_defined_tables_limit() { + let limits = ModuleLimits { + tables: 0, + ..Default::default() + }; + + let mut module = Module::default(); + assert_eq!(limits.validate_module(&module), Ok(())); + + module.table_plans.push(TablePlan { + style: TableStyle::CallerChecksSignature, + table: Table { + wasm_ty: WasmType::FuncRef, + ty: TableElementType::Func, + minimum: 0, + maximum: None, + }, + }); + assert_eq!( + limits.validate_module(&module), + Err("defined tables count of 1 exceeds the limit of 0".into()) + ); + } + + #[test] + fn test_module_defined_memories_limit() { + let limits = ModuleLimits { + memories: 0, + ..Default::default() + }; + + let mut module = Module::default(); + assert_eq!(limits.validate_module(&module), Ok(())); + + module.memory_plans.push(MemoryPlan { + style: MemoryStyle::Static { bound: 0 }, + memory: Memory { + minimum: 0, + maximum: None, + shared: false, + }, + offset_guard_size: 0, + }); + assert_eq!( + limits.validate_module(&module), + Err("defined memories count of 1 exceeds the limit of 0".into()) + ); + } + + #[test] + fn test_module_defined_globals_limit() { + let limits = ModuleLimits { + globals: 0, + ..Default::default() + }; + + let mut module = Module::default(); + assert_eq!(limits.validate_module(&module), Ok(())); + + module.globals.push(Global { + wasm_ty: WasmType::I32, + ty: Type::int(32).unwrap(), + mutability: false, + initializer: GlobalInit::I32Const(0), + }); + assert_eq!( + limits.validate_module(&module), + Err("defined globals count of 1 exceeds the limit of 0".into()) + ); + } + + #[test] + fn test_module_table_minimum_elements_limit() { + let limits = ModuleLimits { + tables: 1, + table_elements: 10, + ..Default::default() + }; + + let mut module = Module::default(); + module.table_plans.push(TablePlan { + style: TableStyle::CallerChecksSignature, + table: Table { + wasm_ty: WasmType::FuncRef, + ty: TableElementType::Func, + minimum: 11, + maximum: None, + }, + }); + assert_eq!( + limits.validate_module(&module), + Err( + "table index 0 has a minimum element size of 11 which exceeds the limit of 10" + .into() + ) + ); + } + + #[test] + fn test_module_memory_minimum_size_limit() { + let limits = ModuleLimits { + memories: 1, + memory_pages: 5, + ..Default::default() + }; + + let mut module = Module::default(); + module.memory_plans.push(MemoryPlan { + style: MemoryStyle::Static { bound: 0 }, + memory: Memory { + minimum: 6, + maximum: None, + shared: false, + }, + offset_guard_size: 0, + }); + assert_eq!( + limits.validate_module(&module), + Err("memory index 0 has a minimum page size of 6 which exceeds the limit of 5".into()) + ); + } + + #[test] + fn test_module_with_dynamic_memory_style() { + let limits = ModuleLimits { + memories: 1, + memory_pages: 5, + ..Default::default() + }; + + let mut module = Module::default(); + module.memory_plans.push(MemoryPlan { + style: MemoryStyle::Dynamic, + memory: Memory { + minimum: 1, + maximum: None, + shared: false, + }, + offset_guard_size: 0, + }); + assert_eq!( + limits.validate_module(&module), + Err("memory index 0 has an unsupported dynamic memory plan style".into()) + ); + } + + #[test] + fn test_next_available_allocation_strategy() { + let strat = PoolingAllocationStrategy::NextAvailable; + assert_eq!(strat.next(10), 9); + assert_eq!(strat.next(5), 4); + assert_eq!(strat.next(1), 0); + } + + #[test] + fn test_random_allocation_strategy() { + let strat = PoolingAllocationStrategy::Random; + assert!(strat.next(100) < 100); + assert_eq!(strat.next(1), 0); + } + + #[test] + fn test_base_pointer_iterator() { + let mut iter = BasePointerIterator::new(std::ptr::null_mut(), 5, 3); + + assert_eq!(iter.next(), Some(0usize as _)); + assert_eq!(iter.next(), Some(3usize as _)); + assert_eq!(iter.next(), Some(6usize as _)); + assert_eq!(iter.next(), Some(9usize as _)); + assert_eq!(iter.next(), Some(12usize as _)); + assert_eq!(iter.next(), None); + + let mut iter = BasePointerIterator::new(std::ptr::null_mut(), 0, 10); + assert_eq!(iter.next(), None); + } + + #[cfg(target_pointer_width = "64")] + #[test] + fn test_instance_pool() -> Result<(), String> { + let module_limits = ModuleLimits { + imported_functions: 0, + imported_tables: 0, + imported_memories: 0, + imported_globals: 0, + types: 0, + functions: 0, + tables: 1, + memories: 1, + globals: 0, + table_elements: 10, + memory_pages: 10, + }; + let instance_limits = InstanceLimits { + count: 3, + address_space_size: 4096, + }; + + let instances = InstancePool::new(&module_limits, &instance_limits)?; + + assert_eq!( + instances.offsets.pointer_size, + std::mem::size_of::<*const u8>() as u8 + ); + assert_eq!(instances.offsets.num_signature_ids, 0); + assert_eq!(instances.offsets.num_imported_functions, 0); + assert_eq!(instances.offsets.num_imported_tables, 0); + assert_eq!(instances.offsets.num_imported_memories, 0); + assert_eq!(instances.offsets.num_imported_globals, 0); + assert_eq!(instances.offsets.num_defined_functions, 0); + assert_eq!(instances.offsets.num_defined_tables, 1); + assert_eq!(instances.offsets.num_defined_memories, 1); + assert_eq!(instances.offsets.num_defined_globals, 0); + assert_eq!(instances.instance_size, 4096); + assert_eq!(instances.max_instances, 3); + + assert_eq!( + &*instances + .free_list + .lock() + .map_err(|_| "failed to lock".to_string())?, + &[0, 1, 2], + ); + + let mut handles = Vec::new(); + let module = Arc::new(Module::default()); + let finished_functions = &PrimaryMap::new(); + + for _ in (0..3).rev() { + handles.push( + instances + .allocate( + PoolingAllocationStrategy::NextAvailable, + InstanceAllocationRequest { + module: module.clone(), + finished_functions, + imports: Imports { + functions: &[], + tables: &[], + memories: &[], + globals: &[], + }, + lookup_shared_signature: &|_| VMSharedSignatureIndex::default(), + host_state: Box::new(()), + interrupts: std::ptr::null(), + externref_activations_table: std::ptr::null_mut(), + stack_map_registry: std::ptr::null_mut(), + }, + ) + .expect("allocation should succeed"), + ); + } + + assert_eq!( + &*instances + .free_list + .lock() + .map_err(|_| "failed to lock".to_string())?, + &[], + ); + + match instances.allocate( + PoolingAllocationStrategy::NextAvailable, + InstanceAllocationRequest { + module: module.clone(), + finished_functions, + imports: Imports { + functions: &[], + tables: &[], + memories: &[], + globals: &[], + }, + lookup_shared_signature: &|_| VMSharedSignatureIndex::default(), + host_state: Box::new(()), + interrupts: std::ptr::null(), + externref_activations_table: std::ptr::null_mut(), + stack_map_registry: std::ptr::null_mut(), + }, + ) { + Err(InstantiationError::Limit(3)) => {} + _ => panic!("unexpected error"), + }; + + for handle in handles.drain(..) { + instances.deallocate(&handle); + } + + assert_eq!( + &*instances + .free_list + .lock() + .map_err(|_| "failed to lock".to_string())?, + &[2, 1, 0], + ); + + Ok(()) + } + + #[cfg(target_pointer_width = "64")] + #[test] + fn test_memory_pool() -> Result<(), String> { + let pool = MemoryPool::new( + &ModuleLimits { + imported_functions: 0, + imported_tables: 0, + imported_memories: 0, + imported_globals: 0, + types: 0, + functions: 0, + tables: 0, + memories: 3, + globals: 0, + table_elements: 0, + memory_pages: 10, + }, + &InstanceLimits { + count: 5, + address_space_size: WASM_PAGE_SIZE as u64, + }, + )?; + + assert_eq!(pool.memory_size, WASM_PAGE_SIZE as usize); + assert_eq!(pool.max_memories, 3); + assert_eq!(pool.max_instances, 5); + assert_eq!(pool.max_wasm_pages, 10); + + let base = pool.mapping.as_ptr() as usize; + + for i in 0..5 { + let mut iter = pool.get(i); + + for j in 0..3 { + assert_eq!( + iter.next().unwrap() as usize - base, + ((i * 3) + j) * pool.memory_size + ); + } + + assert_eq!(iter.next(), None); + } + + Ok(()) + } + + #[cfg(target_pointer_width = "64")] + #[test] + fn test_table_pool() -> Result<(), String> { + let pool = TablePool::new( + &ModuleLimits { + imported_functions: 0, + imported_tables: 0, + imported_memories: 0, + imported_globals: 0, + types: 0, + functions: 0, + tables: 4, + memories: 0, + globals: 0, + table_elements: 100, + memory_pages: 0, + }, + &InstanceLimits { + count: 7, + address_space_size: WASM_PAGE_SIZE as u64, + }, + )?; + + assert_eq!(pool.table_size, 4096); + assert_eq!(pool.max_tables, 4); + assert_eq!(pool.max_instances, 7); + assert_eq!(pool.page_size, 4096); + assert_eq!(pool.max_elements, 100); + + let base = pool.mapping.as_ptr() as usize; + + for i in 0..7 { + let mut iter = pool.get(i); + + for j in 0..4 { + assert_eq!( + iter.next().unwrap() as usize - base, + ((i * 4) + j) * pool.table_size + ); + } + + assert_eq!(iter.next(), None); + } + + Ok(()) + } + + #[cfg(target_pointer_width = "64")] + #[test] + fn test_stack_pool() -> Result<(), String> { + let pool = StackPool::new( + &InstanceLimits { + count: 10, + address_space_size: 0, + }, + 1, + )?; + + assert_eq!(pool.stack_size, 8192); + assert_eq!(pool.max_instances, 10); + assert_eq!(pool.page_size, 4096); + + assert_eq!( + &*pool + .free_list + .lock() + .map_err(|_| "failed to lock".to_string())?, + &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9], + ); + + let base = pool.mapping.as_ptr() as usize; + + let mut stacks = Vec::new(); + for i in (0..10).rev() { + let stack = pool + .allocate(PoolingAllocationStrategy::NextAvailable) + .expect("allocation should succeed"); + assert_eq!(((stack as usize - base) / pool.stack_size) - 1, i); + stacks.push(stack); + } + + assert_eq!( + &*pool + .free_list + .lock() + .map_err(|_| "failed to lock".to_string())?, + &[], + ); + + match pool + .allocate(PoolingAllocationStrategy::NextAvailable) + .unwrap_err() + { + FiberStackError::Limit(10) => {} + _ => panic!("unexpected error"), + }; + + for stack in stacks { + pool.deallocate(stack); + } + + assert_eq!( + &*pool + .free_list + .lock() + .map_err(|_| "failed to lock".to_string())?, + &[9, 8, 7, 6, 5, 4, 3, 2, 1, 0], + ); + + Ok(()) + } + + #[test] + fn test_pooling_allocator_with_zero_instance_count() { + assert_eq!( + PoolingInstanceAllocator::new( + PoolingAllocationStrategy::Random, + ModuleLimits::default(), + InstanceLimits { + count: 0, + ..Default::default() + }, + 4096 + ) + .expect_err("expected a failure constructing instance allocator"), + "the instance count limit cannot be zero" + ); + } + + #[test] + fn test_pooling_allocator_with_memory_pages_exeeded() { + assert_eq!( + PoolingInstanceAllocator::new( + PoolingAllocationStrategy::Random, + ModuleLimits { + memory_pages: 0x10001, + ..Default::default() + }, + InstanceLimits { + count: 1, + address_space_size: 1, + }, + 4096 + ) + .expect_err("expected a failure constructing instance allocator"), + "module memory page limit of 65537 exceeds the maximum of 65536" + ); + } + + #[test] + fn test_pooling_allocator_with_address_space_exeeded() { + assert_eq!( + PoolingInstanceAllocator::new( + PoolingAllocationStrategy::Random, + ModuleLimits { + memory_pages: 2, + ..Default::default() + }, + InstanceLimits { + count: 1, + address_space_size: 1, + }, + 4096, + ) + .expect_err("expected a failure constructing instance allocator"), + "module memory page limit of 2 pages exeeds the instance address space size limit of 65536 bytes" + ); + } + + #[cfg_attr(target_arch = "aarch64", ignore)] // https://github.com/bytecodealliance/wasmtime/pull/2518#issuecomment-747280133 + #[cfg(unix)] + #[test] + fn test_stack_zeroed() -> Result<(), String> { + let allocator = PoolingInstanceAllocator::new( + PoolingAllocationStrategy::NextAvailable, + ModuleLimits { + imported_functions: 0, + types: 0, + functions: 0, + tables: 0, + memories: 0, + globals: 0, + table_elements: 0, + memory_pages: 0, + ..Default::default() + }, + InstanceLimits { + count: 1, + address_space_size: 1, + }, + 4096, + )?; + + unsafe { + for _ in 0..10 { + let stack = allocator + .allocate_fiber_stack() + .map_err(|e| format!("failed to allocate stack: {}", e))?; + + // The stack pointer is at the top, so decerement it first + let addr = stack.sub(1); + + assert_eq!(*addr, 0); + *addr = 1; + + allocator.deallocate_fiber_stack(stack); + } + } + + Ok(()) + } +} diff --git a/crates/runtime/src/instance/allocator/pooling/linux.rs b/crates/runtime/src/instance/allocator/pooling/linux.rs new file mode 100644 index 000000000000..4a8a43f08f03 --- /dev/null +++ b/crates/runtime/src/instance/allocator/pooling/linux.rs @@ -0,0 +1,22 @@ +use crate::Mmap; + +pub unsafe fn make_accessible(addr: *mut u8, len: usize) -> bool { + region::protect(addr, len, region::Protection::READ_WRITE).is_ok() +} + +pub unsafe fn decommit(addr: *mut u8, len: usize) { + region::protect(addr, len, region::Protection::NONE).unwrap(); + + // On Linux, this is enough to cause the kernel to initialize the pages to 0 on next access + assert_eq!( + libc::madvise(addr as _, len, libc::MADV_DONTNEED), + 0, + "madvise failed to mark pages as missing: {}", + std::io::Error::last_os_error() + ); +} + +pub fn create_memory_map(accessible_size: usize, mapping_size: usize) -> Result { + Mmap::accessible_reserved(accessible_size, mapping_size) + .map_err(|e| format!("failed to allocate pool memory: {}", e)) +} diff --git a/crates/runtime/src/instance/allocator/pooling/unix.rs b/crates/runtime/src/instance/allocator/pooling/unix.rs new file mode 100644 index 000000000000..900e73d1747e --- /dev/null +++ b/crates/runtime/src/instance/allocator/pooling/unix.rs @@ -0,0 +1,26 @@ +use crate::Mmap; + +pub unsafe fn make_accessible(addr: *mut u8, len: usize) -> bool { + region::protect(addr, len, region::Protection::READ_WRITE).is_ok() +} + +pub unsafe fn decommit(addr: *mut u8, len: usize) { + assert_eq!( + libc::mmap( + addr as _, + len, + libc::PROT_NONE, + libc::MAP_PRIVATE | libc::MAP_ANON | libc::MAP_FIXED, + -1, + 0, + ) as *mut u8, + addr, + "mmap failed to remap pages: {}", + std::io::Error::last_os_error() + ); +} + +pub fn create_memory_map(accessible_size: usize, mapping_size: usize) -> Result { + Mmap::accessible_reserved(accessible_size, mapping_size) + .map_err(|e| format!("failed to allocate pool memory: {}", e)) +} diff --git a/crates/runtime/src/instance/allocator/pooling/windows.rs b/crates/runtime/src/instance/allocator/pooling/windows.rs new file mode 100644 index 000000000000..fe8566558ae0 --- /dev/null +++ b/crates/runtime/src/instance/allocator/pooling/windows.rs @@ -0,0 +1,21 @@ +use crate::Mmap; +use winapi::um::memoryapi::{VirtualAlloc, VirtualFree}; +use winapi::um::winnt::{MEM_COMMIT, MEM_DECOMMIT, PAGE_READWRITE}; + +pub unsafe fn make_accessible(addr: *mut u8, len: usize) -> bool { + // This doesn't use the `region` crate because the memory needs to be committed + !VirtualAlloc(addr as _, len, MEM_COMMIT, PAGE_READWRITE).is_null() +} + +pub unsafe fn decommit(addr: *mut u8, len: usize) { + assert!( + VirtualFree(addr as _, len, MEM_DECOMMIT) != 0, + "failed to decommit memory pages: {}", + std::io::Error::last_os_error() + ); +} + +pub fn create_memory_map(accessible_size: usize, mapping_size: usize) -> Result { + Mmap::accessible_reserved(accessible_size, mapping_size) + .map_err(|e| format!("failed to allocate pool memory: {}", e)) +} diff --git a/crates/runtime/src/lib.rs b/crates/runtime/src/lib.rs index f33cfd8f381c..b53e73c1623c 100644 --- a/crates/runtime/src/lib.rs +++ b/crates/runtime/src/lib.rs @@ -38,8 +38,9 @@ pub use crate::export::*; pub use crate::externref::*; pub use crate::imports::Imports; pub use crate::instance::{ - FiberStackError, InstanceAllocationRequest, InstanceAllocator, InstanceHandle, - InstantiationError, LinkError, OnDemandInstanceAllocator, RuntimeInstance, + FiberStackError, InstanceAllocationRequest, InstanceAllocator, InstanceHandle, InstanceLimits, + InstantiationError, LinkError, ModuleLimits, OnDemandInstanceAllocator, + PoolingAllocationStrategy, PoolingInstanceAllocator, RuntimeInstance, }; pub use crate::jit_int::GdbJitImageRegistration; pub use crate::memory::{Memory, RuntimeLinearMemory, RuntimeMemoryCreator}; diff --git a/crates/runtime/src/memory.rs b/crates/runtime/src/memory.rs index 1b865c0eecda..7d248b136189 100644 --- a/crates/runtime/src/memory.rs +++ b/crates/runtime/src/memory.rs @@ -164,7 +164,7 @@ impl RuntimeLinearMemory for MmapMemory { /// Return a `VMMemoryDefinition` for exposing the memory to compiled wasm code. fn vmmemory(&self) -> VMMemoryDefinition { - let mut mmap = self.mmap.borrow_mut(); + let mmap = self.mmap.borrow(); VMMemoryDefinition { base: mmap.alloc.as_mut_ptr(), current_length: mmap.size as usize * WASM_PAGE_SIZE as usize, @@ -177,7 +177,7 @@ enum MemoryStorage { base: *mut u8, size: Cell, maximum: u32, - make_accessible: Option bool>, + make_accessible: unsafe fn(*mut u8, usize) -> bool, }, Dynamic(Box), } @@ -203,13 +203,13 @@ impl Memory { plan: &MemoryPlan, base: *mut u8, maximum: u32, - make_accessible: Option bool>, + make_accessible: unsafe fn(*mut u8, usize) -> bool, ) -> Result { if plan.memory.minimum > 0 { - if let Some(make_accessible) = &make_accessible { - if !make_accessible(base, plan.memory.minimum as usize * WASM_PAGE_SIZE as usize) { - return Err("memory cannot be made accessible".into()); - } + if unsafe { + !make_accessible(base, plan.memory.minimum as usize * WASM_PAGE_SIZE as usize) + } { + return Err("memory cannot be made accessible".into()); } } @@ -258,10 +258,8 @@ impl Memory { let start = usize::try_from(old_size).unwrap() * WASM_PAGE_SIZE as usize; let len = usize::try_from(delta).unwrap() * WASM_PAGE_SIZE as usize; - if let Some(make_accessible) = make_accessible { - if !make_accessible(unsafe { base.add(start) }, len) { - return None; - } + if unsafe { !make_accessible(base.add(start), len) } { + return None; } size.set(new_size); diff --git a/crates/runtime/src/mmap.rs b/crates/runtime/src/mmap.rs index 483ec5be0963..61ccda9276a1 100644 --- a/crates/runtime/src/mmap.rs +++ b/crates/runtime/src/mmap.rs @@ -234,7 +234,7 @@ impl Mmap { } /// Return the allocated memory as a mutable pointer to u8. - pub fn as_mut_ptr(&mut self) -> *mut u8 { + pub fn as_mut_ptr(&self) -> *mut u8 { self.ptr as *mut u8 } @@ -247,6 +247,11 @@ impl Mmap { pub fn is_empty(&self) -> bool { self.len() == 0 } + + #[allow(dead_code)] + pub(crate) unsafe fn from_raw(ptr: usize, len: usize) -> Self { + Self { ptr, len } + } } impl Drop for Mmap { diff --git a/crates/runtime/src/table.rs b/crates/runtime/src/table.rs index 4a2d0bd4de0e..f8281c9cfa06 100644 --- a/crates/runtime/src/table.rs +++ b/crates/runtime/src/table.rs @@ -66,6 +66,20 @@ enum TableElements { ExternRefs(Vec>), } +// Ideally this should be static assertion that table elements are pointer-sized +#[inline(always)] +pub(crate) fn max_table_element_size() -> usize { + debug_assert_eq!( + std::mem::size_of::<*mut VMCallerCheckedAnyfunc>(), + std::mem::size_of::<*const ()>() + ); + debug_assert_eq!( + std::mem::size_of::>(), + std::mem::size_of::<*const ()>() + ); + std::mem::size_of::<*const ()>() +} + #[derive(Debug)] enum TableStorage { Static { diff --git a/crates/wasmtime/src/config.rs b/crates/wasmtime/src/config.rs index 19151600b54f..77d5c52be789 100644 --- a/crates/wasmtime/src/config.rs +++ b/crates/wasmtime/src/config.rs @@ -14,7 +14,10 @@ use wasmtime_environ::settings::{self, Configurable, SetError}; use wasmtime_environ::{isa, isa::TargetIsa, Tunables}; use wasmtime_jit::{native, CompilationStrategy, Compiler}; use wasmtime_profiling::{JitDumpAgent, NullProfilerAgent, ProfilingAgent, VTuneAgent}; -use wasmtime_runtime::{InstanceAllocator, OnDemandInstanceAllocator}; +use wasmtime_runtime::{InstanceAllocator, OnDemandInstanceAllocator, PoolingInstanceAllocator}; + +// Re-export the limit structures for the pooling allocator +pub use wasmtime_runtime::{InstanceLimits, ModuleLimits, PoolingAllocationStrategy}; /// Represents the module instance allocation strategy to use. #[derive(Clone)] @@ -26,6 +29,19 @@ pub enum InstanceAllocationStrategy { /// /// This is the default allocation strategy for Wasmtime. OnDemand, + /// The pooling instance allocation strategy. + /// + /// A pool of resources is created in advance and module instantiation reuses resources + /// from the pool. Resources are returned to the pool when the `Store` referencing the instance + /// is dropped. + Pooling { + /// The allocation strategy to use. + strategy: PoolingAllocationStrategy, + /// The module limits to use. + module_limits: ModuleLimits, + /// The instance limits to use. + instance_limits: InstanceLimits, + }, } impl Default for InstanceAllocationStrategy { @@ -205,6 +221,9 @@ impl Config { /// on stack overflow, a host function that overflows the stack will /// abort the process. /// + /// `max_wasm_stack` must be set prior to setting an instance allocation + /// strategy. + /// /// By default this option is 1 MiB. pub fn max_wasm_stack(&mut self, size: usize) -> Result<&mut Self> { #[cfg(feature = "async")] @@ -216,6 +235,12 @@ impl Config { bail!("wasm stack size cannot be zero"); } + if self.instance_allocator.is_some() { + bail!( + "wasm stack size cannot be modified after setting an instance allocation strategy" + ); + } + self.max_wasm_stack = size; Ok(self) } @@ -230,12 +255,20 @@ impl Config { /// close to one another; doing so may cause host functions to overflow the /// stack and abort the process. /// + /// `async_stack_size` must be set prior to setting an instance allocation + /// strategy. + /// /// By default this option is 2 MiB. #[cfg(feature = "async")] pub fn async_stack_size(&mut self, size: usize) -> Result<&mut Self> { if size < self.max_wasm_stack { bail!("async stack size cannot be less than the maximum wasm stack size"); } + if self.instance_allocator.is_some() { + bail!( + "async stack size cannot be modified after setting an instance allocation strategy" + ); + } self.async_stack_size = size; Ok(self) } @@ -577,14 +610,35 @@ impl Config { } /// Sets the instance allocation strategy to use. - pub fn with_instance_allocation_strategy( + pub fn with_allocation_strategy( &mut self, strategy: InstanceAllocationStrategy, - ) -> &mut Self { + ) -> Result<&mut Self> { self.instance_allocator = match strategy { InstanceAllocationStrategy::OnDemand => None, + InstanceAllocationStrategy::Pooling { + strategy, + module_limits, + instance_limits, + } => { + #[cfg(feature = "async")] + let stack_size = self.async_stack_size; + + #[cfg(not(feature = "async"))] + let stack_size = 0; + + Some(Arc::new( + PoolingInstanceAllocator::new( + strategy, + module_limits, + instance_limits, + stack_size, + ) + .map_err(|e| anyhow::anyhow!(e))?, + )) + } }; - self + Ok(self) } /// Configures the maximum size, in bytes, where a linear memory is diff --git a/tests/all/async_functions.rs b/tests/all/async_functions.rs index 2f83b62bf01a..eb3ea0961199 100644 --- a/tests/all/async_functions.rs +++ b/tests/all/async_functions.rs @@ -364,3 +364,37 @@ fn fuel_eventually_finishes() { let instance = Instance::new_async(&store, &module, &[]); run(instance).unwrap(); } + +#[test] +fn async_with_pooling_stacks() { + let mut config = Config::new(); + config + .with_allocation_strategy(InstanceAllocationStrategy::Pooling { + strategy: PoolingAllocationStrategy::NextAvailable, + module_limits: ModuleLimits { + memory_pages: 1, + table_elements: 0, + ..Default::default() + }, + instance_limits: InstanceLimits { + count: 1, + address_space_size: 1, + }, + }) + .expect("pooling allocator created"); + + let engine = Engine::new(&config); + let store = Store::new_async(&engine); + let func = Func::new_async( + &store, + FuncType::new(None, None), + (), + move |_caller, _state, _params, _results| Box::new(async { Ok(()) }), + ); + run(func.call_async(&[])).unwrap(); + run(func.call_async(&[])).unwrap(); + let future1 = func.call_async(&[]); + let future2 = func.call_async(&[]); + run(future2).unwrap(); + run(future1).unwrap(); +} diff --git a/tests/all/main.rs b/tests/all/main.rs index 526c5eb4c6b1..d965bbd1accb 100644 --- a/tests/all/main.rs +++ b/tests/all/main.rs @@ -18,6 +18,7 @@ mod module; mod module_linking; mod module_serialize; mod name; +mod pooling_allocator; mod stack_overflow; mod table; mod traps; diff --git a/tests/all/pooling_allocator.rs b/tests/all/pooling_allocator.rs new file mode 100644 index 000000000000..846dac647e4b --- /dev/null +++ b/tests/all/pooling_allocator.rs @@ -0,0 +1,430 @@ +use anyhow::Result; +use wasmtime::*; + +#[test] +fn successful_instantiation() -> Result<()> { + let mut config = Config::new(); + config.with_allocation_strategy(InstanceAllocationStrategy::Pooling { + strategy: PoolingAllocationStrategy::NextAvailable, + module_limits: ModuleLimits { + memory_pages: 1, + table_elements: 10, + ..Default::default() + }, + instance_limits: InstanceLimits { + count: 1, + address_space_size: 1, + }, + })?; + + let engine = Engine::new(&config); + let module = Module::new(&engine, r#"(module (memory 1) (table 10 funcref))"#)?; + + // Module should instantiate + let store = Store::new(&engine); + Instance::new(&store, &module, &[])?; + + Ok(()) +} + +#[test] +fn memory_limit() -> Result<()> { + let mut config = Config::new(); + config.with_allocation_strategy(InstanceAllocationStrategy::Pooling { + strategy: PoolingAllocationStrategy::NextAvailable, + module_limits: ModuleLimits { + memory_pages: 3, + table_elements: 10, + ..Default::default() + }, + instance_limits: InstanceLimits { + count: 1, + address_space_size: 196608, + }, + })?; + + let engine = Engine::new(&config); + + // Module should fail to validate because the minimum is greater than the configured limit + match Module::new(&engine, r#"(module (memory 4))"#) { + Ok(_) => panic!("module compilation should fail"), + Err(e) => assert_eq!(e.to_string(), "Validation error: memory index 0 has a minimum page size of 4 which exceeds the limit of 3") + } + + let module = Module::new( + &engine, + r#"(module (memory (export "m") 0) (func (export "f") (result i32) (memory.grow (i32.const 1))))"#, + )?; + + // Instantiate the module and grow the memory via the `f` function + { + let store = Store::new(&engine); + let instance = Instance::new(&store, &module, &[])?; + let f = instance.get_func("f").unwrap().get0::().unwrap(); + + assert_eq!(f().expect("function should not trap"), 0); + assert_eq!(f().expect("function should not trap"), 1); + assert_eq!(f().expect("function should not trap"), 2); + assert_eq!(f().expect("function should not trap"), -1); + assert_eq!(f().expect("function should not trap"), -1); + } + + // Instantiate the module and grow the memory via the Wasmtime API + let store = Store::new(&engine); + let instance = Instance::new(&store, &module, &[])?; + + let memory = instance.get_memory("m").unwrap(); + assert_eq!(memory.size(), 0); + assert_eq!(memory.grow(1).expect("memory should grow"), 0); + assert_eq!(memory.size(), 1); + assert_eq!(memory.grow(1).expect("memory should grow"), 1); + assert_eq!(memory.size(), 2); + assert_eq!(memory.grow(1).expect("memory should grow"), 2); + assert_eq!(memory.size(), 3); + assert!(memory.grow(1).is_err()); + + Ok(()) +} + +#[test] +fn memory_init() -> Result<()> { + let mut config = Config::new(); + config.with_allocation_strategy(InstanceAllocationStrategy::Pooling { + strategy: PoolingAllocationStrategy::NextAvailable, + module_limits: ModuleLimits { + memory_pages: 2, + table_elements: 0, + ..Default::default() + }, + instance_limits: InstanceLimits { + count: 1, + ..Default::default() + }, + })?; + + let engine = Engine::new(&config); + + let module = Module::new( + &engine, + r#"(module (memory (export "m") 2) (data (i32.const 65530) "this data spans multiple pages") (data (i32.const 10) "hello world"))"#, + )?; + + let store = Store::new(&engine); + let instance = Instance::new(&store, &module, &[])?; + let memory = instance.get_memory("m").unwrap(); + + unsafe { + assert_eq!( + &memory.data_unchecked()[65530..65560], + b"this data spans multiple pages" + ); + assert_eq!(&memory.data_unchecked()[10..21], b"hello world"); + } + + Ok(()) +} + +#[test] +fn memory_guard_page_trap() -> Result<()> { + let mut config = Config::new(); + config.with_allocation_strategy(InstanceAllocationStrategy::Pooling { + strategy: PoolingAllocationStrategy::NextAvailable, + module_limits: ModuleLimits { + memory_pages: 2, + table_elements: 0, + ..Default::default() + }, + instance_limits: InstanceLimits { + count: 1, + ..Default::default() + }, + })?; + + let engine = Engine::new(&config); + + let module = Module::new( + &engine, + r#"(module (memory (export "m") 0) (func (export "f") (param i32) local.get 0 i32.load drop))"#, + )?; + + // Instantiate the module and check for out of bounds trap + for _ in 0..10 { + let store = Store::new(&engine); + let instance = Instance::new(&store, &module, &[])?; + let m = instance.get_memory("m").unwrap(); + let f = instance.get_func("f").unwrap().get1::().unwrap(); + + let trap = f(0).expect_err("function should trap"); + assert!(trap.to_string().contains("out of bounds")); + + let trap = f(1).expect_err("function should trap"); + assert!(trap.to_string().contains("out of bounds")); + + m.grow(1).expect("memory should grow"); + f(0).expect("function should not trap"); + + let trap = f(65536).expect_err("function should trap"); + assert!(trap.to_string().contains("out of bounds")); + + let trap = f(65537).expect_err("function should trap"); + assert!(trap.to_string().contains("out of bounds")); + + m.grow(1).expect("memory should grow"); + f(65536).expect("function should not trap"); + + m.grow(1).expect_err("memory should be at the limit"); + } + + Ok(()) +} + +#[test] +#[cfg_attr(target_arch = "aarch64", ignore)] // https://github.com/bytecodealliance/wasmtime/pull/2518#issuecomment-747280133 +fn memory_zeroed() -> Result<()> { + let mut config = Config::new(); + config.with_allocation_strategy(InstanceAllocationStrategy::Pooling { + strategy: PoolingAllocationStrategy::NextAvailable, + module_limits: ModuleLimits { + memory_pages: 1, + table_elements: 0, + ..Default::default() + }, + instance_limits: InstanceLimits { + count: 1, + address_space_size: 1, + }, + })?; + + let engine = Engine::new(&config); + + let module = Module::new(&engine, r#"(module (memory (export "m") 1))"#)?; + + // Instantiate the module repeatedly after writing data to the entire memory + for _ in 0..10 { + let store = Store::new(&engine); + let instance = Instance::new(&store, &module, &[])?; + let memory = instance.get_memory("m").unwrap(); + + assert_eq!(memory.size(), 1); + assert_eq!(memory.data_size(), 65536); + + let ptr = memory.data_ptr(); + + unsafe { + for i in 0..8192 { + assert_eq!(*ptr.cast::().offset(i), 0); + } + std::ptr::write_bytes(ptr, 0xFE, memory.data_size()); + } + } + + Ok(()) +} + +#[test] +fn table_limit() -> Result<()> { + const TABLE_ELEMENTS: u32 = 10; + let mut config = Config::new(); + config.with_allocation_strategy(InstanceAllocationStrategy::Pooling { + strategy: PoolingAllocationStrategy::NextAvailable, + module_limits: ModuleLimits { + memory_pages: 1, + table_elements: TABLE_ELEMENTS, + ..Default::default() + }, + instance_limits: InstanceLimits { + count: 1, + address_space_size: 1, + }, + })?; + + let engine = Engine::new(&config); + + // Module should fail to validate because the minimum is greater than the configured limit + match Module::new(&engine, r#"(module (table 31 funcref))"#) { + Ok(_) => panic!("module compilation should fail"), + Err(e) => assert_eq!(e.to_string(), "Validation error: table index 0 has a minimum element size of 31 which exceeds the limit of 10") + } + + let module = Module::new( + &engine, + r#"(module (table (export "t") 0 funcref) (func (export "f") (result i32) (table.grow (ref.null func) (i32.const 1))))"#, + )?; + + // Instantiate the module and grow the table via the `f` function + { + let store = Store::new(&engine); + let instance = Instance::new(&store, &module, &[])?; + let f = instance.get_func("f").unwrap().get0::().unwrap(); + + for i in 0..TABLE_ELEMENTS { + assert_eq!(f().expect("function should not trap"), i as i32); + } + + assert_eq!(f().expect("function should not trap"), -1); + assert_eq!(f().expect("function should not trap"), -1); + } + + // Instantiate the module and grow the table via the Wasmtime API + let store = Store::new(&engine); + let instance = Instance::new(&store, &module, &[])?; + + let table = instance.get_table("t").unwrap(); + + for i in 0..TABLE_ELEMENTS { + assert_eq!(table.size(), i); + assert_eq!( + table + .grow(1, Val::FuncRef(None)) + .expect("table should grow"), + i + ); + } + + assert_eq!(table.size(), TABLE_ELEMENTS); + assert!(table.grow(1, Val::FuncRef(None)).is_err()); + + Ok(()) +} + +#[test] +fn table_init() -> Result<()> { + let mut config = Config::new(); + config.with_allocation_strategy(InstanceAllocationStrategy::Pooling { + strategy: PoolingAllocationStrategy::NextAvailable, + module_limits: ModuleLimits { + memory_pages: 0, + table_elements: 6, + ..Default::default() + }, + instance_limits: InstanceLimits { + count: 1, + ..Default::default() + }, + })?; + + let engine = Engine::new(&config); + + let module = Module::new( + &engine, + r#"(module (table (export "t") 6 funcref) (elem (i32.const 1) 1 2 3 4) (elem (i32.const 0) 0) (func) (func (param i32)) (func (param i32 i32)) (func (param i32 i32 i32)) (func (param i32 i32 i32 i32)))"#, + )?; + + let store = Store::new(&engine); + let instance = Instance::new(&store, &module, &[])?; + let table = instance.get_table("t").unwrap(); + + for i in 0..5 { + let v = table.get(i).expect("table should have entry"); + let f = v + .funcref() + .expect("expected funcref") + .expect("expected non-null value"); + assert_eq!(f.ty().params().len(), i as usize); + } + + assert!( + table + .get(5) + .expect("table should have entry") + .funcref() + .expect("expected funcref") + .is_none(), + "funcref should be null" + ); + + Ok(()) +} + +#[test] +#[cfg_attr(target_arch = "aarch64", ignore)] // https://github.com/bytecodealliance/wasmtime/pull/2518#issuecomment-747280133 +fn table_zeroed() -> Result<()> { + let mut config = Config::new(); + config.with_allocation_strategy(InstanceAllocationStrategy::Pooling { + strategy: PoolingAllocationStrategy::NextAvailable, + module_limits: ModuleLimits { + memory_pages: 1, + table_elements: 10, + ..Default::default() + }, + instance_limits: InstanceLimits { + count: 1, + address_space_size: 1, + }, + })?; + + let engine = Engine::new(&config); + + let module = Module::new(&engine, r#"(module (table (export "t") 10 funcref))"#)?; + + // Instantiate the module repeatedly after filling table elements + for _ in 0..10 { + let store = Store::new(&engine); + let instance = Instance::new(&store, &module, &[])?; + let table = instance.get_table("t").unwrap(); + let f = Func::wrap(&store, || {}); + + assert_eq!(table.size(), 10); + + for i in 0..10 { + match table.get(i).unwrap() { + Val::FuncRef(r) => assert!(r.is_none()), + _ => panic!("expected a funcref"), + } + table.set(i, Val::FuncRef(Some(f.clone()))).unwrap(); + } + } + + Ok(()) +} + +#[test] +fn instantiation_limit() -> Result<()> { + const INSTANCE_LIMIT: u32 = 10; + let mut config = Config::new(); + config.with_allocation_strategy(InstanceAllocationStrategy::Pooling { + strategy: PoolingAllocationStrategy::NextAvailable, + module_limits: ModuleLimits { + memory_pages: 1, + table_elements: 10, + ..Default::default() + }, + instance_limits: InstanceLimits { + count: INSTANCE_LIMIT, + address_space_size: 1, + }, + })?; + + let engine = Engine::new(&config); + let module = Module::new(&engine, r#"(module)"#)?; + + // Instantiate to the limit + { + let store = Store::new(&engine); + + for _ in 0..INSTANCE_LIMIT { + Instance::new(&store, &module, &[])?; + } + + match Instance::new(&store, &module, &[]) { + Ok(_) => panic!("instantiation should fail"), + Err(e) => assert_eq!( + e.to_string(), + format!( + "Limit of {} concurrent instances has been reached", + INSTANCE_LIMIT + ) + ), + } + } + + // With the above store dropped, ensure instantiations can be made + + let store = Store::new(&engine); + + for _ in 0..INSTANCE_LIMIT { + Instance::new(&store, &module, &[])?; + } + + Ok(()) +} From a2c439117a6595fadc28623466adb922576b3d77 Mon Sep 17 00:00:00 2001 From: Peter Huene Date: Wed, 10 Feb 2021 20:29:20 -0800 Subject: [PATCH 10/33] Implement user fault handling with `userfaultfd` on Linux. This commit implements the `uffd` feature which turns on support for utilizing the `userfaultfd` system call on Linux for the pooling instance allocator. By handling page faults in userland, we are able to detect guard page accesses without having to constantly change memory page protections. This should help reduce the number of syscalls as well as kernel lock contentions when many threads are allocating and deallocating instances. Additionally, the user fault handler can lazy initialize linear memories of an instance (implementation to come). --- .github/workflows/main.yml | 1 + Cargo.toml | 1 + crates/runtime/Cargo.toml | 6 + crates/runtime/src/instance.rs | 43 + crates/runtime/src/instance/allocator.rs | 2 + .../runtime/src/instance/allocator/pooling.rs | 74 +- .../src/instance/allocator/pooling/uffd.rs | 752 ++++++++++++++++++ crates/wasmtime/Cargo.toml | 3 + 8 files changed, 874 insertions(+), 8 deletions(-) create mode 100644 crates/runtime/src/instance/allocator/pooling/uffd.rs diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 02185746bcd6..38cb9584f061 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -122,6 +122,7 @@ jobs: - run: cargo check --manifest-path crates/wasmtime/Cargo.toml --features jitdump - run: cargo check --manifest-path crates/wasmtime/Cargo.toml --features cache - run: cargo check --manifest-path crates/wasmtime/Cargo.toml --features async + - run: cargo check --manifest-path crates/wasmtime/Cargo.toml --features uffd # Check some feature combinations of the `wasmtime-c-api` crate - run: cargo check --manifest-path crates/c-api/Cargo.toml --no-default-features diff --git a/Cargo.toml b/Cargo.toml index a0c5c3f6e2ac..1262115d190d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -89,6 +89,7 @@ jitdump = ["wasmtime/jitdump"] vtune = ["wasmtime/vtune"] wasi-crypto = ["wasmtime-wasi-crypto"] wasi-nn = ["wasmtime-wasi-nn"] +uffd = ["wasmtime/uffd"] # Try the experimental, work-in-progress new x86_64 backend. This is not stable # as of June 2020. diff --git a/crates/runtime/Cargo.toml b/crates/runtime/Cargo.toml index bdcb99755aac..af73a7d1028d 100644 --- a/crates/runtime/Cargo.toml +++ b/crates/runtime/Cargo.toml @@ -37,3 +37,9 @@ cc = "1.0" [badges] maintenance = { status = "actively-developed" } + +[features] +default = [] + +# Enables support for userfaultfd in the pooling allocator when building on Linux +uffd = ["userfaultfd"] diff --git a/crates/runtime/src/instance.rs b/crates/runtime/src/instance.rs index 91a0dd5c46eb..c90c267f6bf2 100644 --- a/crates/runtime/src/instance.rs +++ b/crates/runtime/src/instance.rs @@ -67,6 +67,11 @@ pub(crate) struct Instance { /// Hosts can store arbitrary per-instance information here. host_state: Box, + /// Stores guard page faults in memory relating to the instance. + /// This is used for the pooling allocator with uffd enabled on Linux. + #[cfg(all(feature = "uffd", target_os = "linux"))] + guard_page_faults: RefCell bool)>>, + /// Additional context used by compiled wasm code. This field is last, and /// represents a dynamically-sized array that extends beyond the nominal /// end of the struct (similar to a flexible array member). @@ -376,6 +381,10 @@ impl Instance { /// Returns `None` if memory can't be grown by the specified amount /// of pages. pub(crate) fn memory_grow(&self, memory_index: DefinedMemoryIndex, delta: u32) -> Option { + // Reset all guard pages before growing any memory + #[cfg(all(feature = "uffd", target_os = "linux"))] + self.reset_guard_pages().ok()?; + let result = self .memories .get(memory_index) @@ -803,6 +812,40 @@ impl Instance { (foreign_table_index, foreign_instance) } } + + /// Records a faulted guard page. + /// + /// This is used to track faulted guard pages that need to be reset. + #[cfg(all(feature = "uffd", target_os = "linux"))] + pub(crate) fn record_guard_page_fault( + &self, + page_addr: *mut u8, + size: usize, + reset: unsafe fn(*mut u8, usize) -> bool, + ) { + self.guard_page_faults + .borrow_mut() + .push((page_addr, size, reset)); + } + + /// Resets previously faulted guard pages. + /// + /// This is used to reset the protection of any guard pages that were previously faulted. + /// + /// Resetting the guard pages is required before growing memory. + #[cfg(all(feature = "uffd", target_os = "linux"))] + pub(crate) fn reset_guard_pages(&self) -> Result<(), String> { + let mut faults = self.guard_page_faults.borrow_mut(); + for (addr, len, reset) in faults.drain(..) { + unsafe { + if !reset(addr, len) { + return Err("failed to reset previously faulted memory guard page".into()); + } + } + } + + Ok(()) + } } /// A handle holding an `Instance` of a WebAssembly module. diff --git a/crates/runtime/src/instance/allocator.rs b/crates/runtime/src/instance/allocator.rs index a1d806e4a17e..927afd9e41ab 100644 --- a/crates/runtime/src/instance/allocator.rs +++ b/crates/runtime/src/instance/allocator.rs @@ -525,6 +525,8 @@ unsafe impl InstanceAllocator for OnDemandInstanceAllocator { )), dropped_data: RefCell::new(EntitySet::with_capacity(req.module.passive_data.len())), host_state: req.host_state, + #[cfg(all(feature = "uffd", target_os = "linux"))] + guard_page_faults: RefCell::new(Vec::new()), vmctx: VMContext {}, }; let layout = instance.alloc_layout(); diff --git a/crates/runtime/src/instance/allocator/pooling.rs b/crates/runtime/src/instance/allocator/pooling.rs index 5bf139555a8f..e40198831259 100644 --- a/crates/runtime/src/instance/allocator/pooling.rs +++ b/crates/runtime/src/instance/allocator/pooling.rs @@ -31,6 +31,11 @@ cfg_if::cfg_if! { if #[cfg(windows)] { mod windows; use windows as imp; + } else if #[cfg(all(feature = "uffd", target_os = "linux"))] { + mod uffd; + use uffd as imp; + use imp::{PageFaultHandler, reset_guard_page}; + use std::sync::atomic::{AtomicBool, Ordering}; } else if #[cfg(target_os = "linux")] { mod linux; use linux as imp; @@ -335,6 +340,9 @@ impl Iterator for BasePointerIterator { /// structure depending on the limits used to create the pool. /// /// The pool maintains a free list for fast instance allocation. +/// +/// The userfault handler relies on how instances are stored in the mapping, +/// so make sure the uffd implementation is kept up-to-date. #[derive(Debug)] struct InstancePool { mapping: Mmap, @@ -413,6 +421,8 @@ impl InstancePool { dropped_elements: RefCell::new(EntitySet::new()), dropped_data: RefCell::new(EntitySet::new()), host_state: Box::new(()), + #[cfg(all(feature = "uffd", target_os = "linux"))] + guard_page_faults: RefCell::new(Vec::new()), vmctx: VMContext {}, }, ); @@ -523,6 +533,12 @@ impl InstancePool { ) -> Result<(), InstantiationError> { let module = instance.module.as_ref(); + // Reset all guard pages before reusing the instance + #[cfg(all(feature = "uffd", target_os = "linux"))] + instance + .reset_guard_pages() + .map_err(InstantiationError::Resource)?; + instance.memories.clear(); for plan in @@ -590,6 +606,10 @@ impl Drop for InstancePool { /// /// Each instance index into the pool returns an iterator over the base addresses /// of the instance's linear memories. +/// +/// +/// The userfault handler relies on how memories are stored in the mapping, +/// so make sure the uffd implementation is kept up-to-date. #[derive(Debug)] struct MemoryPool { mapping: Mmap, @@ -646,6 +666,9 @@ impl MemoryPool { /// /// Each instance index into the pool returns an iterator over the base addresses /// of the instance's tables. +/// +/// The userfault handler relies on how tables are stored in the mapping, +/// so make sure the uffd implementation is kept up-to-date. #[derive(Debug)] struct TablePool { mapping: Mmap, @@ -710,6 +733,9 @@ impl TablePool { /// /// The top of the stack (starting stack pointer) is returned when a stack is allocated /// from the pool. +/// +/// The userfault handler relies on how stacks are stored in the mapping, +/// so make sure the uffd implementation is kept up-to-date. #[derive(Debug)] struct StackPool { mapping: Mmap, @@ -717,6 +743,8 @@ struct StackPool { max_instances: usize, page_size: usize, free_list: Mutex>, + #[cfg(all(feature = "uffd", target_os = "linux"))] + faulted_guard_pages: Arc<[AtomicBool]>, } impl StackPool { @@ -745,6 +773,11 @@ impl StackPool { max_instances, page_size, free_list: Mutex::new((0..max_instances).collect()), + #[cfg(all(feature = "uffd", target_os = "linux"))] + faulted_guard_pages: std::iter::repeat_with(|| false.into()) + .take(max_instances) + .collect::>() + .into(), }) } @@ -774,11 +807,25 @@ impl StackPool { .as_mut_ptr() .add((index * self.stack_size) + self.page_size); - // Make the stack accessible (excluding the guard page) - if !make_accessible(bottom_of_stack, size_without_guard) { - return Err(FiberStackError::Resource( - "failed to make instance memory accessible".into(), - )); + cfg_if::cfg_if! { + if #[cfg(all(feature = "uffd", target_os = "linux"))] { + // Check to see if a guard page needs to be reset + if self.faulted_guard_pages[index].swap(false, Ordering::SeqCst) { + if !reset_guard_page(bottom_of_stack.sub(self.page_size), self.page_size) { + return Err(FiberStackError::Resource( + "failed to reset stack guard page".into(), + )); + } + } + + } else { + // Make the stack accessible (excluding the guard page) + if !make_accessible(bottom_of_stack, size_without_guard) { + return Err(FiberStackError::Resource( + "failed to make instance memory accessible".into(), + )); + } + } } // The top of the stack should be returned @@ -824,6 +871,8 @@ pub struct PoolingInstanceAllocator { instance_limits: InstanceLimits, instances: mem::ManuallyDrop, stacks: mem::ManuallyDrop, + #[cfg(all(feature = "uffd", target_os = "linux"))] + _fault_handler: PageFaultHandler, } impl PoolingInstanceAllocator { @@ -866,19 +915,28 @@ impl PoolingInstanceAllocator { )); } + let instances = InstancePool::new(&module_limits, &instance_limits)?; + let stacks = StackPool::new(&instance_limits, stack_size)?; + + #[cfg(all(feature = "uffd", target_os = "linux"))] + let _fault_handler = PageFaultHandler::new(&instances, &stacks)?; + Ok(Self { strategy, module_limits, instance_limits, - instances: mem::ManuallyDrop::new(InstancePool::new(&module_limits, &instance_limits)?), - stacks: mem::ManuallyDrop::new(StackPool::new(&instance_limits, stack_size)?), + instances: mem::ManuallyDrop::new(instances), + stacks: mem::ManuallyDrop::new(stacks), + #[cfg(all(feature = "uffd", target_os = "linux"))] + _fault_handler, }) } } impl Drop for PoolingInstanceAllocator { fn drop(&mut self) { - // There are manually dropped for the future uffd implementation + // Manually drop the pools before the fault handler (if uffd is enabled) + // This ensures that any fault handler thread monitoring the pool memory terminates unsafe { mem::ManuallyDrop::drop(&mut self.instances); mem::ManuallyDrop::drop(&mut self.stacks); diff --git a/crates/runtime/src/instance/allocator/pooling/uffd.rs b/crates/runtime/src/instance/allocator/pooling/uffd.rs new file mode 100644 index 000000000000..ed236a8d4384 --- /dev/null +++ b/crates/runtime/src/instance/allocator/pooling/uffd.rs @@ -0,0 +1,752 @@ +//! Implements user-mode page fault handling with the `userfaultfd` ("uffd") system call on Linux. +//! +//! Handling page faults for memory accesses in regions relating to WebAssembly instances +//! enables the implementation of guard pages in user space rather than kernel space. +//! +//! This reduces the number of system calls and kernel locks needed to provide correct +//! WebAssembly memory semantics. +//! +//! Additionally, linear memories and WebAssembly tables can be lazy-initialized upon access. +//! +//! This feature requires a Linux kernel 4.11 or newer to use. + +use super::{InstancePool, StackPool}; +use crate::{instance::Instance, Mmap}; +use std::convert::TryInto; +use std::ptr; +use std::sync::{ + atomic::{AtomicBool, Ordering}, + Arc, +}; +use std::thread; +use userfaultfd::{Event, FeatureFlags, IoctlFlags, Uffd, UffdBuilder}; +use wasmtime_environ::{wasm::DefinedMemoryIndex, WASM_PAGE_SIZE}; + +pub unsafe fn make_accessible(_addr: *mut u8, _len: usize) -> bool { + // A no-op when userfaultfd is used + true +} + +pub unsafe fn reset_guard_page(addr: *mut u8, len: usize) -> bool { + // Guard pages are READ_WRITE with uffd until faulted + region::protect(addr, len, region::Protection::READ_WRITE).is_ok() +} + +pub unsafe fn decommit(addr: *mut u8, len: usize) { + // Use MADV_DONTNEED to mark the pages as missing + // This will cause a missing page fault for next access on any page in the given range + assert_eq!( + libc::madvise(addr as _, len, libc::MADV_DONTNEED), + 0, + "madvise failed to mark pages as missing: {}", + std::io::Error::last_os_error() + ); +} + +pub fn create_memory_map(_accessible_size: usize, mapping_size: usize) -> Result { + // Allocate a single read-write region at once + // As writable pages need to count towards commit charge, use MAP_NORESERVE to override. + // This implies that the kernel is configured to allow overcommit or else + // this allocation will almost certainly fail without a plethora of physical memory to back the alloction. + // The consequence of not reserving is that our process may segfault on any write to a memory + // page that cannot be backed (i.e. out of memory conditions). + + if mapping_size == 0 { + return Ok(Mmap::new()); + } + + unsafe { + let ptr = libc::mmap( + ptr::null_mut(), + mapping_size, + libc::PROT_READ | libc::PROT_WRITE, + libc::MAP_PRIVATE | libc::MAP_ANON | libc::MAP_NORESERVE, + -1, + 0, + ); + + if ptr as isize == -1_isize { + return Err(format!( + "failed to allocate pool memory: {}", + std::io::Error::last_os_error() + )); + } + + Ok(Mmap::from_raw(ptr as usize, mapping_size)) + } +} + +/// Represents a location of a page fault within monitored regions of memory. +enum AddressLocation<'a> { + /// The address location is in a WebAssembly table page. + /// The fault handler will zero the page as tables are initialized at instantiation-time. + TablePage { + /// The address of the page being accessed. + page_addr: *mut u8, + /// The length of the page being accessed. + len: usize, + }, + /// The address location is in a WebAssembly linear memory page. + /// The fault handler will copy the pages from initialization data if necessary. + MemoryPage { + /// The address of the page being accessed. + page_addr: *mut u8, + /// The length of the page being accessed. + len: usize, + /// The instance related to the memory page that was accessed. + instance: &'a Instance, + /// The index of the memory that was accessed. + memory_index: usize, + /// The Wasm page index to initialize if the access was not a guard page. + page_index: Option, + }, + /// The address location is in an execution stack. + /// The fault handler will zero the page. + StackPage { + /// The address of the page being accessed. + page_addr: *mut u8, + /// The length of the page being accessed. + len: usize, + /// The index of the stack that was accessed. + index: usize, + /// Whether or not the access was to a guard page. + guard_page: bool, + }, +} + +/// Used to resolve fault addresses to address locations. +/// +/// This implementation relies heavily on how the various resource pools utilize their memory. +/// +/// `usize` is used here instead of pointers to keep this `Send` as it gets sent to the handler thread. +struct AddressLocator { + instances_start: usize, + instance_size: usize, + max_instances: usize, + memories_start: usize, + memories_end: usize, + memory_size: usize, + max_memories: usize, + tables_start: usize, + tables_end: usize, + table_size: usize, + stacks_start: usize, + stacks_end: usize, + stack_size: usize, + page_size: usize, +} + +impl AddressLocator { + fn new(instances: &InstancePool, stacks: &StackPool) -> Self { + let instances_start = instances.mapping.as_ptr() as usize; + let memories_start = instances.memories.mapping.as_ptr() as usize; + let memories_end = memories_start + instances.memories.mapping.len(); + let tables_start = instances.tables.mapping.as_ptr() as usize; + let tables_end = tables_start + instances.tables.mapping.len(); + let stacks_start = stacks.mapping.as_ptr() as usize; + let stacks_end = stacks_start + stacks.mapping.len(); + let stack_size = stacks.stack_size; + + // Should always have instances + debug_assert!(instances_start != 0); + + Self { + instances_start, + instance_size: instances.instance_size, + max_instances: instances.max_instances, + memories_start, + memories_end, + memory_size: instances.memories.memory_size, + max_memories: instances.memories.max_memories, + tables_start, + tables_end, + table_size: instances.tables.table_size, + stacks_start, + stacks_end, + stack_size, + page_size: instances.tables.page_size, + } + } + + // This is super-duper unsafe as it is used from the handler thread + // to access instance data without any locking primitives. + /// + /// It is assumed that the thread that owns the instance being accessed is + /// currently suspended waiting on a fault to be handled. + /// + /// Of course a stray faulting memory access from a thread that does not own + /// the instance might introduce a race, but this implementation considers + /// such to be a serious bug. + /// + /// If the assumption holds true, accessing the instance data from the handler thread + /// should, in theory, be safe. + unsafe fn get_instance(&self, index: usize) -> &mut Instance { + debug_assert!(index < self.max_instances); + &mut *((self.instances_start + (index * self.instance_size)) as *mut Instance) + } + + unsafe fn get_location(&self, addr: usize) -> Option { + // Check for a memory location + if addr >= self.memories_start && addr < self.memories_end { + let index = (addr - self.memories_start) / self.memory_size; + let memory_index = index % self.max_memories; + let memory_start = self.memories_start + (index * self.memory_size); + let page_index = (addr - memory_start) / (WASM_PAGE_SIZE as usize); + let instance = self.get_instance(index / self.max_memories); + + let init_page_index = instance + .memories + .get( + DefinedMemoryIndex::from_u32(memory_index as u32) + .try_into() + .unwrap(), + ) + .and_then(|m| { + if page_index < m.size() as usize { + Some(page_index) + } else { + None + } + }); + + return Some(AddressLocation::MemoryPage { + page_addr: (memory_start + page_index * (WASM_PAGE_SIZE as usize)) as _, + len: WASM_PAGE_SIZE as usize, + instance, + memory_index, + page_index: init_page_index, + }); + } + + // Check for a table location + if addr >= self.tables_start && addr < self.tables_end { + let index = (addr - self.tables_start) / self.table_size; + let table_start = self.tables_start + (index * self.table_size); + let table_offset = addr - table_start; + let page_index = table_offset / self.page_size; + + return Some(AddressLocation::TablePage { + page_addr: (table_start + (page_index * self.page_size)) as _, + len: self.page_size, + }); + } + + // Check for a stack location + if addr >= self.stacks_start && addr < self.stacks_end { + let index = (addr - self.stacks_start) / self.stack_size; + let stack_start = self.stacks_start + (index * self.stack_size); + let stack_offset = addr - stack_start; + let page_offset = (stack_offset / self.page_size) * self.page_size; + + return Some(AddressLocation::StackPage { + page_addr: (stack_start + page_offset) as _, + len: self.page_size, + index, + guard_page: stack_offset < self.page_size, + }); + } + + None + } +} + +fn wake_guard_page_access(uffd: &Uffd, page_addr: *const u8, len: usize) -> Result<(), String> { + unsafe { + // Set the page to NONE to induce a SIGSEV for the access on the next retry + region::protect(page_addr, len, region::Protection::NONE) + .map_err(|e| format!("failed to change guard page protection: {}", e))?; + + uffd.wake(page_addr as _, len).map_err(|e| { + format!( + "failed to wake page at {:p} with length {}: {}", + page_addr, len, e + ) + })?; + + Ok(()) + } +} + +fn handler_thread( + uffd: Uffd, + locator: AddressLocator, + mut registrations: usize, + faulted_stack_guard_pages: Arc<[AtomicBool]>, +) -> Result<(), String> { + loop { + match uffd.read_event().expect("failed to read event") { + Some(Event::Unmap { start, end }) => { + log::trace!("memory region unmapped: {:p}-{:p}", start, end); + + let (start, end) = (start as usize, end as usize); + + if (start == locator.memories_start && end == locator.memories_end) + || (start == locator.tables_start && end == locator.tables_end) + || (start == locator.stacks_start && end == locator.stacks_end) + { + registrations -= 1; + if registrations == 0 { + break; + } + } else { + panic!("unexpected memory region unmapped"); + } + } + Some(Event::Pagefault { + addr: access_addr, .. + }) => { + unsafe { + match locator.get_location(access_addr as usize) { + Some(AddressLocation::TablePage { page_addr, len }) => { + log::trace!( + "handling fault in table at address {:p} on page {:p}", + access_addr, + page_addr, + ); + + // Tables are always initialized upon instantiation, so zero the page + uffd.zeropage(page_addr as _, len, true).map_err(|e| { + format!( + "failed to zero page at {:p} with length {}: {}", + page_addr, len, e + ) + })?; + } + Some(AddressLocation::MemoryPage { + page_addr, + len, + instance, + memory_index, + page_index, + }) => { + log::trace!( + "handling fault in linear memory at address {:p} on page {:p}", + access_addr, + page_addr + ); + + match page_index { + Some(page_index) => { + // TODO: copy the memory initialization data rather than zero the page + uffd.zeropage(page_addr as _, len, true).map_err(|e| { + format!( + "failed to zero page at {:p} with length {}: {}", + page_addr, len, e + ) + })?; + } + None => { + log::trace!("out of bounds memory access at {:p}", access_addr); + + // Record the guard page fault with the instance so it can be reset later. + instance.record_guard_page_fault( + page_addr, + len, + reset_guard_page, + ); + wake_guard_page_access(&uffd, page_addr, len)?; + } + } + } + Some(AddressLocation::StackPage { + page_addr, + len, + index, + guard_page, + }) => { + log::trace!( + "handling fault in stack {} at address {:p}", + index, + access_addr, + ); + + if guard_page { + // Logging as trace as stack guard pages might be a trap condition in the future + log::trace!("stack overflow fault at {:p}", access_addr); + + // Mark the stack as having a faulted guard page + // The next time the stack is used the guard page will be reset + faulted_stack_guard_pages[index].store(true, Ordering::SeqCst); + wake_guard_page_access(&uffd, page_addr, len)?; + continue; + } + + // Always zero stack pages + uffd.zeropage(page_addr as _, len, true).map_err(|e| { + format!( + "failed to zero page at {:p} with length {}: {}", + page_addr, len, e + ) + })?; + } + None => { + return Err(format!( + "failed to locate fault address {:p} in registered memory regions", + access_addr + )); + } + } + } + } + Some(_) => continue, + None => break, + } + } + + Ok(()) +} + +#[derive(Debug)] +pub struct PageFaultHandler { + thread: Option>>, +} + +impl PageFaultHandler { + pub(super) fn new(instances: &InstancePool, stacks: &StackPool) -> Result { + let uffd = UffdBuilder::new() + .close_on_exec(true) + .require_features(FeatureFlags::EVENT_UNMAP) + .create() + .map_err(|e| format!("failed to create user fault descriptor: {}", e))?; + + // Register the ranges with the userfault fd + let mut registrations = 0; + for (start, len) in &[ + ( + instances.memories.mapping.as_ptr() as usize, + instances.memories.mapping.len(), + ), + ( + instances.tables.mapping.as_ptr() as usize, + instances.tables.mapping.len(), + ), + (stacks.mapping.as_ptr() as usize, stacks.mapping.len()), + ] { + if *start == 0 || *len == 0 { + continue; + } + + let ioctls = uffd + .register(*start as _, *len) + .map_err(|e| format!("failed to register user fault range: {}", e))?; + + if !ioctls.contains(IoctlFlags::WAKE | IoctlFlags::COPY | IoctlFlags::ZEROPAGE) { + return Err(format!( + "required user fault ioctls not supported; found: {:?}", + ioctls, + )); + } + + registrations += 1; + } + + let thread = if registrations == 0 { + log::trace!("user fault handling disabled as there are no regions to monitor"); + None + } else { + log::trace!( + "user fault handling enabled on {} memory regions", + registrations + ); + + let locator = AddressLocator::new(&instances, &stacks); + + let faulted_stack_guard_pages = stacks.faulted_guard_pages.clone(); + + Some( + thread::Builder::new() + .name("page fault handler".into()) + .spawn(move || { + handler_thread(uffd, locator, registrations, faulted_stack_guard_pages) + }) + .map_err(|e| format!("failed to spawn page fault handler thread: {}", e))?, + ) + }; + + Ok(Self { thread }) + } +} + +impl Drop for PageFaultHandler { + fn drop(&mut self) { + if let Some(thread) = self.thread.take() { + thread + .join() + .expect("failed to join page fault handler thread") + .expect("fault handler thread failed"); + } + } +} + +#[cfg(test)] +mod test { + use super::*; + use crate::{ + table::max_table_element_size, Imports, InstanceAllocationRequest, InstanceLimits, + ModuleLimits, PoolingAllocationStrategy, VMSharedSignatureIndex, + }; + use wasmtime_environ::{ + entity::PrimaryMap, + wasm::{Memory, Table, TableElementType, WasmType}, + MemoryPlan, MemoryStyle, Module, TablePlan, TableStyle, + }; + + #[cfg(target_pointer_width = "64")] + #[test] + fn test_address_locator() { + let module_limits = ModuleLimits { + imported_functions: 0, + imported_tables: 0, + imported_memories: 0, + imported_globals: 0, + types: 0, + functions: 0, + tables: 3, + memories: 2, + globals: 0, + table_elements: 1000, + memory_pages: 2, + }; + let instance_limits = InstanceLimits { + count: 3, + address_space_size: (WASM_PAGE_SIZE * 10) as u64, + }; + + let instances = + InstancePool::new(&module_limits, &instance_limits).expect("should allocate"); + let stacks = StackPool::new(&instance_limits, 8192).expect("should allocate"); + + let locator = AddressLocator::new(&instances, &stacks); + + assert_eq!(locator.instances_start, instances.mapping.as_ptr() as usize); + assert_eq!(locator.instance_size, 4096); + assert_eq!(locator.max_instances, 3); + assert_eq!( + locator.memories_start, + instances.memories.mapping.as_ptr() as usize + ); + assert_eq!( + locator.memories_end, + locator.memories_start + instances.memories.mapping.len() + ); + assert_eq!(locator.memory_size, (WASM_PAGE_SIZE * 10) as usize); + assert_eq!(locator.max_memories, 2); + assert_eq!( + locator.tables_start, + instances.tables.mapping.as_ptr() as usize + ); + assert_eq!( + locator.tables_end, + locator.tables_start + instances.tables.mapping.len() + ); + assert_eq!(locator.table_size, 8192); + + assert_eq!(locator.stacks_start, stacks.mapping.as_ptr() as usize); + assert_eq!( + locator.stacks_end, + locator.stacks_start + stacks.mapping.len() + ); + assert_eq!(locator.stack_size, 12288); + + unsafe { + assert!(locator.get_location(0).is_none()); + assert!(locator + .get_location(std::cmp::max( + locator.memories_end, + std::cmp::max(locator.tables_end, locator.stacks_end) + )) + .is_none()); + + let mut module = Module::new(); + + for _ in 0..module_limits.memories { + module.memory_plans.push(MemoryPlan { + memory: Memory { + minimum: 2, + maximum: Some(2), + shared: false, + }, + style: MemoryStyle::Static { bound: 1 }, + offset_guard_size: 0, + }); + } + + for _ in 0..module_limits.tables { + module.table_plans.push(TablePlan { + table: Table { + wasm_ty: WasmType::FuncRef, + ty: TableElementType::Func, + minimum: 800, + maximum: Some(900), + }, + style: TableStyle::CallerChecksSignature, + }); + } + + module_limits + .validate_module(&module) + .expect("should validate"); + + let mut handles = Vec::new(); + let module = Arc::new(module); + let finished_functions = &PrimaryMap::new(); + + // Allocate the maximum number of instances with the maxmimum number of memories and tables + for _ in 0..instances.max_instances { + handles.push( + instances + .allocate( + PoolingAllocationStrategy::Random, + InstanceAllocationRequest { + module: module.clone(), + finished_functions, + imports: Imports { + functions: &[], + tables: &[], + memories: &[], + globals: &[], + }, + lookup_shared_signature: &|_| VMSharedSignatureIndex::default(), + host_state: Box::new(()), + interrupts: std::ptr::null(), + externref_activations_table: std::ptr::null_mut(), + stack_map_registry: std::ptr::null_mut(), + }, + ) + .expect("instance should allocate"), + ); + } + + // Validate memory locations + for instance_index in 0..instances.max_instances { + for memory_index in 0..instances.memories.max_memories { + let memory_start = locator.memories_start + + (instance_index * locator.memory_size * locator.max_memories) + + (memory_index * locator.memory_size); + + // Test for access to first page + match locator.get_location(memory_start + 10000) { + Some(AddressLocation::MemoryPage { + page_addr, + len, + instance: _, + memory_index: mem_index, + page_index, + }) => { + assert_eq!(page_addr, memory_start as _); + assert_eq!(len, WASM_PAGE_SIZE as usize); + assert_eq!(mem_index, memory_index); + assert_eq!(page_index, Some(0)); + } + _ => panic!("expected a memory page location"), + } + + // Test for access to second page + match locator.get_location(memory_start + 1024 + WASM_PAGE_SIZE as usize) { + Some(AddressLocation::MemoryPage { + page_addr, + len, + instance: _, + memory_index: mem_index, + page_index, + }) => { + assert_eq!(page_addr, (memory_start + WASM_PAGE_SIZE as usize) as _); + assert_eq!(len, WASM_PAGE_SIZE as usize); + assert_eq!(mem_index, memory_index); + assert_eq!(page_index, Some(1)); + } + _ => panic!("expected a memory page location"), + } + + // Test for guard page + match locator.get_location(memory_start + 10 + 9 * WASM_PAGE_SIZE as usize) { + Some(AddressLocation::MemoryPage { + page_addr, + len, + instance: _, + memory_index: mem_index, + page_index, + }) => { + assert_eq!( + page_addr, + (memory_start + (9 * WASM_PAGE_SIZE as usize)) as _ + ); + assert_eq!(len, WASM_PAGE_SIZE as usize); + assert_eq!(mem_index, memory_index); + assert_eq!(page_index, None); + } + _ => panic!("expected a memory page location"), + } + } + } + + // Validate table locations + for instance_index in 0..instances.max_instances { + for table_index in 0..instances.tables.max_tables { + let table_start = locator.tables_start + + (instance_index * locator.table_size * instances.tables.max_tables) + + (table_index * locator.table_size); + + // Check for an access of index 107 (first page) + match locator.get_location(table_start + (107 * max_table_element_size())) { + Some(AddressLocation::TablePage { page_addr, len }) => { + assert_eq!(page_addr, table_start as _); + assert_eq!(len, locator.page_size); + } + _ => panic!("expected a table page location"), + } + + // Check for an access of index 799 (second page) + match locator.get_location(table_start + (799 * max_table_element_size())) { + Some(AddressLocation::TablePage { page_addr, len }) => { + assert_eq!(page_addr, (table_start + locator.page_size) as _); + assert_eq!(len, locator.page_size); + } + _ => panic!("expected a table page location"), + } + } + } + + // Validate stack locations + for stack_index in 0..instances.max_instances { + let stack_start = locator.stacks_start + (stack_index * locator.stack_size); + + // Check for stack page location + match locator.get_location(stack_start + locator.page_size * 2) { + Some(AddressLocation::StackPage { + page_addr, + len, + index, + guard_page, + }) => { + assert_eq!(page_addr, (stack_start + locator.page_size * 2) as _); + assert_eq!(len, locator.page_size); + assert_eq!(index, stack_index); + assert!(!guard_page); + } + _ => panic!("expected a stack page location"), + } + + // Check for guard page + match locator.get_location(stack_start) { + Some(AddressLocation::StackPage { + page_addr, + len, + index, + guard_page, + }) => { + assert_eq!(page_addr, stack_start as _); + assert_eq!(len, locator.page_size); + assert_eq!(index, stack_index); + assert!(guard_page); + } + _ => panic!("expected a stack page location"), + } + } + + for handle in handles.drain(..) { + instances.deallocate(&handle); + } + } + } +} diff --git a/crates/wasmtime/Cargo.toml b/crates/wasmtime/Cargo.toml index 752b3c7ff855..466e3cb61dc8 100644 --- a/crates/wasmtime/Cargo.toml +++ b/crates/wasmtime/Cargo.toml @@ -73,3 +73,6 @@ experimental_x64 = ["wasmtime-jit/experimental_x64"] # Enables support for "async stores" as well as defining host functions as # `async fn` and calling functions asynchronously. async = ["wasmtime-fiber"] + +# Enables userfaultfd support in the runtime's pooling allocator when building on Linux +uffd = ["wasmtime-runtime/uffd"] From 5b2f8789b2a0027df52cdb8e98f8c0915d84fced Mon Sep 17 00:00:00 2001 From: Peter Huene Date: Wed, 10 Feb 2021 23:04:04 -0800 Subject: [PATCH 11/33] Allow zero-sized allocations on Windows for `Mmap`. --- crates/runtime/src/mmap.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/crates/runtime/src/mmap.rs b/crates/runtime/src/mmap.rs index 61ccda9276a1..e60e87e3783a 100644 --- a/crates/runtime/src/mmap.rs +++ b/crates/runtime/src/mmap.rs @@ -124,6 +124,10 @@ impl Mmap { use winapi::um::memoryapi::VirtualAlloc; use winapi::um::winnt::{MEM_COMMIT, MEM_RESERVE, PAGE_NOACCESS, PAGE_READWRITE}; + if mapping_size == 0 { + return Ok(Self::new()); + } + let page_size = region::page::size(); assert_le!(accessible_size, mapping_size); assert_eq!(mapping_size & (page_size - 1), 0); From a82f1a323fc66fb0f7a54f6b91aed9f200abf868 Mon Sep 17 00:00:00 2001 From: Peter Huene Date: Thu, 11 Feb 2021 11:05:16 -0800 Subject: [PATCH 12/33] Skip the stack tests on Windows. As Windows uses the native fiber implementation, the stack tests should be ignored on Windows as the implementation intentionally errors when handing out stacks. --- crates/runtime/src/instance/allocator/pooling.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/runtime/src/instance/allocator/pooling.rs b/crates/runtime/src/instance/allocator/pooling.rs index e40198831259..9351e94ca647 100644 --- a/crates/runtime/src/instance/allocator/pooling.rs +++ b/crates/runtime/src/instance/allocator/pooling.rs @@ -1557,7 +1557,7 @@ mod test { Ok(()) } - #[cfg(target_pointer_width = "64")] + #[cfg(all(unix, target_pointer_width = "64"))] #[test] fn test_stack_pool() -> Result<(), String> { let pool = StackPool::new( @@ -1680,7 +1680,7 @@ mod test { } #[cfg_attr(target_arch = "aarch64", ignore)] // https://github.com/bytecodealliance/wasmtime/pull/2518#issuecomment-747280133 - #[cfg(unix)] + #[cfg(all(unix, target_pointer_width = "64"))] #[test] fn test_stack_zeroed() -> Result<(), String> { let allocator = PoolingInstanceAllocator::new( From f5c4d87c45edd99d0789e56de85c77057b258848 Mon Sep 17 00:00:00 2001 From: Peter Huene Date: Tue, 16 Feb 2021 23:27:14 -0800 Subject: [PATCH 13/33] Implement on-demand memory initialization for the uffd feature. This commit implements copying paged initialization data upon a fault of a linear memory page. If the initialization data is "paged", then the appropriate pages are copied into the Wasm page (or zeroed if the page is not present in the initialization data). If the initialization data is not "paged", the Wasm page is zeroed so that module instantiation can initialize the pages. --- Cargo.lock | 1 + crates/environ/Cargo.toml | 1 + crates/environ/src/module.rs | 191 ++++++++- crates/environ/src/module_environ.rs | 61 +-- crates/jit/Cargo.toml | 2 +- crates/jit/src/instantiate.rs | 45 +- crates/runtime/src/instance/allocator.rs | 394 ++++++++++-------- .../runtime/src/instance/allocator/pooling.rs | 56 ++- .../src/instance/allocator/pooling/uffd.rs | 139 ++++-- crates/wasmtime/src/instance.rs | 6 +- 10 files changed, 562 insertions(+), 334 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3b22f19a5515..73dbdecd48e9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3324,6 +3324,7 @@ dependencies = [ "indexmap", "log", "more-asserts", + "region", "serde", "thiserror", "wasmparser", diff --git a/crates/environ/Cargo.toml b/crates/environ/Cargo.toml index 230295aecbc2..12c39d6ac657 100644 --- a/crates/environ/Cargo.toml +++ b/crates/environ/Cargo.toml @@ -13,6 +13,7 @@ edition = "2018" [dependencies] anyhow = "1.0" +region = "2.2.0" cranelift-codegen = { path = "../../cranelift/codegen", version = "0.71.0", features = ["enable-serde"] } cranelift-entity = { path = "../../cranelift/entity", version = "0.71.0", features = ["enable-serde"] } cranelift-wasm = { path = "../../cranelift/wasm", version = "0.71.0", features = ["enable-serde"] } diff --git a/crates/environ/src/module.rs b/crates/environ/src/module.rs index 8daefaf0790b..79342f354676 100644 --- a/crates/environ/src/module.rs +++ b/crates/environ/src/module.rs @@ -1,7 +1,7 @@ //! Data structures for representing decoded wasm modules. use crate::tunables::Tunables; -use crate::WASM_MAX_PAGES; +use crate::{DataInitializer, WASM_MAX_PAGES, WASM_PAGE_SIZE}; use cranelift_codegen::ir; use cranelift_entity::{EntityRef, PrimaryMap}; use cranelift_wasm::*; @@ -10,19 +10,6 @@ use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::sync::Arc; -/// A WebAssembly table initializer. -#[derive(Clone, Debug, Hash, Serialize, Deserialize)] -pub struct TableElements { - /// The index of a table to initialize. - pub table_index: TableIndex, - /// Optionally, a global variable giving a base index. - pub base: Option, - /// The offset to add to the base. - pub offset: usize, - /// The values to write into the table elements. - pub elements: Box<[FuncIndex]>, -} - /// Implemenation styles for WebAssembly linear memory. #[derive(Debug, Clone, Hash, Serialize, Deserialize)] pub enum MemoryStyle { @@ -92,6 +79,164 @@ impl MemoryPlan { } } +/// A WebAssembly linear memory initializer. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct MemoryInitializer { + /// The index of a linear memory to initialize. + pub memory_index: MemoryIndex, + /// Optionally, a global variable giving a base index. + pub base: Option, + /// The offset to add to the base. + pub offset: usize, + /// The data to write into the linear memory. + pub data: Box<[u8]>, +} + +impl From> for MemoryInitializer { + fn from(initializer: DataInitializer) -> Self { + Self { + memory_index: initializer.memory_index, + base: initializer.base, + offset: initializer.offset, + data: initializer.data.into(), + } + } +} + +/// The type of WebAssembly linear memory initialization. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub enum MemoryInitialization { + /// Memory initialization is paged. + /// + /// To be paged, the following requirements must be met: + /// + /// * All data segments must reference defined memories. + /// * All data segments must not use a global base. + /// * All data segments must be in bounds. + /// + /// Paged initialization is performed by memcopying individual pages to the linear memory. + Paged { + /// The size of each page stored in the map. + /// This is expected to be the host page size. + page_size: usize, + /// The map of defined memory index to a list of page data. + /// The list of page data is sparse, with None representing a zero page. + /// The size of the list will be the maximum page written to by a data segment. + map: PrimaryMap>>>, + }, + /// Memory initialization is out of bounds. + /// + /// To be out of bounds, the following requirements must be met: + /// + /// * All data segments must reference defined memories. + /// * All data segments must not use a global base. + /// * At least one data segments was out of bounds. + /// + /// This can be used to quickly return an error when the module is instantiated. + OutOfBounds, + /// Memory initialization is segmented. + /// + /// To be segmented, at least one of the following requirements must be met: + /// + /// * A data segment referenced an imported memory. + /// * A data segment uses a global base. + /// + /// Segmented initialization is performed by processing the complete set of data segments + /// when the module is instantiated. + /// + /// This ensures that initialization side-effects are observed according to the bulk-memory proposal. + Segmented(Box<[MemoryInitializer]>), +} + +impl MemoryInitialization { + /// Creates a new memory initialization for a module and its data initializers. + pub fn new(module: &Module, initializers: Vec) -> Self { + let page_size = region::page::size(); + let num_defined_memories = module.memory_plans.len() - module.num_imported_memories; + let mut out_of_bounds = false; + let mut memories = PrimaryMap::with_capacity(num_defined_memories); + + for _ in 0..num_defined_memories { + memories.push(Vec::new()); + } + + for initializer in &initializers { + match ( + module.defined_memory_index(initializer.memory_index), + initializer.base.is_some(), + ) { + (None, _) | (_, true) => { + // If the initializer references an imported memory or uses a global base, + // the complete set of segments will need to be processed at module instantiation + return Self::Segmented( + initializers + .into_iter() + .map(Into::into) + .collect::>() + .into_boxed_slice(), + ); + } + (Some(index), false) => { + if out_of_bounds { + continue; + } + + // Perform a bounds check on the segment + if (initializer.offset + initializer.data.len()) + > ((module.memory_plans[initializer.memory_index].memory.minimum as usize) + * (WASM_PAGE_SIZE as usize)) + { + out_of_bounds = true; + continue; + } + + let pages = &mut memories[index]; + let mut page_index = initializer.offset / page_size; + let mut page_offset = initializer.offset % page_size; + let mut data_offset = 0; + let mut data_remaining = initializer.data.len(); + + if data_remaining == 0 { + continue; + } + + // Copy the initialization data by each page + loop { + if page_index >= pages.len() { + pages.resize(page_index + 1, None); + } + + let page = pages[page_index] + .get_or_insert_with(|| vec![0; page_size].into_boxed_slice()); + let len = std::cmp::min(data_remaining, page_size - page_offset); + + page[page_offset..page_offset + len] + .copy_from_slice(&initializer.data[data_offset..(data_offset + len)]); + + if len == data_remaining { + break; + } + + page_index += 1; + page_offset = 0; + data_offset += len; + data_remaining -= len; + } + } + }; + } + + if out_of_bounds { + Self::OutOfBounds + } else { + Self::Paged { + page_size, + map: memories, + } + } + } +} + /// Implemenation styles for WebAssembly tables. #[derive(Debug, Clone, Hash, Serialize, Deserialize)] pub enum TableStyle { @@ -124,6 +269,19 @@ impl TablePlan { } } +/// A WebAssembly table initializer. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct TableInitializer { + /// The index of a table to initialize. + pub table_index: TableIndex, + /// Optionally, a global variable giving a base index. + pub base: Option, + /// The offset to add to the base. + pub offset: usize, + /// The values to write into the table elements. + pub elements: Box<[FuncIndex]>, +} + /// Different types that can appear in a module. /// /// Note that each of these variants are intended to index further into a @@ -164,7 +322,10 @@ pub struct Module { pub start_func: Option, /// WebAssembly table initializers. - pub table_elements: Vec, + pub table_initializers: Vec, + + /// WebAssembly linear memory initializer. + pub memory_initialization: Option, /// WebAssembly passive elements. pub passive_elements: Vec>, diff --git a/crates/environ/src/module_environ.rs b/crates/environ/src/module_environ.rs index 800455358032..636fa2893e7d 100644 --- a/crates/environ/src/module_environ.rs +++ b/crates/environ/src/module_environ.rs @@ -1,6 +1,6 @@ use crate::module::{ Initializer, InstanceSignature, MemoryPlan, Module, ModuleSignature, ModuleType, ModuleUpvar, - TableElements, TablePlan, TypeTables, + TableInitializer, TablePlan, TypeTables, }; use crate::tunables::Tunables; use cranelift_codegen::ir; @@ -13,7 +13,6 @@ use cranelift_wasm::{ ModuleIndex, ModuleTypeIndex, SignatureIndex, Table, TableIndex, TargetEnvironment, TypeIndex, WasmError, WasmFuncType, WasmResult, }; -use serde::{Deserialize, Serialize}; use std::collections::{hash_map::Entry, HashMap}; use std::convert::TryFrom; use std::mem; @@ -684,7 +683,7 @@ impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data fn reserve_table_elements(&mut self, num: u32) -> WasmResult<()> { self.result .module - .table_elements + .table_initializers .reserve_exact(usize::try_from(num).unwrap()); Ok(()) } @@ -696,12 +695,15 @@ impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data offset: usize, elements: Box<[FuncIndex]>, ) -> WasmResult<()> { - self.result.module.table_elements.push(TableElements { - table_index, - base, - offset, - elements, - }); + self.result + .module + .table_initializers + .push(TableInitializer { + table_index, + base, + offset, + elements, + }); Ok(()) } @@ -774,11 +776,9 @@ impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data data: &'data [u8], ) -> WasmResult<()> { self.result.data_initializers.push(DataInitializer { - location: DataInitializerLocation { - memory_index, - base, - offset, - }, + memory_index, + base, + offset, data, }); Ok(()) @@ -1072,10 +1072,8 @@ pub fn translate_signature(mut sig: ir::Signature, pointer_type: ir::Type) -> ir sig } -/// A memory index and offset within that memory where a data initialization -/// should is to be performed. -#[derive(Clone, Serialize, Deserialize)] -pub struct DataInitializerLocation { +/// A data initializer for linear memory. +pub struct DataInitializer<'data> { /// The index of the memory to initialize. pub memory_index: MemoryIndex, @@ -1084,34 +1082,7 @@ pub struct DataInitializerLocation { /// A constant offset to initialize at. pub offset: usize, -} - -/// A data initializer for linear memory. -pub struct DataInitializer<'data> { - /// The location where the initialization is to be performed. - pub location: DataInitializerLocation, /// The initialization data. pub data: &'data [u8], } - -/// Similar to `DataInitializer`, but owns its own copy of the data rather -/// than holding a slice of the original module. -#[derive(Serialize, Deserialize)] -pub struct OwnedDataInitializer { - /// The location where the initialization is to be performed. - pub location: DataInitializerLocation, - - /// The initialization data. - pub data: Box<[u8]>, -} - -impl OwnedDataInitializer { - /// Creates a new owned data initializer from a borrowed data initializer. - pub fn new(borrowed: DataInitializer<'_>) -> Self { - Self { - location: borrowed.location.clone(), - data: borrowed.data.into(), - } - } -} diff --git a/crates/jit/Cargo.toml b/crates/jit/Cargo.toml index 803d88eb438a..b4f6faef74e0 100644 --- a/crates/jit/Cargo.toml +++ b/crates/jit/Cargo.toml @@ -25,7 +25,7 @@ wasmtime-debug = { path = "../debug", version = "0.24.0" } wasmtime-profiling = { path = "../profiling", version = "0.24.0" } wasmtime-obj = { path = "../obj", version = "0.24.0" } rayon = { version = "1.0", optional = true } -region = "2.1.0" +region = "2.2.0" thiserror = "1.0.4" target-lexicon = { version = "0.11.0", default-features = false } wasmparser = "0.76" diff --git a/crates/jit/src/instantiate.rs b/crates/jit/src/instantiate.rs index 4d43f43ba12b..b4875305be09 100644 --- a/crates/jit/src/instantiate.rs +++ b/crates/jit/src/instantiate.rs @@ -21,8 +21,9 @@ use wasmtime_environ::wasm::{ DefinedFuncIndex, InstanceTypeIndex, ModuleTypeIndex, SignatureIndex, WasmFuncType, }; use wasmtime_environ::{ - CompileError, DebugInfoData, FunctionAddressMap, InstanceSignature, Module, ModuleEnvironment, - ModuleSignature, ModuleTranslation, OwnedDataInitializer, StackMapInformation, TrapInformation, + CompileError, DebugInfoData, FunctionAddressMap, InstanceSignature, MemoryInitialization, + Module, ModuleEnvironment, ModuleSignature, ModuleTranslation, StackMapInformation, + TrapInformation, }; use wasmtime_profiling::ProfilingAgent; use wasmtime_runtime::{GdbJitImageRegistration, InstantiationError, VMFunctionBody, VMTrampoline}; @@ -62,10 +63,6 @@ pub struct CompilationArtifacts { /// Unwind information for function code. unwind_info: Box<[ObjectUnwindInfo]>, - /// Data initiailizers. - #[serde(with = "arc_slice_serde")] - data_initializers: Arc<[OwnedDataInitializer]>, - /// Descriptions of compiled functions funcs: PrimaryMap, @@ -122,18 +119,15 @@ impl CompilationArtifacts { } = compiler.compile(&mut translation, &types)?; let ModuleTranslation { - module, + mut module, data_initializers, debuginfo, has_unparsed_debuginfo, .. } = translation; - let data_initializers = data_initializers - .into_iter() - .map(OwnedDataInitializer::new) - .collect::>() - .into(); + module.memory_initialization = + Some(MemoryInitialization::new(&module, data_initializers)); let obj = obj.write().map_err(|_| { SetupError::Instantiate(InstantiationError::Resource( @@ -145,7 +139,6 @@ impl CompilationArtifacts { module: Arc::new(module), obj: obj.into_boxed_slice(), unwind_info: unwind_info.into_boxed_slice(), - data_initializers, funcs: funcs .into_iter() .map(|(_, func)| FunctionInfo { @@ -280,11 +273,6 @@ impl CompiledModule { &self.artifacts } - /// Returns the data initializers from the compiled module. - pub fn data_initializers(&self) -> &Arc<[OwnedDataInitializer]> { - &self.artifacts.data_initializers - } - /// Return a reference-counting pointer to a module. pub fn module(&self) -> &Arc { &self.artifacts.module @@ -546,24 +534,3 @@ mod arc_serde { Ok(Arc::new(T::deserialize(de)?)) } } - -mod arc_slice_serde { - use super::Arc; - use serde::{Deserialize, Deserializer, Serialize, Serializer}; - - pub(super) fn serialize(arc: &Arc<[T]>, ser: S) -> Result - where - S: Serializer, - T: Serialize, - { - (**arc).serialize(ser) - } - - pub(super) fn deserialize<'de, D, T>(de: D) -> Result, D::Error> - where - D: Deserializer<'de>, - T: Deserialize<'de>, - { - Ok(Vec::::deserialize(de)?.into()) - } -} diff --git a/crates/runtime/src/instance/allocator.rs b/crates/runtime/src/instance/allocator.rs index 927afd9e41ab..c45270e17314 100644 --- a/crates/runtime/src/instance/allocator.rs +++ b/crates/runtime/src/instance/allocator.rs @@ -23,7 +23,8 @@ use wasmtime_environ::wasm::{ TableElementType, WasmType, }; use wasmtime_environ::{ - ir, Module, ModuleTranslation, ModuleType, OwnedDataInitializer, TableElements, VMOffsets, + ir, MemoryInitialization, MemoryInitializer, Module, ModuleTranslation, ModuleType, + TableInitializer, VMOffsets, }; mod pooling; @@ -139,7 +140,6 @@ pub unsafe trait InstanceAllocator: Send + Sync { &self, handle: &InstanceHandle, is_bulk_memory: bool, - data_initializers: &Arc<[OwnedDataInitializer]>, ) -> Result<(), InstantiationError>; /// Deallocates a previously allocated instance. @@ -169,6 +169,228 @@ pub unsafe trait InstanceAllocator: Send + Sync { unsafe fn deallocate_fiber_stack(&self, stack: *mut u8); } +fn get_table_init_start(init: &TableInitializer, instance: &Instance) -> usize { + let mut start = init.offset; + + if let Some(base) = init.base { + let val = unsafe { + if let Some(def_index) = instance.module.defined_global_index(base) { + *instance.global(def_index).as_u32() + } else { + *(*instance.imported_global(base).from).as_u32() + } + }; + start += usize::try_from(val).unwrap(); + } + + start +} + +fn check_table_init_bounds(instance: &Instance) -> Result<(), InstantiationError> { + for init in &instance.module.table_initializers { + let start = get_table_init_start(init, instance); + let table = instance.get_table(init.table_index); + + let size = usize::try_from(table.size()).unwrap(); + if size < start + init.elements.len() { + return Err(InstantiationError::Link(LinkError( + "table out of bounds: elements segment does not fit".to_owned(), + ))); + } + } + + Ok(()) +} + +fn initialize_tables(instance: &Instance) -> Result<(), InstantiationError> { + for init in &instance.module.table_initializers { + let start = get_table_init_start(init, instance); + let table = instance.get_table(init.table_index); + + if start + .checked_add(init.elements.len()) + .map_or(true, |end| end > table.size() as usize) + { + return Err(InstantiationError::Trap(Trap::wasm( + ir::TrapCode::TableOutOfBounds, + ))); + } + + for (i, func_idx) in init.elements.iter().enumerate() { + let item = match table.element_type() { + TableElementType::Func => instance + .get_caller_checked_anyfunc(*func_idx) + .map_or(ptr::null_mut(), |f: &VMCallerCheckedAnyfunc| { + f as *const VMCallerCheckedAnyfunc as *mut VMCallerCheckedAnyfunc + }) + .into(), + TableElementType::Val(_) => { + assert!(*func_idx == FuncIndex::reserved_value()); + TableElement::ExternRef(None) + } + }; + table.set(u32::try_from(start + i).unwrap(), item).unwrap(); + } + } + + Ok(()) +} + +fn get_memory_init_start(init: &MemoryInitializer, instance: &Instance) -> usize { + let mut start = init.offset; + + if let Some(base) = init.base { + let val = unsafe { + if let Some(def_index) = instance.module.defined_global_index(base) { + *instance.global(def_index).as_u32() + } else { + *(*instance.imported_global(base).from).as_u32() + } + }; + start += usize::try_from(val).unwrap(); + } + + start +} + +unsafe fn get_memory_slice<'instance>( + init: &MemoryInitializer, + instance: &'instance Instance, +) -> &'instance mut [u8] { + let memory = if let Some(defined_memory_index) = + instance.module.defined_memory_index(init.memory_index) + { + instance.memory(defined_memory_index) + } else { + let import = instance.imported_memory(init.memory_index); + let foreign_instance = (&mut *(import).vmctx).instance(); + let foreign_memory = &mut *(import).from; + let foreign_index = foreign_instance.memory_index(foreign_memory); + foreign_instance.memory(foreign_index) + }; + slice::from_raw_parts_mut(memory.base, memory.current_length) +} + +fn check_memory_init_bounds( + instance: &Instance, + initializers: &[MemoryInitializer], +) -> Result<(), InstantiationError> { + for init in initializers { + let start = get_memory_init_start(init, instance); + unsafe { + let mem_slice = get_memory_slice(init, instance); + if mem_slice.get_mut(start..start + init.data.len()).is_none() { + return Err(InstantiationError::Link(LinkError( + "memory out of bounds: data segment does not fit".into(), + ))); + } + } + } + + Ok(()) +} + +fn initialize_memories( + instance: &Instance, + initializers: &[MemoryInitializer], +) -> Result<(), InstantiationError> { + for init in initializers { + let memory = instance.get_memory(init.memory_index); + + let start = get_memory_init_start(init, instance); + if start + .checked_add(init.data.len()) + .map_or(true, |end| end > memory.current_length) + { + return Err(InstantiationError::Trap(Trap::wasm( + ir::TrapCode::HeapOutOfBounds, + ))); + } + + unsafe { + let mem_slice = get_memory_slice(init, instance); + let end = start + init.data.len(); + let to_init = &mut mem_slice[start..end]; + to_init.copy_from_slice(&init.data); + } + } + + Ok(()) +} + +fn check_init_bounds(instance: &Instance) -> Result<(), InstantiationError> { + check_table_init_bounds(instance)?; + + match &instance.module.memory_initialization { + Some(MemoryInitialization::Paged { .. }) | None => { + // Bounds were checked at compile-time + } + Some(MemoryInitialization::OutOfBounds) => { + return Err(InstantiationError::Link(LinkError( + "memory out of bounds: data segment does not fit".into(), + ))); + } + Some(MemoryInitialization::Segmented(initializers)) => { + check_memory_init_bounds(instance, initializers)?; + } + } + + Ok(()) +} + +fn initialize_instance( + instance: &Instance, + is_bulk_memory: bool, +) -> Result<(), InstantiationError> { + // If bulk memory is not enabled, bounds check the data and element segments before + // making any changes. With bulk memory enabled, initializers are processed + // in-order and side effects are observed up to the point of an out-of-bounds + // initializer, so the early checking is not desired. + if !is_bulk_memory { + check_init_bounds(instance)?; + } + + // Initialize the tables + initialize_tables(instance)?; + + // Initialize the memories + match &instance.module.memory_initialization { + Some(MemoryInitialization::Paged { page_size, map }) => { + for (index, pages) in map { + let memory = instance.memory(index); + + for (page_index, page) in pages.iter().enumerate() { + if let Some(data) = page { + // Bounds checking should have occurred when the module was compiled + // The data should always be page sized + assert!((page_index * page_size) < memory.current_length); + assert_eq!(data.len(), *page_size); + + unsafe { + ptr::copy_nonoverlapping( + data.as_ptr(), + memory.base.add(page_index * page_size), + data.len(), + ); + } + } + } + } + } + Some(MemoryInitialization::OutOfBounds) => { + return Err(InstantiationError::Trap(Trap::wasm( + ir::TrapCode::HeapOutOfBounds, + ))) + } + Some(MemoryInitialization::Segmented(initializers)) => { + initialize_memories(instance, initializers)?; + } + None => {} + } + + Ok(()) +} + unsafe fn initialize_vmcontext( instance: &Instance, functions: &[VMFunctionImport], @@ -350,157 +572,6 @@ impl OnDemandInstanceAllocator { } Ok(memories) } - - fn check_table_init_bounds(instance: &Instance) -> Result<(), InstantiationError> { - for init in &instance.module.table_elements { - let start = Self::get_table_init_start(init, instance); - let table = instance.get_table(init.table_index); - - let size = usize::try_from(table.size()).unwrap(); - if size < start + init.elements.len() { - return Err(InstantiationError::Link(LinkError( - "table out of bounds: elements segment does not fit".to_owned(), - ))); - } - } - - Ok(()) - } - - fn get_memory_init_start(init: &OwnedDataInitializer, instance: &Instance) -> usize { - let mut start = init.location.offset; - - if let Some(base) = init.location.base { - let val = unsafe { - if let Some(def_index) = instance.module.defined_global_index(base) { - *instance.global(def_index).as_u32() - } else { - *(*instance.imported_global(base).from).as_u32() - } - }; - start += usize::try_from(val).unwrap(); - } - - start - } - - unsafe fn get_memory_slice<'instance>( - init: &OwnedDataInitializer, - instance: &'instance Instance, - ) -> &'instance mut [u8] { - let memory = if let Some(defined_memory_index) = instance - .module - .defined_memory_index(init.location.memory_index) - { - instance.memory(defined_memory_index) - } else { - let import = instance.imported_memory(init.location.memory_index); - let foreign_instance = (&mut *(import).vmctx).instance(); - let foreign_memory = &mut *(import).from; - let foreign_index = foreign_instance.memory_index(foreign_memory); - foreign_instance.memory(foreign_index) - }; - slice::from_raw_parts_mut(memory.base, memory.current_length) - } - - fn check_memory_init_bounds( - instance: &Instance, - data_initializers: &[OwnedDataInitializer], - ) -> Result<(), InstantiationError> { - for init in data_initializers { - let start = Self::get_memory_init_start(init, instance); - unsafe { - let mem_slice = Self::get_memory_slice(init, instance); - if mem_slice.get_mut(start..start + init.data.len()).is_none() { - return Err(InstantiationError::Link(LinkError( - "memory out of bounds: data segment does not fit".into(), - ))); - } - } - } - - Ok(()) - } - - fn get_table_init_start(init: &TableElements, instance: &Instance) -> usize { - let mut start = init.offset; - - if let Some(base) = init.base { - let val = unsafe { - if let Some(def_index) = instance.module.defined_global_index(base) { - *instance.global(def_index).as_u32() - } else { - *(*instance.imported_global(base).from).as_u32() - } - }; - start += usize::try_from(val).unwrap(); - } - - start - } - - fn initialize_tables(instance: &Instance) -> Result<(), InstantiationError> { - for init in &instance.module.table_elements { - let start = Self::get_table_init_start(init, instance); - let table = instance.get_table(init.table_index); - - if start - .checked_add(init.elements.len()) - .map_or(true, |end| end > table.size() as usize) - { - return Err(InstantiationError::Trap(Trap::wasm( - ir::TrapCode::TableOutOfBounds, - ))); - } - - for (i, func_idx) in init.elements.iter().enumerate() { - let item = match table.element_type() { - TableElementType::Func => instance - .get_caller_checked_anyfunc(*func_idx) - .map_or(ptr::null_mut(), |f: &VMCallerCheckedAnyfunc| { - f as *const VMCallerCheckedAnyfunc as *mut VMCallerCheckedAnyfunc - }) - .into(), - TableElementType::Val(_) => { - assert!(*func_idx == FuncIndex::reserved_value()); - TableElement::ExternRef(None) - } - }; - table.set(u32::try_from(start + i).unwrap(), item).unwrap(); - } - } - - Ok(()) - } - - /// Initialize the table memory from the provided initializers. - fn initialize_memories( - instance: &Instance, - data_initializers: &[OwnedDataInitializer], - ) -> Result<(), InstantiationError> { - for init in data_initializers { - let memory = instance.get_memory(init.location.memory_index); - - let start = Self::get_memory_init_start(init, instance); - if start - .checked_add(init.data.len()) - .map_or(true, |end| end > memory.current_length) - { - return Err(InstantiationError::Trap(Trap::wasm( - ir::TrapCode::HeapOutOfBounds, - ))); - } - - unsafe { - let mem_slice = Self::get_memory_slice(init, instance); - let end = start + init.data.len(); - let to_init = &mut mem_slice[start..end]; - to_init.copy_from_slice(&init.data); - } - } - - Ok(()) - } } unsafe impl InstanceAllocator for OnDemandInstanceAllocator { @@ -561,23 +632,8 @@ unsafe impl InstanceAllocator for OnDemandInstanceAllocator { &self, handle: &InstanceHandle, is_bulk_memory: bool, - data_initializers: &Arc<[OwnedDataInitializer]>, ) -> Result<(), InstantiationError> { - // Check initializer bounds before initializing anything. Only do this - // when bulk memory is disabled, since the bulk memory proposal changes - // instantiation such that the intermediate results of failed - // initializations are visible. - if !is_bulk_memory { - Self::check_table_init_bounds(handle.instance())?; - Self::check_memory_init_bounds(handle.instance(), data_initializers.as_ref())?; - } - - // Apply fallible initializers. Note that this can "leak" state even if - // it fails. - Self::initialize_tables(handle.instance())?; - Self::initialize_memories(handle.instance(), data_initializers.as_ref())?; - - Ok(()) + initialize_instance(handle.instance(), is_bulk_memory) } unsafe fn deallocate(&self, handle: &InstanceHandle) { diff --git a/crates/runtime/src/instance/allocator/pooling.rs b/crates/runtime/src/instance/allocator/pooling.rs index 9351e94ca647..dfcbd7bb0a8c 100644 --- a/crates/runtime/src/instance/allocator/pooling.rs +++ b/crates/runtime/src/instance/allocator/pooling.rs @@ -8,13 +8,10 @@ //! when modules can be constrained based on configurable limits. use super::{ - initialize_vmcontext, FiberStackError, InstanceAllocationRequest, InstanceAllocator, - InstanceHandle, InstantiationError, -}; -use crate::{ - instance::Instance, table::max_table_element_size, Memory, Mmap, OnDemandInstanceAllocator, - Table, VMContext, + initialize_instance, initialize_vmcontext, FiberStackError, InstanceAllocationRequest, + InstanceAllocator, InstanceHandle, InstantiationError, }; +use crate::{instance::Instance, table::max_table_element_size, Memory, Mmap, Table, VMContext}; use rand::Rng; use std::cell::RefCell; use std::cmp::min; @@ -23,8 +20,7 @@ use std::mem; use std::sync::{Arc, Mutex}; use wasmtime_environ::{ entity::{EntitySet, PrimaryMap}, - MemoryStyle, Module, ModuleTranslation, OwnedDataInitializer, Tunables, VMOffsets, - WASM_PAGE_SIZE, + MemoryStyle, Module, ModuleTranslation, Tunables, VMOffsets, WASM_PAGE_SIZE, }; cfg_if::cfg_if! { @@ -35,6 +31,8 @@ cfg_if::cfg_if! { mod uffd; use uffd as imp; use imp::{PageFaultHandler, reset_guard_page}; + use super::{check_init_bounds, initialize_tables}; + use wasmtime_environ::MemoryInitialization; use std::sync::atomic::{AtomicBool, Ordering}; } else if #[cfg(target_os = "linux")] { mod linux; @@ -979,31 +977,29 @@ unsafe impl InstanceAllocator for PoolingInstanceAllocator { &self, handle: &InstanceHandle, is_bulk_memory: bool, - data_initializers: &Arc<[OwnedDataInitializer]>, ) -> Result<(), InstantiationError> { - // TODO: refactor this implementation - - // Check initializer bounds before initializing anything. Only do this - // when bulk memory is disabled, since the bulk memory proposal changes - // instantiation such that the intermediate results of failed - // initializations are visible. - if !is_bulk_memory { - OnDemandInstanceAllocator::check_table_init_bounds(handle.instance())?; - OnDemandInstanceAllocator::check_memory_init_bounds( - handle.instance(), - data_initializers.as_ref(), - )?; - } + let instance = handle.instance(); + + cfg_if::cfg_if! { + if #[cfg(all(feature = "uffd", target_os = "linux"))] { + match instance.module.memory_initialization { + Some(MemoryInitialization::Paged{ .. }) => { + if !is_bulk_memory { + check_init_bounds(instance)?; + } - // Apply fallible initializers. Note that this can "leak" state even if - // it fails. - OnDemandInstanceAllocator::initialize_tables(handle.instance())?; - OnDemandInstanceAllocator::initialize_memories( - handle.instance(), - data_initializers.as_ref(), - )?; + // Initialize the tables + initialize_tables(instance)?; - Ok(()) + // Don't initialize the memory; the fault handler will fill the pages when accessed + Ok(()) + }, + _ => initialize_instance(instance, is_bulk_memory) + } + } else { + initialize_instance(instance, is_bulk_memory) + } + } } unsafe fn deallocate(&self, handle: &InstanceHandle) { diff --git a/crates/runtime/src/instance/allocator/pooling/uffd.rs b/crates/runtime/src/instance/allocator/pooling/uffd.rs index ed236a8d4384..d18db383ef2d 100644 --- a/crates/runtime/src/instance/allocator/pooling/uffd.rs +++ b/crates/runtime/src/instance/allocator/pooling/uffd.rs @@ -20,7 +20,9 @@ use std::sync::{ }; use std::thread; use userfaultfd::{Event, FeatureFlags, IoctlFlags, Uffd, UffdBuilder}; -use wasmtime_environ::{wasm::DefinedMemoryIndex, WASM_PAGE_SIZE}; +use wasmtime_environ::{entity::EntityRef, wasm::DefinedMemoryIndex, MemoryInitialization}; + +const WASM_PAGE_SIZE: usize = wasmtime_environ::WASM_PAGE_SIZE as usize; pub unsafe fn make_accessible(_addr: *mut u8, _len: usize) -> bool { // A no-op when userfaultfd is used @@ -191,7 +193,7 @@ impl AddressLocator { let index = (addr - self.memories_start) / self.memory_size; let memory_index = index % self.max_memories; let memory_start = self.memories_start + (index * self.memory_size); - let page_index = (addr - memory_start) / (WASM_PAGE_SIZE as usize); + let page_index = (addr - memory_start) / WASM_PAGE_SIZE; let instance = self.get_instance(index / self.max_memories); let init_page_index = instance @@ -210,8 +212,8 @@ impl AddressLocator { }); return Some(AddressLocation::MemoryPage { - page_addr: (memory_start + page_index * (WASM_PAGE_SIZE as usize)) as _, - len: WASM_PAGE_SIZE as usize, + page_addr: (memory_start + page_index * WASM_PAGE_SIZE) as _, + len: WASM_PAGE_SIZE, instance, memory_index, page_index: init_page_index, @@ -250,18 +252,98 @@ impl AddressLocator { } } -fn wake_guard_page_access(uffd: &Uffd, page_addr: *const u8, len: usize) -> Result<(), String> { - unsafe { - // Set the page to NONE to induce a SIGSEV for the access on the next retry - region::protect(page_addr, len, region::Protection::NONE) - .map_err(|e| format!("failed to change guard page protection: {}", e))?; +unsafe fn wake_guard_page_access( + uffd: &Uffd, + page_addr: *const u8, + len: usize, +) -> Result<(), String> { + // Set the page to NONE to induce a SIGSEV for the access on the next retry + region::protect(page_addr, len, region::Protection::NONE) + .map_err(|e| format!("failed to change guard page protection: {}", e))?; + + uffd.wake(page_addr as _, len).map_err(|e| { + format!( + "failed to wake page at {:p} with length {}: {}", + page_addr, len, e + ) + })?; + + Ok(()) +} + +unsafe fn initialize_wasm_page( + uffd: &Uffd, + instance: &Instance, + page_addr: *const u8, + memory_index: usize, + page_index: usize, +) -> Result<(), String> { + if let Some(MemoryInitialization::Paged { page_size, map }) = + &instance.module.memory_initialization + { + let memory_index = DefinedMemoryIndex::new(memory_index); + let memory = instance.memory(memory_index); + let pages = &map[memory_index]; + debug_assert_eq!(WASM_PAGE_SIZE % page_size, 0); + + let count = WASM_PAGE_SIZE / page_size; + let start = page_index * count; + + for i in start..start + count { + let dst = memory.base.add(i * page_size); + + match pages.get(i) { + Some(Some(data)) => { + log::trace!( + "copying page initialization data from {:p} to {:p} with length {}", + data, + dst, + page_size + ); + + // Copy the page data without waking + uffd.copy(data.as_ptr() as _, dst as _, *page_size, false) + .map_err(|e| { + format!( + "failed to copy page from {:p} to {:p} with length {}: {}", + data, dst, page_size, e + ) + })?; + } + _ => { + log::trace!("zeroing page at {:p} with length {}", dst, page_size); + + // No data, zero the page without waking + uffd.zeropage(dst as _, *page_size, false).map_err(|e| { + format!( + "failed to zero page at {:p} with length {}: {}", + dst, page_size, e + ) + })?; + } + } + } - uffd.wake(page_addr as _, len).map_err(|e| { + // Finally wake the entire wasm page + uffd.wake(page_addr as _, WASM_PAGE_SIZE).map_err(|e| { format!( "failed to wake page at {:p} with length {}: {}", - page_addr, len, e + page_addr, WASM_PAGE_SIZE, e ) - })?; + }) + } else { + log::trace!( + "initialization data is not paged; zeroing Wasm page at {:p}", + page_addr + ); + + uffd.zeropage(page_addr as _, WASM_PAGE_SIZE, true) + .map_err(|e| { + format!( + "failed to zero page at {:p} with length {}: {}", + page_addr, WASM_PAGE_SIZE, e + ) + })?; Ok(()) } @@ -327,13 +409,13 @@ fn handler_thread( match page_index { Some(page_index) => { - // TODO: copy the memory initialization data rather than zero the page - uffd.zeropage(page_addr as _, len, true).map_err(|e| { - format!( - "failed to zero page at {:p} with length {}: {}", - page_addr, len, e - ) - })?; + initialize_wasm_page( + &uffd, + instance, + page_addr, + memory_index, + page_index, + )?; } None => { log::trace!("out of bounds memory access at {:p}", access_addr); @@ -529,7 +611,7 @@ mod test { locator.memories_end, locator.memories_start + instances.memories.mapping.len() ); - assert_eq!(locator.memory_size, (WASM_PAGE_SIZE * 10) as usize); + assert_eq!(locator.memory_size, WASM_PAGE_SIZE * 10); assert_eq!(locator.max_memories, 2); assert_eq!( locator.tables_start, @@ -634,7 +716,7 @@ mod test { page_index, }) => { assert_eq!(page_addr, memory_start as _); - assert_eq!(len, WASM_PAGE_SIZE as usize); + assert_eq!(len, WASM_PAGE_SIZE); assert_eq!(mem_index, memory_index); assert_eq!(page_index, Some(0)); } @@ -642,7 +724,7 @@ mod test { } // Test for access to second page - match locator.get_location(memory_start + 1024 + WASM_PAGE_SIZE as usize) { + match locator.get_location(memory_start + 1024 + WASM_PAGE_SIZE) { Some(AddressLocation::MemoryPage { page_addr, len, @@ -650,8 +732,8 @@ mod test { memory_index: mem_index, page_index, }) => { - assert_eq!(page_addr, (memory_start + WASM_PAGE_SIZE as usize) as _); - assert_eq!(len, WASM_PAGE_SIZE as usize); + assert_eq!(page_addr, (memory_start + WASM_PAGE_SIZE) as _); + assert_eq!(len, WASM_PAGE_SIZE); assert_eq!(mem_index, memory_index); assert_eq!(page_index, Some(1)); } @@ -659,7 +741,7 @@ mod test { } // Test for guard page - match locator.get_location(memory_start + 10 + 9 * WASM_PAGE_SIZE as usize) { + match locator.get_location(memory_start + 10 + 9 * WASM_PAGE_SIZE) { Some(AddressLocation::MemoryPage { page_addr, len, @@ -667,11 +749,8 @@ mod test { memory_index: mem_index, page_index, }) => { - assert_eq!( - page_addr, - (memory_start + (9 * WASM_PAGE_SIZE as usize)) as _ - ); - assert_eq!(len, WASM_PAGE_SIZE as usize); + assert_eq!(page_addr, (memory_start + (9 * WASM_PAGE_SIZE)) as _); + assert_eq!(len, WASM_PAGE_SIZE); assert_eq!(mem_index, memory_index); assert_eq!(page_index, None); } diff --git a/crates/wasmtime/src/instance.rs b/crates/wasmtime/src/instance.rs index 7048d9333813..9e2992c9b8b5 100644 --- a/crates/wasmtime/src/instance.rs +++ b/crates/wasmtime/src/instance.rs @@ -522,11 +522,7 @@ impl<'a> Instantiator<'a> { // initialization is successful, we need to keep the instance alive. let instance = self.store.add_instance(instance, false); allocator - .initialize( - &instance.handle, - config.features.bulk_memory, - &compiled_module.data_initializers(), - ) + .initialize(&instance.handle, config.features.bulk_memory) .map_err(|e| -> Error { match e { InstantiationError::Trap(trap) => { From 9091f13dcdf13b92c6df2429c378ad3e81ad7597 Mon Sep 17 00:00:00 2001 From: Peter Huene Date: Wed, 17 Feb 2021 18:22:36 -0800 Subject: [PATCH 14/33] Refactor `initialize_vmcontext`. This was originally written to support sourcing the table and memory definitions differently for the pooling allocator. However, both allocators do the exact same thing, so the closure arguments are no longer necessary. Additionally, this cleans up the code a bit to pass in the allocation request rather than having individual parameters. --- crates/runtime/src/instance/allocator.rs | 84 +++++++------------ .../runtime/src/instance/allocator/pooling.rs | 26 ++---- 2 files changed, 40 insertions(+), 70 deletions(-) diff --git a/crates/runtime/src/instance/allocator.rs b/crates/runtime/src/instance/allocator.rs index c45270e17314..aae17c7f4d91 100644 --- a/crates/runtime/src/instance/allocator.rs +++ b/crates/runtime/src/instance/allocator.rs @@ -6,8 +6,8 @@ use crate::table::{Table, TableElement}; use crate::traphandlers::Trap; use crate::vmcontext::{ VMBuiltinFunctionsArray, VMCallerCheckedAnyfunc, VMContext, VMFunctionBody, VMFunctionImport, - VMGlobalDefinition, VMGlobalImport, VMInterrupts, VMMemoryDefinition, VMMemoryImport, - VMSharedSignatureIndex, VMTableDefinition, VMTableImport, + VMGlobalDefinition, VMGlobalImport, VMInterrupts, VMMemoryImport, VMSharedSignatureIndex, + VMTableImport, }; use std::alloc; use std::any::Any; @@ -391,31 +391,18 @@ fn initialize_instance( Ok(()) } -unsafe fn initialize_vmcontext( - instance: &Instance, - functions: &[VMFunctionImport], - tables: &[VMTableImport], - memories: &[VMMemoryImport], - globals: &[VMGlobalImport], - finished_functions: &PrimaryMap, - lookup_shared_signature: &dyn Fn(SignatureIndex) -> VMSharedSignatureIndex, - interrupts: *const VMInterrupts, - externref_activations_table: *mut VMExternRefActivationsTable, - stack_map_registry: *mut StackMapRegistry, - get_mem_def: impl Fn(DefinedMemoryIndex) -> VMMemoryDefinition, - get_table_def: impl Fn(DefinedTableIndex) -> VMTableDefinition, -) { +unsafe fn initialize_vmcontext(instance: &Instance, req: InstanceAllocationRequest) { let module = &instance.module; - *instance.interrupts() = interrupts; - *instance.externref_activations_table() = externref_activations_table; - *instance.stack_map_registry() = stack_map_registry; + *instance.interrupts() = req.interrupts; + *instance.externref_activations_table() = req.externref_activations_table; + *instance.stack_map_registry() = req.stack_map_registry; // Initialize shared signatures let mut ptr = instance.signature_ids_ptr(); for sig in module.types.values() { *ptr = match sig { - ModuleType::Function(sig) => lookup_shared_signature(*sig), + ModuleType::Function(sig) => (req.lookup_shared_signature)(*sig), _ => VMSharedSignatureIndex::new(u32::max_value()), }; ptr = ptr.add(1); @@ -428,38 +415,38 @@ unsafe fn initialize_vmcontext( ); // Initialize the imports - debug_assert_eq!(functions.len(), module.num_imported_funcs); + debug_assert_eq!(req.imports.functions.len(), module.num_imported_funcs); ptr::copy( - functions.as_ptr(), + req.imports.functions.as_ptr(), instance.imported_functions_ptr() as *mut VMFunctionImport, - functions.len(), + req.imports.functions.len(), ); - debug_assert_eq!(tables.len(), module.num_imported_tables); + debug_assert_eq!(req.imports.tables.len(), module.num_imported_tables); ptr::copy( - tables.as_ptr(), + req.imports.tables.as_ptr(), instance.imported_tables_ptr() as *mut VMTableImport, - tables.len(), + req.imports.tables.len(), ); - debug_assert_eq!(memories.len(), module.num_imported_memories); + debug_assert_eq!(req.imports.memories.len(), module.num_imported_memories); ptr::copy( - memories.as_ptr(), + req.imports.memories.as_ptr(), instance.imported_memories_ptr() as *mut VMMemoryImport, - memories.len(), + req.imports.memories.len(), ); - debug_assert_eq!(globals.len(), module.num_imported_globals); + debug_assert_eq!(req.imports.globals.len(), module.num_imported_globals); ptr::copy( - globals.as_ptr(), + req.imports.globals.as_ptr(), instance.imported_globals_ptr() as *mut VMGlobalImport, - globals.len(), + req.imports.globals.len(), ); // Initialize the functions for (index, sig) in instance.module.functions.iter() { - let type_index = lookup_shared_signature(*sig); + let type_index = (req.lookup_shared_signature)(*sig); let (func_ptr, vmctx) = if let Some(def_index) = instance.module.defined_func_index(index) { ( - NonNull::new(finished_functions[def_index] as *mut _).unwrap(), + NonNull::new(req.finished_functions[def_index] as *mut _).unwrap(), instance.vmctx_ptr(), ) } else { @@ -480,14 +467,17 @@ unsafe fn initialize_vmcontext( // Initialize the defined tables let mut ptr = instance.tables_ptr(); for i in 0..module.table_plans.len() - module.num_imported_tables { - ptr::write(ptr, get_table_def(DefinedTableIndex::new(i))); + ptr::write(ptr, instance.tables[DefinedTableIndex::new(i)].vmtable()); ptr = ptr.add(1); } // Initialize the defined memories let mut ptr = instance.memories_ptr(); for i in 0..module.memory_plans.len() - module.num_imported_memories { - ptr::write(ptr, get_mem_def(DefinedMemoryIndex::new(i))); + ptr::write( + ptr, + instance.memories[DefinedMemoryIndex::new(i)].vmmemory(), + ); ptr = ptr.add(1); } @@ -577,7 +567,7 @@ impl OnDemandInstanceAllocator { unsafe impl InstanceAllocator for OnDemandInstanceAllocator { unsafe fn allocate( &self, - req: InstanceAllocationRequest, + mut req: InstanceAllocationRequest, ) -> Result { debug_assert!(!req.externref_activations_table.is_null()); debug_assert!(!req.stack_map_registry.is_null()); @@ -585,6 +575,8 @@ unsafe impl InstanceAllocator for OnDemandInstanceAllocator { let memories = self.create_memories(&req.module)?; let tables = Self::create_tables(&req.module); + let host_state = std::mem::replace(&mut req.host_state, Box::new(())); + let handle = { let instance = Instance { module: req.module.clone(), @@ -595,7 +587,7 @@ unsafe impl InstanceAllocator for OnDemandInstanceAllocator { req.module.passive_elements.len(), )), dropped_data: RefCell::new(EntitySet::with_capacity(req.module.passive_data.len())), - host_state: req.host_state, + host_state, #[cfg(all(feature = "uffd", target_os = "linux"))] guard_page_faults: RefCell::new(Vec::new()), vmctx: VMContext {}, @@ -609,21 +601,7 @@ unsafe impl InstanceAllocator for OnDemandInstanceAllocator { InstanceHandle::new(instance_ptr) }; - let instance = handle.instance(); - initialize_vmcontext( - instance, - req.imports.functions, - req.imports.tables, - req.imports.memories, - req.imports.globals, - req.finished_functions, - req.lookup_shared_signature, - req.interrupts, - req.externref_activations_table, - req.stack_map_registry, - &|index| instance.memories[index].vmmemory(), - &|index| instance.tables[index].vmtable(), - ); + initialize_vmcontext(handle.instance(), req); Ok(handle) } diff --git a/crates/runtime/src/instance/allocator/pooling.rs b/crates/runtime/src/instance/allocator/pooling.rs index dfcbd7bb0a8c..394b231c62f4 100644 --- a/crates/runtime/src/instance/allocator/pooling.rs +++ b/crates/runtime/src/instance/allocator/pooling.rs @@ -430,7 +430,7 @@ impl InstancePool { fn allocate( &self, strategy: PoolingAllocationStrategy, - req: InstanceAllocationRequest, + mut req: InstanceAllocationRequest, ) -> Result { let index = { let mut free_list = self.free_list.lock().unwrap(); @@ -441,17 +441,19 @@ impl InstancePool { free_list.swap_remove(free_index) }; + let host_state = std::mem::replace(&mut req.host_state, Box::new(())); + unsafe { debug_assert!(index < self.max_instances); let instance = &mut *(self.mapping.as_mut_ptr().add(index * self.instance_size) as *mut Instance); - instance.module = req.module; + instance.module = req.module.clone(); instance.offsets = VMOffsets::new( std::mem::size_of::<*const u8>() as u8, instance.module.as_ref(), ); - instance.host_state = req.host_state; + instance.host_state = host_state; Self::set_instance_memories( instance, @@ -460,20 +462,7 @@ impl InstancePool { )?; Self::set_instance_tables(instance, self.tables.get(index), self.tables.max_elements)?; - initialize_vmcontext( - instance, - req.imports.functions, - req.imports.tables, - req.imports.memories, - req.imports.globals, - req.finished_functions, - req.lookup_shared_signature, - req.interrupts, - req.externref_activations_table, - req.stack_map_registry, - &|index| instance.memories[index].vmmemory(), - &|index| instance.tables[index].vmtable(), - ); + initialize_vmcontext(instance, req); Ok(InstanceHandle::new(instance as _)) } @@ -517,6 +506,9 @@ impl InstancePool { decommit(base, size); } } + + // Drop the host state + (*handle.instance).host_state = Box::new(()); } { From f170d0b32881b94093498fad32a39cd12b381dd3 Mon Sep 17 00:00:00 2001 From: Peter Huene Date: Wed, 17 Feb 2021 22:15:42 -0800 Subject: [PATCH 15/33] Test the uffd feature on Linux. --- .github/workflows/main.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 38cb9584f061..33577c08e78d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -304,6 +304,14 @@ jobs: env: RUST_BACKTRACE: 1 + # Test uffd functionality on Linux + - run: | + cargo test --features uffd -p wasmtime-runtime instance::allocator::pooling + cargo test --features uffd -p wasmtime-cli pooling_allocator + if: matrix.os == 'ubuntu-latest' + env: + RUST_BACKTRACE: 1 + # Build and test lightbeam. Note that # Lightbeam tests fail right now, but we don't want to block on that. - run: cargo build --package lightbeam From f48d1e2be4b2dcdbc7f872b3cb7457049fffd457 Mon Sep 17 00:00:00 2001 From: Peter Huene Date: Thu, 18 Feb 2021 10:23:57 -0800 Subject: [PATCH 16/33] Use slice::fill for filling tables. Now that `slice::fill` is stable, update the table implementation in the runtime to use it. --- crates/runtime/src/table.rs | 36 ++++++++++++------------------------ 1 file changed, 12 insertions(+), 24 deletions(-) diff --git a/crates/runtime/src/table.rs b/crates/runtime/src/table.rs index f8281c9cfa06..0469c4ff3981 100644 --- a/crates/runtime/src/table.rs +++ b/crates/runtime/src/table.rs @@ -180,30 +180,18 @@ impl Table { } match val { - TableElement::FuncRef(r) => { - unsafe { - self.with_funcrefs_mut(move |elements| { - let elements = elements.unwrap(); - - // TODO: replace this with slice::fill (https://github.com/rust-lang/rust/issues/70758) when stabilized - for e in &mut elements[start as usize..end as usize] { - *e = r; - } - }); - } - } - TableElement::ExternRef(r) => { - unsafe { - self.with_externrefs_mut(move |elements| { - let elements = elements.unwrap(); - - // TODO: replace this with slice::fill (https://github.com/rust-lang/rust/issues/70758) when stabilized - for e in &mut elements[start as usize..end as usize] { - *e = r.clone(); - } - }); - } - } + TableElement::FuncRef(r) => unsafe { + self.with_funcrefs_mut(move |elements| { + let elements = elements.unwrap(); + elements[start as usize..end as usize].fill(r); + }); + }, + TableElement::ExternRef(r) => unsafe { + self.with_externrefs_mut(move |elements| { + let elements = elements.unwrap(); + elements[start as usize..end as usize].fill(r); + }); + }, } Ok(()) From f533df029cacd36333c445257b788c3a1a4011c4 Mon Sep 17 00:00:00 2001 From: Peter Huene Date: Thu, 18 Feb 2021 10:25:07 -0800 Subject: [PATCH 17/33] Update the rustc badge to better reflect the supported version. Wasmtime documentation says stable is the supported rustc version, and that's what we test with CI, so the badge should reflect that. Wasmtime doesn't even build with 1.37 any longer anyway. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 53c268baa631..2095e08ba1e2 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@

build status zulip chat - min rustc + supported rustc stable Documentation Status

From 89d3b5d25c08df72d1bf3976feec497779c5581e Mon Sep 17 00:00:00 2001 From: Peter Huene Date: Thu, 18 Feb 2021 11:07:38 -0800 Subject: [PATCH 18/33] Switch CI back to latest nightly. The issue that required the pin to the older version has been resolved. This keeps the x64 backend tests on an older nightly version to support the `-Z` flags being passed without having to update Cargo.toml to a new feature resolver version. The doc task is also kept on the older nightly for the same reason. --- .github/workflows/main.yml | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 33577c08e78d..8b69ccfec2a1 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -73,14 +73,11 @@ jobs: - uses: actions/checkout@v2 with: submodules: true - # Note that we use nightly Rust here to get intra-doc links which are a - # nightly-only feature right now. + # Note that we use nightly Rust for the doc_cfg feature (enabled via `nightlydoc` above) + # This version is an older nightly for the new x64 backend (see below) - uses: ./.github/actions/install-rust with: - # TODO (rust-lang/rust#79661): We are seeing an internal compiler error when - # building with the latest (2020-12-06) nightly; pin on a slightly older - # version for now. - toolchain: nightly-2020-11-29 + toolchain: nightly-2020-12-26 - run: cargo doc --no-deps --all --exclude wasmtime-cli --exclude test-programs --exclude cranelift-codegen-meta - run: cargo doc --package cranelift-codegen-meta --document-private-items - uses: actions/upload-artifact@v1 @@ -168,7 +165,7 @@ jobs: # flags to rustc. - uses: ./.github/actions/install-rust with: - toolchain: nightly-2020-11-29 + toolchain: nightly - run: cargo install cargo-fuzz --vers "^0.8" - run: cargo fetch working-directory: ./fuzz @@ -225,7 +222,7 @@ jobs: rust: beta - build: nightly os: ubuntu-latest - rust: nightly-2020-11-29 + rust: nightly - build: macos os: macos-latest rust: stable @@ -321,8 +318,10 @@ jobs: RUST_BACKTRACE: 1 # Perform all tests (debug mode) for `wasmtime` with the experimental x64 - # backend. This runs on the nightly channel of Rust (because of issues with - # unifying Cargo features on stable) on Ubuntu. + # backend. This runs on an older nightly of Rust (because of issues with + # unifying Cargo features on stable) on Ubuntu such that it's new enough + # to build Wasmtime, but old enough where the -Z options being used + # haven't been stabilized yet. test_x64: name: Test x64 new backend runs-on: ubuntu-latest @@ -332,7 +331,7 @@ jobs: submodules: true - uses: ./.github/actions/install-rust with: - toolchain: nightly-2020-11-29 + toolchain: nightly-2020-12-26 - uses: ./.github/actions/define-llvm-env # Install wasm32 targets in order to build various tests throughout the @@ -343,7 +342,7 @@ jobs: # Run the x64 CI script. - run: ./ci/run-experimental-x64-ci.sh env: - CARGO_VERSION: "+nightly-2020-11-29" + CARGO_VERSION: "+nightly-2020-12-26" RUST_BACKTRACE: 1 # Build and test the wasi-nn module. @@ -356,7 +355,7 @@ jobs: submodules: true - uses: ./.github/actions/install-rust with: - toolchain: nightly-2020-11-29 + toolchain: nightly - run: rustup target add wasm32-wasi - uses: ./.github/actions/install-openvino - run: ./ci/run-wasi-nn-example.sh From a481e11e630c2680578742f8c3c498d8a1389f27 Mon Sep 17 00:00:00 2001 From: Peter Huene Date: Fri, 26 Feb 2021 12:33:54 -0800 Subject: [PATCH 19/33] Add the `uffd` feature to the wasmtime crate docs. --- crates/wasmtime/src/lib.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/crates/wasmtime/src/lib.rs b/crates/wasmtime/src/lib.rs index 12dd72b456f4..8b8a74c50860 100644 --- a/crates/wasmtime/src/lib.rs +++ b/crates/wasmtime/src/lib.rs @@ -172,6 +172,12 @@ //! * `vtune` - Not enabled by default, this feature compiles in support for //! supporting VTune profiling of JIT code. //! +//! * `uffd` - Not enabled by default. This feature enables `userfaultfd` support +//! when using the pooling instance allocator. As handling page faults in userspace +//! comes with a performance penalty, this feature should only be enabled when kernel +//! lock contention is hampering multithreading throughput. This feature is only +//! supported on Linux and requires a Linux kernel version 4.11 or higher. +//! //! ## Examples //! //! In addition to the examples below be sure to check out the [online embedding From 505437e35325912e096fd4fb1bb663e9e6dabc01 Mon Sep 17 00:00:00 2001 From: Peter Huene Date: Fri, 26 Feb 2021 18:41:33 -0800 Subject: [PATCH 20/33] Code cleanup. Last minute code clean up to fix some comments and rename `address_space_size` to `memory_reservation_size` to better describe what the option is doing. --- crates/environ/src/module.rs | 2 +- .../runtime/src/instance/allocator/pooling.rs | 72 ++++++++++--------- .../src/instance/allocator/pooling/uffd.rs | 20 +++--- crates/wasmtime/src/lib.rs | 2 +- tests/all/async_functions.rs | 2 +- tests/all/pooling_allocator.rs | 12 ++-- 6 files changed, 57 insertions(+), 53 deletions(-) diff --git a/crates/environ/src/module.rs b/crates/environ/src/module.rs index 79342f354676..b70748ec723c 100644 --- a/crates/environ/src/module.rs +++ b/crates/environ/src/module.rs @@ -33,7 +33,7 @@ impl MemoryStyle { let maximum = std::cmp::min( memory.maximum.unwrap_or(WASM_MAX_PAGES), if tunables.static_memory_bound_is_maximum { - tunables.static_memory_bound + std::cmp::min(tunables.static_memory_bound, WASM_MAX_PAGES) } else { WASM_MAX_PAGES }, diff --git a/crates/runtime/src/instance/allocator/pooling.rs b/crates/runtime/src/instance/allocator/pooling.rs index 394b231c62f4..c759780ad856 100644 --- a/crates/runtime/src/instance/allocator/pooling.rs +++ b/crates/runtime/src/instance/allocator/pooling.rs @@ -60,7 +60,7 @@ pub struct ModuleLimits { /// The maximum number of imported tables for a module (default is 0). pub imported_tables: u32, - /// The maximum number of imported memories for a module (default is 0). + /// The maximum number of imported linear memories for a module (default is 0). pub imported_memories: u32, /// The maximum number of imported globals for a module (default is 0). @@ -75,7 +75,7 @@ pub struct ModuleLimits { /// The maximum number of defined tables for a module (default is 1). pub tables: u32, - /// The maximum number of defined memories for a module (default is 1). + /// The maximum number of defined linear memories for a module (default is 1). pub memories: u32, /// The maximum number of defined globals for a module (default is 10). @@ -90,7 +90,7 @@ pub struct ModuleLimits { /// the maximum will be `table_elements` for the purpose of any `table.grow` instruction. pub table_elements: u32, - /// The maximum number of pages for any memory defined in a module (default is 160). + /// The maximum number of pages for any linear memory defined in a module (default is 160). /// /// The default of 160 means at most 10 MiB of host memory may be committed for each instance. /// @@ -100,7 +100,7 @@ pub struct ModuleLimits { /// If a memory's maximum page limit is unbounded or greater than this value, /// the maximum will be `memory_pages` for the purpose of any `memory.grow` instruction. /// - /// This value cannot exceed any address space limits placed on instances. + /// This value cannot exceed any memory reservation size limits placed on instances. pub memory_pages: u32, } @@ -234,14 +234,14 @@ pub struct InstanceLimits { /// The maximum number of concurrent instances supported (default is 1000). pub count: u32, - /// The maximum reserved host address space size to use for each instance in bytes. + /// The maximum size, in bytes, of host address space to reserve for each linear memory of an instance. /// /// Note: this value has important performance ramifications. /// /// On 64-bit platforms, the default for this value will be 6 GiB. A value of less than 4 GiB will /// force runtime bounds checking for memory accesses and thus will negatively impact performance. /// Any value above 4 GiB will start eliding bounds checks provided the `offset` of the memory access is - /// less than (`address_space_size` - 4 GiB). A value of 8 GiB will completely elide *all* bounds + /// less than (`memory_reservation_size` - 4 GiB). A value of 8 GiB will completely elide *all* bounds /// checks; consequently, 8 GiB will be the maximum supported value. The default of 6 GiB reserves /// less host address space for each instance, but a memory access with an offet above 2 GiB will incur /// runtime bounds checks. @@ -251,7 +251,7 @@ pub struct InstanceLimits { /// for all memory accesses. For better runtime performance, a 64-bit host is recommended. /// /// This value will be rounded up by the WebAssembly page size (64 KiB). - pub address_space_size: u64, + pub memory_reservation_size: u64, } impl Default for InstanceLimits { @@ -260,9 +260,9 @@ impl Default for InstanceLimits { Self { count: 1000, #[cfg(target_pointer_width = "32")] - address_space_size: 0xA00000, + memory_reservation_size: 0xA00000, #[cfg(target_pointer_width = "64")] - address_space_size: 0x180000000, + memory_reservation_size: 0x180000000, } } } @@ -611,8 +611,8 @@ struct MemoryPool { impl MemoryPool { fn new(module_limits: &ModuleLimits, instance_limits: &InstanceLimits) -> Result { - let memory_size = usize::try_from(instance_limits.address_space_size) - .map_err(|_| "address space size exceeds addressable memory".to_string())?; + let memory_size = usize::try_from(instance_limits.memory_reservation_size) + .map_err(|_| "memory reservation size exceeds addressable memory".to_string())?; debug_assert!( memory_size % region::page::size() == 0, @@ -627,7 +627,7 @@ impl MemoryPool { .checked_mul(max_memories) .and_then(|c| c.checked_mul(max_instances)) .ok_or_else(|| { - "total size of instance address space exceeds addressable memory".to_string() + "total size of memory reservation exceeds addressable memory".to_string() })?; Ok(Self { @@ -877,15 +877,16 @@ impl PoolingInstanceAllocator { return Err("the instance count limit cannot be zero".into()); } - // Round the instance address space size to the nearest Wasm page size - instance_limits.address_space_size = u64::try_from(round_up_to_pow2( - usize::try_from(instance_limits.address_space_size).unwrap(), + // Round the memory reservation size to the nearest Wasm page size + instance_limits.memory_reservation_size = u64::try_from(round_up_to_pow2( + usize::try_from(instance_limits.memory_reservation_size).unwrap(), WASM_PAGE_SIZE as usize, )) .unwrap(); - // Cap the instance address space size to 8 GiB (maximum 4 GiB address space + 4 GiB of guard region) - instance_limits.address_space_size = min(instance_limits.address_space_size, 0x200000000); + // Cap the memory reservation size to 8 GiB (maximum 4 GiB accessible + 4 GiB of guard region) + instance_limits.memory_reservation_size = + min(instance_limits.memory_reservation_size, 0x200000000); // The maximum module memory page count cannot exceed 65536 pages if module_limits.memory_pages > 0x10000 { @@ -895,13 +896,14 @@ impl PoolingInstanceAllocator { )); } - // The maximum module memory page count cannot exceed the instance address space size - if (module_limits.memory_pages * WASM_PAGE_SIZE) as u64 > instance_limits.address_space_size + // The maximum module memory page count cannot exceed the memory reservation size + if (module_limits.memory_pages * WASM_PAGE_SIZE) as u64 + > instance_limits.memory_reservation_size { return Err(format!( - "module memory page limit of {} pages exeeds the instance address space size limit of {} bytes", + "module memory page limit of {} pages exeeds the memory reservation size limit of {} bytes", module_limits.memory_pages, - instance_limits.address_space_size + instance_limits.memory_reservation_size )); } @@ -940,15 +942,15 @@ unsafe impl InstanceAllocator for PoolingInstanceAllocator { } fn adjust_tunables(&self, tunables: &mut Tunables) { - let address_space_size = self.instance_limits.address_space_size; + let memory_reservation_size = self.instance_limits.memory_reservation_size; - // For address spaces larger than 4 GiB, use a guard region to elide - if address_space_size >= 0x100000000 { + // For reservation sizes larger than 4 GiB, use a guard region to elide bounds checks + if memory_reservation_size >= 0x100000000 { tunables.static_memory_bound = 0x10000; // in Wasm pages - tunables.static_memory_offset_guard_size = address_space_size - 0x100000000; + tunables.static_memory_offset_guard_size = memory_reservation_size - 0x100000000; } else { tunables.static_memory_bound = - u32::try_from(address_space_size).unwrap() / WASM_PAGE_SIZE; + u32::try_from(memory_reservation_size).unwrap() / WASM_PAGE_SIZE; tunables.static_memory_offset_guard_size = 0; } @@ -1349,7 +1351,7 @@ mod test { }; let instance_limits = InstanceLimits { count: 3, - address_space_size: 4096, + memory_reservation_size: 4096, }; let instances = InstancePool::new(&module_limits, &instance_limits)?; @@ -1471,7 +1473,7 @@ mod test { }, &InstanceLimits { count: 5, - address_space_size: WASM_PAGE_SIZE as u64, + memory_reservation_size: WASM_PAGE_SIZE as u64, }, )?; @@ -1517,7 +1519,7 @@ mod test { }, &InstanceLimits { count: 7, - address_space_size: WASM_PAGE_SIZE as u64, + memory_reservation_size: WASM_PAGE_SIZE as u64, }, )?; @@ -1551,7 +1553,7 @@ mod test { let pool = StackPool::new( &InstanceLimits { count: 10, - address_space_size: 0, + memory_reservation_size: 0, }, 1, )?; @@ -1638,7 +1640,7 @@ mod test { }, InstanceLimits { count: 1, - address_space_size: 1, + memory_reservation_size: 1, }, 4096 ) @@ -1648,7 +1650,7 @@ mod test { } #[test] - fn test_pooling_allocator_with_address_space_exeeded() { + fn test_pooling_allocator_with_reservation_size_exeeded() { assert_eq!( PoolingInstanceAllocator::new( PoolingAllocationStrategy::Random, @@ -1658,12 +1660,12 @@ mod test { }, InstanceLimits { count: 1, - address_space_size: 1, + memory_reservation_size: 1, }, 4096, ) .expect_err("expected a failure constructing instance allocator"), - "module memory page limit of 2 pages exeeds the instance address space size limit of 65536 bytes" + "module memory page limit of 2 pages exeeds the memory reservation size limit of 65536 bytes" ); } @@ -1686,7 +1688,7 @@ mod test { }, InstanceLimits { count: 1, - address_space_size: 1, + memory_reservation_size: 1, }, 4096, )?; diff --git a/crates/runtime/src/instance/allocator/pooling/uffd.rs b/crates/runtime/src/instance/allocator/pooling/uffd.rs index d18db383ef2d..793b63522fa4 100644 --- a/crates/runtime/src/instance/allocator/pooling/uffd.rs +++ b/crates/runtime/src/instance/allocator/pooling/uffd.rs @@ -1,12 +1,12 @@ -//! Implements user-mode page fault handling with the `userfaultfd` ("uffd") system call on Linux. +//! Implements user space page fault handling with the `userfaultfd` ("uffd") system call on Linux. //! //! Handling page faults for memory accesses in regions relating to WebAssembly instances -//! enables the implementation of guard pages in user space rather than kernel space. +//! enables the implementation of protecting guard pages in user space rather than kernel space. //! //! This reduces the number of system calls and kernel locks needed to provide correct //! WebAssembly memory semantics. //! -//! Additionally, linear memories and WebAssembly tables can be lazy-initialized upon access. +//! Additionally, linear memories can be lazy-initialized upon access. //! //! This feature requires a Linux kernel 4.11 or newer to use. @@ -49,7 +49,7 @@ pub fn create_memory_map(_accessible_size: usize, mapping_size: usize) -> Result // Allocate a single read-write region at once // As writable pages need to count towards commit charge, use MAP_NORESERVE to override. // This implies that the kernel is configured to allow overcommit or else - // this allocation will almost certainly fail without a plethora of physical memory to back the alloction. + // this allocation will almost certainly fail without a plethora of physical memory to back the allocation. // The consequence of not reserving is that our process may segfault on any write to a memory // page that cannot be backed (i.e. out of memory conditions). @@ -170,8 +170,8 @@ impl AddressLocator { } } - // This is super-duper unsafe as it is used from the handler thread - // to access instance data without any locking primitives. + /// This is super-duper unsafe as it is used from the handler thread + /// to access instance data without any locking primitives. /// /// It is assumed that the thread that owns the instance being accessed is /// currently suspended waiting on a fault to be handled. @@ -182,9 +182,9 @@ impl AddressLocator { /// /// If the assumption holds true, accessing the instance data from the handler thread /// should, in theory, be safe. - unsafe fn get_instance(&self, index: usize) -> &mut Instance { + unsafe fn get_instance(&self, index: usize) -> &Instance { debug_assert!(index < self.max_instances); - &mut *((self.instances_start + (index * self.instance_size)) as *mut Instance) + &*((self.instances_start + (index * self.instance_size)) as *const Instance) } unsafe fn get_location(&self, addr: usize) -> Option { @@ -475,6 +475,8 @@ fn handler_thread( } } + log::trace!("fault handler thread has successfully terminated"); + Ok(()) } @@ -591,7 +593,7 @@ mod test { }; let instance_limits = InstanceLimits { count: 3, - address_space_size: (WASM_PAGE_SIZE * 10) as u64, + memory_reservation_size: (WASM_PAGE_SIZE * 10) as u64, }; let instances = diff --git a/crates/wasmtime/src/lib.rs b/crates/wasmtime/src/lib.rs index 8b8a74c50860..f0116362c9c1 100644 --- a/crates/wasmtime/src/lib.rs +++ b/crates/wasmtime/src/lib.rs @@ -173,7 +173,7 @@ //! supporting VTune profiling of JIT code. //! //! * `uffd` - Not enabled by default. This feature enables `userfaultfd` support -//! when using the pooling instance allocator. As handling page faults in userspace +//! when using the pooling instance allocator. As handling page faults in user space //! comes with a performance penalty, this feature should only be enabled when kernel //! lock contention is hampering multithreading throughput. This feature is only //! supported on Linux and requires a Linux kernel version 4.11 or higher. diff --git a/tests/all/async_functions.rs b/tests/all/async_functions.rs index eb3ea0961199..6f90657bf455 100644 --- a/tests/all/async_functions.rs +++ b/tests/all/async_functions.rs @@ -378,7 +378,7 @@ fn async_with_pooling_stacks() { }, instance_limits: InstanceLimits { count: 1, - address_space_size: 1, + memory_reservation_size: 1, }, }) .expect("pooling allocator created"); diff --git a/tests/all/pooling_allocator.rs b/tests/all/pooling_allocator.rs index 846dac647e4b..d7b7101fee49 100644 --- a/tests/all/pooling_allocator.rs +++ b/tests/all/pooling_allocator.rs @@ -13,7 +13,7 @@ fn successful_instantiation() -> Result<()> { }, instance_limits: InstanceLimits { count: 1, - address_space_size: 1, + memory_reservation_size: 1, }, })?; @@ -39,7 +39,7 @@ fn memory_limit() -> Result<()> { }, instance_limits: InstanceLimits { count: 1, - address_space_size: 196608, + memory_reservation_size: 196608, }, })?; @@ -191,7 +191,7 @@ fn memory_zeroed() -> Result<()> { }, instance_limits: InstanceLimits { count: 1, - address_space_size: 1, + memory_reservation_size: 1, }, })?; @@ -234,7 +234,7 @@ fn table_limit() -> Result<()> { }, instance_limits: InstanceLimits { count: 1, - address_space_size: 1, + memory_reservation_size: 1, }, })?; @@ -349,7 +349,7 @@ fn table_zeroed() -> Result<()> { }, instance_limits: InstanceLimits { count: 1, - address_space_size: 1, + memory_reservation_size: 1, }, })?; @@ -391,7 +391,7 @@ fn instantiation_limit() -> Result<()> { }, instance_limits: InstanceLimits { count: INSTANCE_LIMIT, - address_space_size: 1, + memory_reservation_size: 1, }, })?; From 4e83392070025b850e41343bff070c6c59909c07 Mon Sep 17 00:00:00 2001 From: Peter Huene Date: Mon, 1 Mar 2021 10:39:59 -0800 Subject: [PATCH 21/33] Fix bad merge. Fix a bad merge with the `async` feature that accidentally removed the allocation of fiber stacks via the instance allocator. --- crates/wasmtime/src/store.rs | 66 ++++++++++++++++++++++++++++-------- 1 file changed, 52 insertions(+), 14 deletions(-) diff --git a/crates/wasmtime/src/store.rs b/crates/wasmtime/src/store.rs index 9445f852a2c4..ae84a6203b17 100644 --- a/crates/wasmtime/src/store.rs +++ b/crates/wasmtime/src/store.rs @@ -754,12 +754,14 @@ impl Store { /// that the various comments are illuminating as to what's going on here. #[cfg(feature = "async")] pub(crate) async fn on_fiber(&self, func: impl FnOnce() -> R) -> Result { + let config = self.inner.engine.config(); + debug_assert!(self.is_async()); + debug_assert!(config.async_stack_size > 0); - // TODO: allocation of a fiber should be much more abstract where we - // shouldn't be allocating huge stacks on every async wasm function call. + type SuspendType = wasmtime_fiber::Suspend, (), Result<(), Trap>>; let mut slot = None; - let fiber = wasmtime_fiber::Fiber::new(10 * 1024 * 1024, |keep_going, suspend| { + let func = |keep_going, suspend: &SuspendType| { // First check and see if we were interrupted/dropped, and only // continue if we haven't been. keep_going?; @@ -777,18 +779,46 @@ impl Store { slot = Some(func()); Ok(()) - }) - .map_err(|e| Trap::from(anyhow::Error::from(e)))?; + }; + + let (fiber, stack) = match config.instance_allocator().allocate_fiber_stack() { + Ok(stack) => { + // Use the returned stack and deallocate it when finished + ( + unsafe { + wasmtime_fiber::Fiber::new_with_stack(stack, func) + .map_err(|e| Trap::from(anyhow::Error::from(e)))? + }, + stack, + ) + } + Err(wasmtime_runtime::FiberStackError::NotSupported) => { + // The allocator doesn't support custom fiber stacks for the current platform + // Request that the fiber itself allocate the stack + ( + wasmtime_fiber::Fiber::new(config.async_stack_size, func) + .map_err(|e| Trap::from(anyhow::Error::from(e)))?, + std::ptr::null_mut(), + ) + } + Err(e) => return Err(Trap::from(anyhow::Error::from(e))), + }; // Once we have the fiber representing our synchronous computation, we // wrap that in a custom future implementation which does the // translation from the future protocol to our fiber API. - FiberFuture { fiber, store: self }.await?; + FiberFuture { + fiber, + store: self, + stack, + } + .await?; return Ok(slot.unwrap()); struct FiberFuture<'a> { fiber: wasmtime_fiber::Fiber<'a, Result<(), Trap>, (), Result<(), Trap>>, store: &'a Store, + stack: *mut u8, } impl Future for FiberFuture<'_> { @@ -845,15 +875,23 @@ impl Store { // completion. impl Drop for FiberFuture<'_> { fn drop(&mut self) { - if self.fiber.done() { - return; + if !self.fiber.done() { + let result = self.fiber.resume(Err(Trap::new("future dropped"))); + // This resumption with an error should always complete the + // fiber. While it's technically possible for host code to catch + // the trap and re-resume, we'd ideally like to signal that to + // callers that they shouldn't be doing that. + debug_assert!(result.is_ok()); + } + if !self.stack.is_null() { + unsafe { + self.store + .engine() + .config() + .instance_allocator() + .deallocate_fiber_stack(self.stack) + }; } - let result = self.fiber.resume(Err(Trap::new("future dropped"))); - // This resumption with an error should always complete the - // fiber. While it's technically possible for host code to catch - // the trap and re-resume, we'd ideally like to signal that to - // callers that they shouldn't be doing that. - debug_assert!(result.is_ok()); } } } From 5ee2b8742a53e93c5b6a7ca48a338096864c8d1f Mon Sep 17 00:00:00 2001 From: Peter Huene Date: Wed, 3 Mar 2021 13:27:56 -0800 Subject: [PATCH 22/33] Have `new_with_stack` impls return `io::Result`. --- crates/fiber/src/lib.rs | 2 +- crates/fiber/src/unix.rs | 4 ++-- crates/fiber/src/windows.rs | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/crates/fiber/src/lib.rs b/crates/fiber/src/lib.rs index 6c835c511f81..9a8d057f35c6 100644 --- a/crates/fiber/src/lib.rs +++ b/crates/fiber/src/lib.rs @@ -66,7 +66,7 @@ impl<'a, Resume, Yield, Return> Fiber<'a, Resume, Yield, Return> { func: impl FnOnce(Resume, &Suspend) -> Return + 'a, ) -> io::Result> { Ok(Fiber { - inner: imp::Fiber::new_with_stack(top_of_stack, func), + inner: imp::Fiber::new_with_stack(top_of_stack, func)?, done: Cell::new(false), _phantom: PhantomData, }) diff --git a/crates/fiber/src/unix.rs b/crates/fiber/src/unix.rs index c14a188f50f5..0cc57ca319ce 100644 --- a/crates/fiber/src/unix.rs +++ b/crates/fiber/src/unix.rs @@ -75,7 +75,7 @@ impl Fiber { Ok(fiber) } - pub fn new_with_stack(top_of_stack: *mut u8, func: F) -> Self + pub fn new_with_stack(top_of_stack: *mut u8, func: F) -> io::Result where F: FnOnce(A, &super::Suspend) -> C, { @@ -86,7 +86,7 @@ impl Fiber { fiber.init(func); - fiber + Ok(fiber) } fn init(&self, func: F) diff --git a/crates/fiber/src/windows.rs b/crates/fiber/src/windows.rs index c40fb8aeb031..b2d657eb8848 100644 --- a/crates/fiber/src/windows.rs +++ b/crates/fiber/src/windows.rs @@ -3,6 +3,7 @@ use std::cell::Cell; use std::io; use std::ptr; use winapi::shared::minwindef::*; +use winapi::shared::winerror::ERROR_NOT_SUPPORTED; use winapi::um::fibersapi::*; use winapi::um::winbase::*; @@ -66,12 +67,11 @@ impl Fiber { } } - pub fn new_with_stack(_top_of_stack: *mut u8, _func: F) -> Self + pub fn new_with_stack(_top_of_stack: *mut u8, _func: F) -> io::Result where F: FnOnce(A, &super::Suspend) -> C, { - // Windows fibers have no support for custom stacks - unimplemented!() + Err(io::Error::from_raw_os_error(ERROR_NOT_SUPPORTED as i32)) } pub(crate) fn resume(&self, result: &Cell>) { From a464465e2f24b33d694acb140fecba001943500b Mon Sep 17 00:00:00 2001 From: Peter Huene Date: Wed, 3 Mar 2021 16:41:33 -0800 Subject: [PATCH 23/33] Code review feedback changes. * Add `anyhow` dependency to `wasmtime-runtime`. * Revert `get_data` back to `fn`. * Remove `DataInitializer` and box the data in `Module` translation instead. * Improve comments on `MemoryInitialization`. * Remove `MemoryInitialization::OutOfBounds` in favor of proper bulk memory semantics. * Use segmented memory initialization except for when the uffd feature is enabled on Linux. * Validate modules with the allocator after translation. * Updated various functions in the runtime to return `anyhow::Result`. * Use a slice when copying pages instead of `ptr::copy_nonoverlapping`. * Remove unnecessary casts in `OnDemandAllocator::deallocate`. * Better document the `uffd` feature. * Use WebAssembly page-sized pages in the paged initialization. * Remove the stack pool from the uffd handler and simply protect just the guard pages. --- Cargo.lock | 1 + crates/cache/src/lib.rs | 2 +- crates/cache/src/tests.rs | 80 +-- crates/environ/src/module.rs | 236 ++++----- crates/environ/src/module_environ.rs | 48 +- crates/jit/src/instantiate.rs | 21 +- crates/obj/src/data_segment.rs | 8 +- crates/obj/src/module.rs | 22 +- crates/runtime/Cargo.toml | 1 + crates/runtime/src/instance.rs | 4 +- crates/runtime/src/instance/allocator.rs | 69 ++- .../runtime/src/instance/allocator/pooling.rs | 307 +++++------ .../src/instance/allocator/pooling/linux.rs | 5 +- .../src/instance/allocator/pooling/uffd.rs | 485 ++++++------------ .../src/instance/allocator/pooling/unix.rs | 5 +- .../src/instance/allocator/pooling/windows.rs | 5 +- crates/wasmtime/src/config.rs | 15 +- crates/wasmtime/src/module.rs | 52 +- tests/all/pooling_allocator.rs | 10 +- 19 files changed, 577 insertions(+), 799 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 73dbdecd48e9..6b76f57d6166 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3450,6 +3450,7 @@ dependencies = [ name = "wasmtime-runtime" version = "0.24.0" dependencies = [ + "anyhow", "backtrace", "cc", "cfg-if 1.0.0", diff --git a/crates/cache/src/lib.rs b/crates/cache/src/lib.rs index fc50ff4dad7c..9fe69c5bafd2 100644 --- a/crates/cache/src/lib.rs +++ b/crates/cache/src/lib.rs @@ -43,7 +43,7 @@ impl<'config> ModuleCacheEntry<'config> { } /// Gets cached data if state matches, otherwise calls the `compute`. - pub fn get_data(&self, state: T, compute: impl Fn(T) -> Result) -> Result + pub fn get_data(&self, state: T, compute: fn(T) -> Result) -> Result where T: Hash, U: Serialize + for<'a> Deserialize<'a>, diff --git a/crates/cache/src/tests.rs b/crates/cache/src/tests.rs index 857a1f0ab76b..4362aaba2222 100644 --- a/crates/cache/src/tests.rs +++ b/crates/cache/src/tests.rs @@ -65,68 +65,28 @@ fn test_write_read_cache() { let entry1 = ModuleCacheEntry::from_inner(ModuleCacheEntryInner::new(compiler1, &cache_config)); let entry2 = ModuleCacheEntry::from_inner(ModuleCacheEntryInner::new(compiler2, &cache_config)); - entry1 - .get_data(1, |_| -> Result { Ok(100) }) - .unwrap(); - entry1 - .get_data(1, |_| -> Result { panic!() }) - .unwrap(); + entry1.get_data::<_, i32, i32>(1, |_| Ok(100)).unwrap(); + entry1.get_data::<_, i32, i32>(1, |_| panic!()).unwrap(); - entry1 - .get_data(2, |_| -> Result { Ok(100) }) - .unwrap(); - entry1 - .get_data(1, |_| -> Result { panic!() }) - .unwrap(); - entry1 - .get_data(2, |_| -> Result { panic!() }) - .unwrap(); + entry1.get_data::<_, i32, i32>(2, |_| Ok(100)).unwrap(); + entry1.get_data::<_, i32, i32>(1, |_| panic!()).unwrap(); + entry1.get_data::<_, i32, i32>(2, |_| panic!()).unwrap(); - entry1 - .get_data(3, |_| -> Result { Ok(100) }) - .unwrap(); - entry1 - .get_data(1, |_| -> Result { panic!() }) - .unwrap(); - entry1 - .get_data(2, |_| -> Result { panic!() }) - .unwrap(); - entry1 - .get_data(3, |_| -> Result { panic!() }) - .unwrap(); + entry1.get_data::<_, i32, i32>(3, |_| Ok(100)).unwrap(); + entry1.get_data::<_, i32, i32>(1, |_| panic!()).unwrap(); + entry1.get_data::<_, i32, i32>(2, |_| panic!()).unwrap(); + entry1.get_data::<_, i32, i32>(3, |_| panic!()).unwrap(); - entry1 - .get_data(4, |_| -> Result { Ok(100) }) - .unwrap(); - entry1 - .get_data(1, |_| -> Result { panic!() }) - .unwrap(); - entry1 - .get_data(2, |_| -> Result { panic!() }) - .unwrap(); - entry1 - .get_data(3, |_| -> Result { panic!() }) - .unwrap(); - entry1 - .get_data(4, |_| -> Result { panic!() }) - .unwrap(); + entry1.get_data::<_, i32, i32>(4, |_| Ok(100)).unwrap(); + entry1.get_data::<_, i32, i32>(1, |_| panic!()).unwrap(); + entry1.get_data::<_, i32, i32>(2, |_| panic!()).unwrap(); + entry1.get_data::<_, i32, i32>(3, |_| panic!()).unwrap(); + entry1.get_data::<_, i32, i32>(4, |_| panic!()).unwrap(); - entry2 - .get_data(1, |_| -> Result { Ok(100) }) - .unwrap(); - entry1 - .get_data(1, |_| -> Result { panic!() }) - .unwrap(); - entry1 - .get_data(2, |_| -> Result { panic!() }) - .unwrap(); - entry1 - .get_data(3, |_| -> Result { panic!() }) - .unwrap(); - entry1 - .get_data(4, |_| -> Result { panic!() }) - .unwrap(); - entry2 - .get_data(1, |_| -> Result { panic!() }) - .unwrap(); + entry2.get_data::<_, i32, i32>(1, |_| Ok(100)).unwrap(); + entry1.get_data::<_, i32, i32>(1, |_| panic!()).unwrap(); + entry1.get_data::<_, i32, i32>(2, |_| panic!()).unwrap(); + entry1.get_data::<_, i32, i32>(3, |_| panic!()).unwrap(); + entry1.get_data::<_, i32, i32>(4, |_| panic!()).unwrap(); + entry2.get_data::<_, i32, i32>(1, |_| panic!()).unwrap(); } diff --git a/crates/environ/src/module.rs b/crates/environ/src/module.rs index b70748ec723c..ee4ff050dc71 100644 --- a/crates/environ/src/module.rs +++ b/crates/environ/src/module.rs @@ -1,7 +1,7 @@ //! Data structures for representing decoded wasm modules. use crate::tunables::Tunables; -use crate::{DataInitializer, WASM_MAX_PAGES, WASM_PAGE_SIZE}; +use crate::WASM_MAX_PAGES; use cranelift_codegen::ir; use cranelift_entity::{EntityRef, PrimaryMap}; use cranelift_wasm::*; @@ -92,152 +92,144 @@ pub struct MemoryInitializer { pub data: Box<[u8]>, } -impl From> for MemoryInitializer { - fn from(initializer: DataInitializer) -> Self { - Self { - memory_index: initializer.memory_index, - base: initializer.base, - offset: initializer.offset, - data: initializer.data.into(), - } - } -} - -/// The type of WebAssembly linear memory initialization. +/// The type of WebAssembly linear memory initialization to use for a module. #[derive(Clone, Debug, Serialize, Deserialize)] pub enum MemoryInitialization { + /// Memory initialization is segmented. + /// + /// Segmented initialization can be used for any module, but it is required if: + /// + /// * A data segment referenced an imported memory. + /// * A data segment uses a global base. + /// + /// Segmented initialization is performed by processing the complete set of data segments + /// when the module is instantiated. + /// + /// This is the default memory initialization type. + Segmented(Vec), /// Memory initialization is paged. /// /// To be paged, the following requirements must be met: /// /// * All data segments must reference defined memories. /// * All data segments must not use a global base. - /// * All data segments must be in bounds. /// - /// Paged initialization is performed by memcopying individual pages to the linear memory. + /// Paged initialization is performed by copying (or mapping) entire WebAssembly pages to each linear memory. + /// + /// The `uffd` feature makes use of this type of memory initialization because it can instruct the kernel + /// to back an entire WebAssembly page from an existing set of in-memory pages. + /// + /// By processing the data segments at module compilation time, the uffd fault handler doesn't have to do + /// any work to point the kernel at the right linear memory page to use. Paged { - /// The size of each page stored in the map. - /// This is expected to be the host page size. - page_size: usize, - /// The map of defined memory index to a list of page data. + /// The map of defined memory index to a list of initialization pages. /// The list of page data is sparse, with None representing a zero page. + /// Each page of initialization data is WebAssembly page-sized (64 KiB). /// The size of the list will be the maximum page written to by a data segment. map: PrimaryMap>>>, + /// Whether or not an out-of-bounds data segment was observed. + /// This is used to fail module instantiation after the pages are initialized. + out_of_bounds: bool, }, - /// Memory initialization is out of bounds. - /// - /// To be out of bounds, the following requirements must be met: - /// - /// * All data segments must reference defined memories. - /// * All data segments must not use a global base. - /// * At least one data segments was out of bounds. - /// - /// This can be used to quickly return an error when the module is instantiated. - OutOfBounds, - /// Memory initialization is segmented. - /// - /// To be segmented, at least one of the following requirements must be met: - /// - /// * A data segment referenced an imported memory. - /// * A data segment uses a global base. - /// - /// Segmented initialization is performed by processing the complete set of data segments - /// when the module is instantiated. - /// - /// This ensures that initialization side-effects are observed according to the bulk-memory proposal. - Segmented(Box<[MemoryInitializer]>), } impl MemoryInitialization { - /// Creates a new memory initialization for a module and its data initializers. - pub fn new(module: &Module, initializers: Vec) -> Self { - let page_size = region::page::size(); - let num_defined_memories = module.memory_plans.len() - module.num_imported_memories; - let mut out_of_bounds = false; - let mut memories = PrimaryMap::with_capacity(num_defined_memories); - - for _ in 0..num_defined_memories { - memories.push(Vec::new()); - } + /// Attempts to convert segmented memory initialization into paged initialization for the given module. + /// + /// Returns `None` if the initialization cannot be paged or if it is already paged. + pub fn to_paged(&self, module: &Module) -> Option { + const WASM_PAGE_SIZE: usize = crate::WASM_PAGE_SIZE as usize; - for initializer in &initializers { - match ( - module.defined_memory_index(initializer.memory_index), - initializer.base.is_some(), - ) { - (None, _) | (_, true) => { - // If the initializer references an imported memory or uses a global base, - // the complete set of segments will need to be processed at module instantiation - return Self::Segmented( - initializers - .into_iter() - .map(Into::into) - .collect::>() - .into_boxed_slice(), - ); + match self { + Self::Paged { .. } => None, + Self::Segmented(initializers) => { + let num_defined_memories = module.memory_plans.len() - module.num_imported_memories; + let mut out_of_bounds = false; + let mut map = PrimaryMap::with_capacity(num_defined_memories); + + for _ in 0..num_defined_memories { + map.push(Vec::new()); } - (Some(index), false) => { - if out_of_bounds { - continue; - } - - // Perform a bounds check on the segment - if (initializer.offset + initializer.data.len()) - > ((module.memory_plans[initializer.memory_index].memory.minimum as usize) - * (WASM_PAGE_SIZE as usize)) - { - out_of_bounds = true; - continue; - } - - let pages = &mut memories[index]; - let mut page_index = initializer.offset / page_size; - let mut page_offset = initializer.offset % page_size; - let mut data_offset = 0; - let mut data_remaining = initializer.data.len(); - - if data_remaining == 0 { - continue; - } - - // Copy the initialization data by each page - loop { - if page_index >= pages.len() { - pages.resize(page_index + 1, None); - } - - let page = pages[page_index] - .get_or_insert_with(|| vec![0; page_size].into_boxed_slice()); - let len = std::cmp::min(data_remaining, page_size - page_offset); - - page[page_offset..page_offset + len] - .copy_from_slice(&initializer.data[data_offset..(data_offset + len)]); - if len == data_remaining { - break; + for initializer in initializers { + match ( + module.defined_memory_index(initializer.memory_index), + initializer.base.is_some(), + ) { + (None, _) | (_, true) => { + // If the initializer references an imported memory or uses a global base, + // the complete set of segments will need to be processed at module instantiation + return None; } - - page_index += 1; - page_offset = 0; - data_offset += len; - data_remaining -= len; - } + (Some(index), false) => { + if out_of_bounds { + continue; + } + + // Perform a bounds check on the segment + // As this segment is referencing a defined memory without a global base, the last byte + // written to by the segment cannot exceed the memory's initial minimum size + if (initializer.offset + initializer.data.len()) + > ((module.memory_plans[initializer.memory_index].memory.minimum + as usize) + * WASM_PAGE_SIZE) + { + out_of_bounds = true; + continue; + } + + let pages = &mut map[index]; + let mut page_index = initializer.offset / WASM_PAGE_SIZE; + let mut page_offset = initializer.offset % WASM_PAGE_SIZE; + let mut data_offset = 0; + let mut data_remaining = initializer.data.len(); + + if data_remaining == 0 { + continue; + } + + // Copy the initialization data by each WebAssembly-sized page (64 KiB) + loop { + if page_index >= pages.len() { + pages.resize(page_index + 1, None); + } + + let page = pages[page_index].get_or_insert_with(|| { + vec![0; WASM_PAGE_SIZE].into_boxed_slice() + }); + let len = + std::cmp::min(data_remaining, WASM_PAGE_SIZE - page_offset); + + page[page_offset..page_offset + len].copy_from_slice( + &initializer.data[data_offset..(data_offset + len)], + ); + + if len == data_remaining { + break; + } + + page_index += 1; + page_offset = 0; + data_offset += len; + data_remaining -= len; + } + } + }; } - }; - } - if out_of_bounds { - Self::OutOfBounds - } else { - Self::Paged { - page_size, - map: memories, + Some(Self::Paged { map, out_of_bounds }) } } } } -/// Implemenation styles for WebAssembly tables. +impl Default for MemoryInitialization { + fn default() -> Self { + Self::Segmented(Vec::new()) + } +} + +/// Implementation styles for WebAssembly tables. #[derive(Debug, Clone, Hash, Serialize, Deserialize)] pub enum TableStyle { /// Signatures are stored in the table and checked in the caller. @@ -325,7 +317,7 @@ pub struct Module { pub table_initializers: Vec, /// WebAssembly linear memory initializer. - pub memory_initialization: Option, + pub memory_initialization: MemoryInitialization, /// WebAssembly passive elements. pub passive_elements: Vec>, @@ -405,7 +397,7 @@ pub enum Initializer { export: String, }, - /// A module is being instantiated with previously configured intializers + /// A module is being instantiated with previously configured initializers /// as arguments. Instantiate { /// The module that this instance is instantiating. @@ -417,7 +409,7 @@ pub enum Initializer { /// A module is being created from a set of compiled artifacts. CreateModule { - /// The index of the artifact that's being convereted into a module. + /// The index of the artifact that's being converted into a module. artifact_index: usize, /// The list of artifacts that this module value will be inheriting. artifacts: Vec, diff --git a/crates/environ/src/module_environ.rs b/crates/environ/src/module_environ.rs index 636fa2893e7d..2f53bb4af838 100644 --- a/crates/environ/src/module_environ.rs +++ b/crates/environ/src/module_environ.rs @@ -1,6 +1,6 @@ use crate::module::{ - Initializer, InstanceSignature, MemoryPlan, Module, ModuleSignature, ModuleType, ModuleUpvar, - TableInitializer, TablePlan, TypeTables, + Initializer, InstanceSignature, MemoryInitialization, MemoryInitializer, MemoryPlan, Module, + ModuleSignature, ModuleType, ModuleUpvar, TableInitializer, TablePlan, TypeTables, }; use crate::tunables::Tunables; use cranelift_codegen::ir; @@ -59,9 +59,6 @@ pub struct ModuleTranslation<'data> { /// References to the function bodies. pub function_body_inputs: PrimaryMap>, - /// References to the data initializers. - pub data_initializers: Vec>, - /// DWARF debug information, if enabled, parsed from the module. pub debuginfo: DebugInfoData<'data>, @@ -762,9 +759,12 @@ impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data } fn reserve_data_initializers(&mut self, num: u32) -> WasmResult<()> { - self.result - .data_initializers - .reserve_exact(usize::try_from(num).unwrap()); + match &mut self.result.module.memory_initialization { + MemoryInitialization::Segmented(initializers) => { + initializers.reserve_exact(usize::try_from(num).unwrap()) + } + _ => unreachable!(), + } Ok(()) } @@ -775,12 +775,17 @@ impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data offset: usize, data: &'data [u8], ) -> WasmResult<()> { - self.result.data_initializers.push(DataInitializer { - memory_index, - base, - offset, - data, - }); + match &mut self.result.module.memory_initialization { + MemoryInitialization::Segmented(initializers) => { + initializers.push(MemoryInitializer { + memory_index, + base, + offset, + data: data.into(), + }); + } + _ => unreachable!(), + } Ok(()) } @@ -1071,18 +1076,3 @@ pub fn translate_signature(mut sig: ir::Signature, pointer_type: ir::Type) -> ir sig.params.insert(1, AbiParam::new(pointer_type)); sig } - -/// A data initializer for linear memory. -pub struct DataInitializer<'data> { - /// The index of the memory to initialize. - pub memory_index: MemoryIndex, - - /// Optionally a globalvar base to initialize at. - pub base: Option, - - /// A constant offset to initialize at. - pub offset: usize, - - /// The initialization data. - pub data: &'data [u8], -} diff --git a/crates/jit/src/instantiate.rs b/crates/jit/src/instantiate.rs index b4875305be09..250b83728190 100644 --- a/crates/jit/src/instantiate.rs +++ b/crates/jit/src/instantiate.rs @@ -21,9 +21,8 @@ use wasmtime_environ::wasm::{ DefinedFuncIndex, InstanceTypeIndex, ModuleTypeIndex, SignatureIndex, WasmFuncType, }; use wasmtime_environ::{ - CompileError, DebugInfoData, FunctionAddressMap, InstanceSignature, MemoryInitialization, - Module, ModuleEnvironment, ModuleSignature, ModuleTranslation, StackMapInformation, - TrapInformation, + CompileError, DebugInfoData, FunctionAddressMap, InstanceSignature, Module, ModuleEnvironment, + ModuleSignature, ModuleTranslation, StackMapInformation, TrapInformation, }; use wasmtime_profiling::ProfilingAgent; use wasmtime_runtime::{GdbJitImageRegistration, InstantiationError, VMFunctionBody, VMTrampoline}; @@ -95,10 +94,14 @@ struct DebugInfo { impl CompilationArtifacts { /// Creates a `CompilationArtifacts` for a singular translated wasm module. + /// + /// The `use_paged_init` argument controls whether or not an attempt is made to + /// organize linear memory initialization data as entire pages or to leave + /// the memory initialization data as individual segments. pub fn build( compiler: &Compiler, data: &[u8], - validate: impl Fn(&ModuleTranslation) -> Result<(), String> + Sync, + use_paged_mem_init: bool, ) -> Result<(usize, Vec, TypeTables), SetupError> { let (main_module, translations, types) = ModuleEnvironment::new( compiler.frontend_config(), @@ -110,8 +113,6 @@ impl CompilationArtifacts { let list = maybe_parallel!(translations.(into_iter | into_par_iter)) .map(|mut translation| { - validate(&translation).map_err(|e| SetupError::Validate(e))?; - let Compilation { obj, unwind_info, @@ -120,14 +121,16 @@ impl CompilationArtifacts { let ModuleTranslation { mut module, - data_initializers, debuginfo, has_unparsed_debuginfo, .. } = translation; - module.memory_initialization = - Some(MemoryInitialization::new(&module, data_initializers)); + if use_paged_mem_init { + if let Some(init) = module.memory_initialization.to_paged(&module) { + module.memory_initialization = init; + } + } let obj = obj.write().map_err(|_| { SetupError::Instantiate(InstantiationError::Resource( diff --git a/crates/obj/src/data_segment.rs b/crates/obj/src/data_segment.rs index 159922f25b53..3e4184eb205e 100644 --- a/crates/obj/src/data_segment.rs +++ b/crates/obj/src/data_segment.rs @@ -1,12 +1,12 @@ use anyhow::Result; use object::write::{Object, StandardSection, Symbol, SymbolSection}; use object::{SymbolFlags, SymbolKind, SymbolScope}; -use wasmtime_environ::DataInitializer; +use wasmtime_environ::MemoryInitializer; /// Declares data segment symbol pub fn declare_data_segment( obj: &mut Object, - _data_initaliazer: &DataInitializer, + _memory_initializer: &MemoryInitializer, index: usize, ) -> Result<()> { let name = format!("_memory_{}", index); @@ -26,12 +26,12 @@ pub fn declare_data_segment( /// Emit segment data and initialization location pub fn emit_data_segment( obj: &mut Object, - data_initaliazer: &DataInitializer, + memory_initializer: &MemoryInitializer, index: usize, ) -> Result<()> { let name = format!("_memory_{}", index); let symbol_id = obj.symbol_id(name.as_bytes()).unwrap(); let section_id = obj.section_id(StandardSection::Data); - obj.add_symbol_data(symbol_id, section_id, data_initaliazer.data, 1); + obj.add_symbol_data(symbol_id, section_id, &memory_initializer.data, 1); Ok(()) } diff --git a/crates/obj/src/module.rs b/crates/obj/src/module.rs index 2adf1aa39368..4150d1c8b85c 100644 --- a/crates/obj/src/module.rs +++ b/crates/obj/src/module.rs @@ -7,7 +7,7 @@ use object::write::{Object, Relocation, StandardSection, Symbol, SymbolSection}; use object::{RelocationEncoding, RelocationKind, SymbolFlags, SymbolKind, SymbolScope}; use wasmtime_debug::DwarfSection; use wasmtime_environ::isa::TargetFrontendConfig; -use wasmtime_environ::{CompiledFunctions, DataInitializer, Module}; +use wasmtime_environ::{CompiledFunctions, MemoryInitialization, Module}; fn emit_vmcontext_init( obj: &mut Object, @@ -54,24 +54,32 @@ pub fn emit_module( target_config: &TargetFrontendConfig, compilation: CompiledFunctions, dwarf_sections: Vec, - data_initializers: &[DataInitializer], ) -> Result { let mut builder = ObjectBuilder::new(target, module, &compilation); builder.set_dwarf_sections(dwarf_sections); let mut obj = builder.build()?; // Append data, table and vmcontext_init code to the object file. - - for (i, initializer) in data_initializers.iter().enumerate() { - declare_data_segment(&mut obj, initializer, i)?; + match &module.memory_initialization { + MemoryInitialization::Segmented(initializers) => { + for (i, initializer) in initializers.iter().enumerate() { + declare_data_segment(&mut obj, initializer, i)?; + } + } + _ => unimplemented!(), } for i in 0..module.table_plans.len() { declare_table(&mut obj, i)?; } - for (i, initializer) in data_initializers.iter().enumerate() { - emit_data_segment(&mut obj, initializer, i)?; + match &module.memory_initialization { + MemoryInitialization::Segmented(initializers) => { + for (i, initializer) in initializers.iter().enumerate() { + emit_data_segment(&mut obj, initializer, i)?; + } + } + _ => unimplemented!(), } for i in 0..module.table_plans.len() { diff --git a/crates/runtime/Cargo.toml b/crates/runtime/Cargo.toml index af73a7d1028d..31a020fd4ede 100644 --- a/crates/runtime/Cargo.toml +++ b/crates/runtime/Cargo.toml @@ -25,6 +25,7 @@ backtrace = "0.3.55" lazy_static = "1.3.0" psm = "0.1.11" rand = "0.7.3" +anyhow = "1.0.38" [target.'cfg(target_os = "windows")'.dependencies] winapi = { version = "0.3.7", features = ["winbase", "memoryapi", "errhandlingapi"] } diff --git a/crates/runtime/src/instance.rs b/crates/runtime/src/instance.rs index c90c267f6bf2..313642a077ab 100644 --- a/crates/runtime/src/instance.rs +++ b/crates/runtime/src/instance.rs @@ -834,12 +834,12 @@ impl Instance { /// /// Resetting the guard pages is required before growing memory. #[cfg(all(feature = "uffd", target_os = "linux"))] - pub(crate) fn reset_guard_pages(&self) -> Result<(), String> { + pub(crate) fn reset_guard_pages(&self) -> anyhow::Result<()> { let mut faults = self.guard_page_faults.borrow_mut(); for (addr, len, reset) in faults.drain(..) { unsafe { if !reset(addr, len) { - return Err("failed to reset previously faulted memory guard page".into()); + anyhow::bail!("failed to reset previously faulted memory guard page"); } } } diff --git a/crates/runtime/src/instance/allocator.rs b/crates/runtime/src/instance/allocator.rs index aae17c7f4d91..c835c144216a 100644 --- a/crates/runtime/src/instance/allocator.rs +++ b/crates/runtime/src/instance/allocator.rs @@ -9,6 +9,7 @@ use crate::vmcontext::{ VMGlobalDefinition, VMGlobalImport, VMInterrupts, VMMemoryImport, VMSharedSignatureIndex, VMTableImport, }; +use anyhow::Result; use std::alloc; use std::any::Any; use std::cell::RefCell; @@ -23,8 +24,8 @@ use wasmtime_environ::wasm::{ TableElementType, WasmType, }; use wasmtime_environ::{ - ir, MemoryInitialization, MemoryInitializer, Module, ModuleTranslation, ModuleType, - TableInitializer, VMOffsets, + ir, MemoryInitialization, MemoryInitializer, Module, ModuleType, TableInitializer, VMOffsets, + WASM_PAGE_SIZE, }; mod pooling; @@ -105,11 +106,9 @@ pub enum FiberStackError { /// /// This trait is unsafe as it requires knowledge of Wasmtime's runtime internals to implement correctly. pub unsafe trait InstanceAllocator: Send + Sync { - /// Validates a module translation. - /// - /// This is used to ensure a module being compiled is supported by the instance allocator. - fn validate_module(&self, translation: &ModuleTranslation) -> Result<(), String> { - drop(translation); + /// Validates that a module is supported by the allocator. + fn validate(&self, module: &Module) -> Result<()> { + drop(module); Ok(()) } @@ -322,15 +321,14 @@ fn check_init_bounds(instance: &Instance) -> Result<(), InstantiationError> { check_table_init_bounds(instance)?; match &instance.module.memory_initialization { - Some(MemoryInitialization::Paged { .. }) | None => { - // Bounds were checked at compile-time - } - Some(MemoryInitialization::OutOfBounds) => { - return Err(InstantiationError::Link(LinkError( - "memory out of bounds: data segment does not fit".into(), - ))); + MemoryInitialization::Paged { out_of_bounds, .. } => { + if *out_of_bounds { + return Err(InstantiationError::Link(LinkError( + "memory out of bounds: data segment does not fit".into(), + ))); + } } - Some(MemoryInitialization::Segmented(initializers)) => { + MemoryInitialization::Segmented(initializers) => { check_memory_init_bounds(instance, initializers)?; } } @@ -355,37 +353,31 @@ fn initialize_instance( // Initialize the memories match &instance.module.memory_initialization { - Some(MemoryInitialization::Paged { page_size, map }) => { + MemoryInitialization::Paged { map, out_of_bounds } => { for (index, pages) in map { let memory = instance.memory(index); + let slice = + unsafe { slice::from_raw_parts_mut(memory.base, memory.current_length) }; for (page_index, page) in pages.iter().enumerate() { if let Some(data) = page { - // Bounds checking should have occurred when the module was compiled - // The data should always be page sized - assert!((page_index * page_size) < memory.current_length); - assert_eq!(data.len(), *page_size); - - unsafe { - ptr::copy_nonoverlapping( - data.as_ptr(), - memory.base.add(page_index * page_size), - data.len(), - ); - } + debug_assert_eq!(data.len(), WASM_PAGE_SIZE as usize); + slice[page_index * WASM_PAGE_SIZE as usize..].copy_from_slice(data); } } } + + // Check for out of bound access after initializing the pages to maintain + // the expected behavior of the bulk memory spec. + if *out_of_bounds { + return Err(InstantiationError::Trap(Trap::wasm( + ir::TrapCode::HeapOutOfBounds, + ))); + } } - Some(MemoryInitialization::OutOfBounds) => { - return Err(InstantiationError::Trap(Trap::wasm( - ir::TrapCode::HeapOutOfBounds, - ))) - } - Some(MemoryInitialization::Segmented(initializers)) => { + MemoryInitialization::Segmented(initializers) => { initialize_memories(instance, initializers)?; } - None => {} } Ok(()) @@ -615,10 +607,9 @@ unsafe impl InstanceAllocator for OnDemandInstanceAllocator { } unsafe fn deallocate(&self, handle: &InstanceHandle) { - let instance = handle.instance(); - let layout = instance.alloc_layout(); - ptr::drop_in_place(instance as *const Instance as *mut Instance); - alloc::dealloc(instance as *const Instance as *mut _, layout); + let layout = handle.instance().alloc_layout(); + ptr::drop_in_place(handle.instance); + alloc::dealloc(handle.instance.cast(), layout); } fn allocate_fiber_stack(&self) -> Result<*mut u8, FiberStackError> { diff --git a/crates/runtime/src/instance/allocator/pooling.rs b/crates/runtime/src/instance/allocator/pooling.rs index c759780ad856..5029d90667b9 100644 --- a/crates/runtime/src/instance/allocator/pooling.rs +++ b/crates/runtime/src/instance/allocator/pooling.rs @@ -12,6 +12,7 @@ use super::{ InstanceAllocator, InstanceHandle, InstantiationError, }; use crate::{instance::Instance, table::max_table_element_size, Memory, Mmap, Table, VMContext}; +use anyhow::{anyhow, bail, Context, Result}; use rand::Rng; use std::cell::RefCell; use std::cmp::min; @@ -20,7 +21,7 @@ use std::mem; use std::sync::{Arc, Mutex}; use wasmtime_environ::{ entity::{EntitySet, PrimaryMap}, - MemoryStyle, Module, ModuleTranslation, Tunables, VMOffsets, WASM_PAGE_SIZE, + MemoryStyle, Module, Tunables, VMOffsets, WASM_PAGE_SIZE, }; cfg_if::cfg_if! { @@ -30,10 +31,9 @@ cfg_if::cfg_if! { } else if #[cfg(all(feature = "uffd", target_os = "linux"))] { mod uffd; use uffd as imp; - use imp::{PageFaultHandler, reset_guard_page}; + use imp::PageFaultHandler; use super::{check_init_bounds, initialize_tables}; use wasmtime_environ::MemoryInitialization; - use std::sync::atomic::{AtomicBool, Ordering}; } else if #[cfg(target_os = "linux")] { mod linux; use linux as imp; @@ -105,73 +105,81 @@ pub struct ModuleLimits { } impl ModuleLimits { - fn validate_module(&self, module: &Module) -> Result<(), String> { + fn validate(&self, module: &Module) -> Result<()> { if module.num_imported_funcs > self.imported_functions as usize { - return Err(format!( + bail!( "imported function count of {} exceeds the limit of {}", - module.num_imported_funcs, self.imported_functions - )); + module.num_imported_funcs, + self.imported_functions + ); } if module.num_imported_tables > self.imported_tables as usize { - return Err(format!( + bail!( "imported tables count of {} exceeds the limit of {}", - module.num_imported_tables, self.imported_tables - )); + module.num_imported_tables, + self.imported_tables + ); } if module.num_imported_memories > self.imported_memories as usize { - return Err(format!( + bail!( "imported memories count of {} exceeds the limit of {}", - module.num_imported_memories, self.imported_memories - )); + module.num_imported_memories, + self.imported_memories + ); } if module.num_imported_globals > self.imported_globals as usize { - return Err(format!( + bail!( "imported globals count of {} exceeds the limit of {}", - module.num_imported_globals, self.imported_globals - )); + module.num_imported_globals, + self.imported_globals + ); } if module.types.len() > self.types as usize { - return Err(format!( + bail!( "defined types count of {} exceeds the limit of {}", module.types.len(), self.types - )); + ); } let functions = module.functions.len() - module.num_imported_funcs; if functions > self.functions as usize { - return Err(format!( + bail!( "defined functions count of {} exceeds the limit of {}", - functions, self.functions - )); + functions, + self.functions + ); } let tables = module.table_plans.len() - module.num_imported_tables; if tables > self.tables as usize { - return Err(format!( + bail!( "defined tables count of {} exceeds the limit of {}", - tables, self.tables - )); + tables, + self.tables + ); } let memories = module.memory_plans.len() - module.num_imported_memories; if memories > self.memories as usize { - return Err(format!( + bail!( "defined memories count of {} exceeds the limit of {}", - memories, self.memories - )); + memories, + self.memories + ); } let globals = module.globals.len() - module.num_imported_globals; if globals > self.globals as usize { - return Err(format!( + bail!( "defined globals count of {} exceeds the limit of {}", - globals, self.globals - )); + globals, + self.globals + ); } for (i, plan) in module.table_plans.values().as_slice()[module.num_imported_tables..] @@ -179,10 +187,12 @@ impl ModuleLimits { .enumerate() { if plan.table.minimum > self.table_elements { - return Err(format!( + bail!( "table index {} has a minimum element size of {} which exceeds the limit of {}", - i, plan.table.minimum, self.table_elements - )); + i, + plan.table.minimum, + self.table_elements + ); } } @@ -191,17 +201,19 @@ impl ModuleLimits { .enumerate() { if plan.memory.minimum > self.memory_pages { - return Err(format!( + bail!( "memory index {} has a minimum page size of {} which exceeds the limit of {}", - i, plan.memory.minimum, self.memory_pages - )); + i, + plan.memory.minimum, + self.memory_pages + ); } if let MemoryStyle::Dynamic = plan.style { - return Err(format!( + bail!( "memory index {} has an unsupported dynamic memory plan style", i, - )); + ); } } @@ -353,7 +365,7 @@ struct InstancePool { } impl InstancePool { - fn new(module_limits: &ModuleLimits, instance_limits: &InstanceLimits) -> Result { + fn new(module_limits: &ModuleLimits, instance_limits: &InstanceLimits) -> Result { let page_size = region::page::size(); // Calculate the maximum size of an Instance structure given the limits @@ -373,7 +385,7 @@ impl InstancePool { let instance_size = round_up_to_pow2( mem::size_of::() .checked_add(offsets.size_of_vmctx() as usize) - .ok_or_else(|| "instance size exceeds addressable memory".to_string())?, + .ok_or_else(|| anyhow!("instance size exceeds addressable memory"))?, page_size, ); @@ -381,7 +393,7 @@ impl InstancePool { let allocation_size = instance_size .checked_mul(max_instances) - .ok_or_else(|| "total size of instance data exceeds addressable memory".to_string())?; + .ok_or_else(|| anyhow!("total size of instance data exceeds addressable memory"))?; let pool = Self { mapping: create_memory_map(allocation_size, allocation_size)?, @@ -527,7 +539,7 @@ impl InstancePool { #[cfg(all(feature = "uffd", target_os = "linux"))] instance .reset_guard_pages() - .map_err(InstantiationError::Resource)?; + .map_err(|e| InstantiationError::Resource(e.to_string()))?; instance.memories.clear(); @@ -610,9 +622,9 @@ struct MemoryPool { } impl MemoryPool { - fn new(module_limits: &ModuleLimits, instance_limits: &InstanceLimits) -> Result { + fn new(module_limits: &ModuleLimits, instance_limits: &InstanceLimits) -> Result { let memory_size = usize::try_from(instance_limits.memory_reservation_size) - .map_err(|_| "memory reservation size exceeds addressable memory".to_string())?; + .map_err(|_| anyhow!("memory reservation size exceeds addressable memory"))?; debug_assert!( memory_size % region::page::size() == 0, @@ -627,7 +639,7 @@ impl MemoryPool { .checked_mul(max_memories) .and_then(|c| c.checked_mul(max_instances)) .ok_or_else(|| { - "total size of memory reservation exceeds addressable memory".to_string() + anyhow!("total size of memory reservation exceeds addressable memory") })?; Ok(Self { @@ -670,13 +682,13 @@ struct TablePool { } impl TablePool { - fn new(module_limits: &ModuleLimits, instance_limits: &InstanceLimits) -> Result { + fn new(module_limits: &ModuleLimits, instance_limits: &InstanceLimits) -> Result { let page_size = region::page::size(); let table_size = round_up_to_pow2( max_table_element_size() .checked_mul(module_limits.table_elements as usize) - .ok_or_else(|| "table size exceeds addressable memory".to_string())?, + .ok_or_else(|| anyhow!("table size exceeds addressable memory"))?, page_size, ); @@ -686,9 +698,7 @@ impl TablePool { let allocation_size = table_size .checked_mul(max_tables) .and_then(|c| c.checked_mul(max_instances)) - .ok_or_else(|| { - "total size of instance tables exceeds addressable memory".to_string() - })?; + .ok_or_else(|| anyhow!("total size of instance tables exceeds addressable memory"))?; Ok(Self { mapping: create_memory_map(0, allocation_size)?, @@ -733,12 +743,10 @@ struct StackPool { max_instances: usize, page_size: usize, free_list: Mutex>, - #[cfg(all(feature = "uffd", target_os = "linux"))] - faulted_guard_pages: Arc<[AtomicBool]>, } impl StackPool { - fn new(instance_limits: &InstanceLimits, stack_size: usize) -> Result { + fn new(instance_limits: &InstanceLimits, stack_size: usize) -> Result { let page_size = region::page::size(); // On Windows, don't allocate any fiber stacks as native fibers are always used @@ -748,26 +756,33 @@ impl StackPool { } else { round_up_to_pow2(stack_size, page_size) .checked_add(page_size) - .ok_or_else(|| "stack size exceeds addressable memory".to_string())? + .ok_or_else(|| anyhow!("stack size exceeds addressable memory"))? }; let max_instances = instance_limits.count as usize; - let allocation_size = stack_size.checked_mul(max_instances).ok_or_else(|| { - "total size of execution stacks exceeds addressable memory".to_string() - })?; + let allocation_size = stack_size + .checked_mul(max_instances) + .ok_or_else(|| anyhow!("total size of execution stacks exceeds addressable memory"))?; + + let mapping = create_memory_map(allocation_size, allocation_size)?; + + // Set up the stack guard pages + unsafe { + for i in 0..max_instances { + // Make the stack guard page inaccessible + let bottom_of_stack = mapping.as_mut_ptr().add(i * stack_size); + region::protect(bottom_of_stack, page_size, region::Protection::NONE) + .context("failed to protect stack guard page")?; + } + } Ok(Self { - mapping: create_memory_map(0, allocation_size)?, + mapping, stack_size, max_instances, page_size, free_list: Mutex::new((0..max_instances).collect()), - #[cfg(all(feature = "uffd", target_os = "linux"))] - faulted_guard_pages: std::iter::repeat_with(|| false.into()) - .take(max_instances) - .collect::>() - .into(), }) } @@ -789,37 +804,8 @@ impl StackPool { debug_assert!(index < self.max_instances); unsafe { - // Remove the guard page from the size - let size_without_guard = self.stack_size - self.page_size; - - let bottom_of_stack = self - .mapping - .as_mut_ptr() - .add((index * self.stack_size) + self.page_size); - - cfg_if::cfg_if! { - if #[cfg(all(feature = "uffd", target_os = "linux"))] { - // Check to see if a guard page needs to be reset - if self.faulted_guard_pages[index].swap(false, Ordering::SeqCst) { - if !reset_guard_page(bottom_of_stack.sub(self.page_size), self.page_size) { - return Err(FiberStackError::Resource( - "failed to reset stack guard page".into(), - )); - } - } - - } else { - // Make the stack accessible (excluding the guard page) - if !make_accessible(bottom_of_stack, size_without_guard) { - return Err(FiberStackError::Resource( - "failed to make instance memory accessible".into(), - )); - } - } - } - - // The top of the stack should be returned - Ok(bottom_of_stack.add(size_without_guard)) + // The top (end) of the stack should be returned + Ok(self.mapping.as_mut_ptr().add((index + 1) * self.stack_size)) } } @@ -872,9 +858,9 @@ impl PoolingInstanceAllocator { module_limits: ModuleLimits, mut instance_limits: InstanceLimits, stack_size: usize, - ) -> Result { + ) -> Result { if instance_limits.count == 0 { - return Err("the instance count limit cannot be zero".into()); + bail!("the instance count limit cannot be zero"); } // Round the memory reservation size to the nearest Wasm page size @@ -890,28 +876,28 @@ impl PoolingInstanceAllocator { // The maximum module memory page count cannot exceed 65536 pages if module_limits.memory_pages > 0x10000 { - return Err(format!( + bail!( "module memory page limit of {} exceeds the maximum of 65536", module_limits.memory_pages - )); + ); } // The maximum module memory page count cannot exceed the memory reservation size if (module_limits.memory_pages * WASM_PAGE_SIZE) as u64 > instance_limits.memory_reservation_size { - return Err(format!( + bail!( "module memory page limit of {} pages exeeds the memory reservation size limit of {} bytes", module_limits.memory_pages, instance_limits.memory_reservation_size - )); + ); } let instances = InstancePool::new(&module_limits, &instance_limits)?; let stacks = StackPool::new(&instance_limits, stack_size)?; #[cfg(all(feature = "uffd", target_os = "linux"))] - let _fault_handler = PageFaultHandler::new(&instances, &stacks)?; + let _fault_handler = PageFaultHandler::new(&instances)?; Ok(Self { strategy, @@ -937,8 +923,8 @@ impl Drop for PoolingInstanceAllocator { } unsafe impl InstanceAllocator for PoolingInstanceAllocator { - fn validate_module(&self, translation: &ModuleTranslation) -> Result<(), String> { - self.module_limits.validate_module(&translation.module) + fn validate(&self, module: &Module) -> Result<()> { + self.module_limits.validate(module) } fn adjust_tunables(&self, tunables: &mut Tunables) { @@ -976,8 +962,8 @@ unsafe impl InstanceAllocator for PoolingInstanceAllocator { cfg_if::cfg_if! { if #[cfg(all(feature = "uffd", target_os = "linux"))] { - match instance.module.memory_initialization { - Some(MemoryInitialization::Paged{ .. }) => { + match &instance.module.memory_initialization { + MemoryInitialization::Paged{ out_of_bounds, .. } => { if !is_bulk_memory { check_init_bounds(instance)?; } @@ -985,7 +971,15 @@ unsafe impl InstanceAllocator for PoolingInstanceAllocator { // Initialize the tables initialize_tables(instance)?; - // Don't initialize the memory; the fault handler will fill the pages when accessed + // Don't initialize the memory; the fault handler will back the pages when accessed + + // If there was an out of bounds access observed in initialization, return a trap + if *out_of_bounds { + return Err(InstantiationError::Trap(crate::traphandlers::Trap::wasm( + wasmtime_environ::ir::TrapCode::HeapOutOfBounds, + ))); + } + Ok(()) }, _ => initialize_instance(instance, is_bulk_memory) @@ -1030,11 +1024,11 @@ mod test { let mut module = Module::default(); module.functions.push(SignatureIndex::new(0)); - assert_eq!(limits.validate_module(&module), Ok(())); + assert!(limits.validate(&module).is_ok()); module.num_imported_funcs = 1; assert_eq!( - limits.validate_module(&module), + limits.validate(&module).map_err(|e| e.to_string()), Err("imported function count of 1 exceeds the limit of 0".into()) ); } @@ -1058,11 +1052,11 @@ mod test { }, }); - assert_eq!(limits.validate_module(&module), Ok(())); + assert!(limits.validate(&module).is_ok()); module.num_imported_tables = 1; assert_eq!( - limits.validate_module(&module), + limits.validate(&module).map_err(|e| e.to_string()), Err("imported tables count of 1 exceeds the limit of 0".into()) ); } @@ -1086,11 +1080,11 @@ mod test { offset_guard_size: 0, }); - assert_eq!(limits.validate_module(&module), Ok(())); + assert!(limits.validate(&module).is_ok()); module.num_imported_memories = 1; assert_eq!( - limits.validate_module(&module), + limits.validate(&module).map_err(|e| e.to_string()), Err("imported memories count of 1 exceeds the limit of 0".into()) ); } @@ -1111,11 +1105,11 @@ mod test { initializer: GlobalInit::I32Const(0), }); - assert_eq!(limits.validate_module(&module), Ok(())); + assert!(limits.validate(&module).is_ok()); module.num_imported_globals = 1; assert_eq!( - limits.validate_module(&module), + limits.validate(&module).map_err(|e| e.to_string()), Err("imported globals count of 1 exceeds the limit of 0".into()) ); } @@ -1128,13 +1122,13 @@ mod test { }; let mut module = Module::default(); - assert_eq!(limits.validate_module(&module), Ok(())); + assert!(limits.validate(&module).is_ok()); module .types .push(ModuleType::Function(SignatureIndex::new(0))); assert_eq!( - limits.validate_module(&module), + limits.validate(&module).map_err(|e| e.to_string()), Err("defined types count of 1 exceeds the limit of 0".into()) ); } @@ -1147,11 +1141,11 @@ mod test { }; let mut module = Module::default(); - assert_eq!(limits.validate_module(&module), Ok(())); + assert!(limits.validate(&module).is_ok()); module.functions.push(SignatureIndex::new(0)); assert_eq!( - limits.validate_module(&module), + limits.validate(&module).map_err(|e| e.to_string()), Err("defined functions count of 1 exceeds the limit of 0".into()) ); } @@ -1164,7 +1158,7 @@ mod test { }; let mut module = Module::default(); - assert_eq!(limits.validate_module(&module), Ok(())); + assert!(limits.validate(&module).is_ok()); module.table_plans.push(TablePlan { style: TableStyle::CallerChecksSignature, @@ -1176,7 +1170,7 @@ mod test { }, }); assert_eq!( - limits.validate_module(&module), + limits.validate(&module).map_err(|e| e.to_string()), Err("defined tables count of 1 exceeds the limit of 0".into()) ); } @@ -1189,7 +1183,7 @@ mod test { }; let mut module = Module::default(); - assert_eq!(limits.validate_module(&module), Ok(())); + assert!(limits.validate(&module).is_ok()); module.memory_plans.push(MemoryPlan { style: MemoryStyle::Static { bound: 0 }, @@ -1201,7 +1195,7 @@ mod test { offset_guard_size: 0, }); assert_eq!( - limits.validate_module(&module), + limits.validate(&module).map_err(|e| e.to_string()), Err("defined memories count of 1 exceeds the limit of 0".into()) ); } @@ -1214,7 +1208,7 @@ mod test { }; let mut module = Module::default(); - assert_eq!(limits.validate_module(&module), Ok(())); + assert!(limits.validate(&module).is_ok()); module.globals.push(Global { wasm_ty: WasmType::I32, @@ -1223,7 +1217,7 @@ mod test { initializer: GlobalInit::I32Const(0), }); assert_eq!( - limits.validate_module(&module), + limits.validate(&module).map_err(|e| e.to_string()), Err("defined globals count of 1 exceeds the limit of 0".into()) ); } @@ -1247,7 +1241,7 @@ mod test { }, }); assert_eq!( - limits.validate_module(&module), + limits.validate(&module).map_err(|e| e.to_string()), Err( "table index 0 has a minimum element size of 11 which exceeds the limit of 10" .into() @@ -1274,7 +1268,7 @@ mod test { offset_guard_size: 0, }); assert_eq!( - limits.validate_module(&module), + limits.validate(&module).map_err(|e| e.to_string()), Err("memory index 0 has a minimum page size of 6 which exceeds the limit of 5".into()) ); } @@ -1298,7 +1292,7 @@ mod test { offset_guard_size: 0, }); assert_eq!( - limits.validate_module(&module), + limits.validate(&module).map_err(|e| e.to_string()), Err("memory index 0 has an unsupported dynamic memory plan style".into()) ); } @@ -1335,7 +1329,7 @@ mod test { #[cfg(target_pointer_width = "64")] #[test] - fn test_instance_pool() -> Result<(), String> { + fn test_instance_pool() -> Result<()> { let module_limits = ModuleLimits { imported_functions: 0, imported_tables: 0, @@ -1372,13 +1366,7 @@ mod test { assert_eq!(instances.instance_size, 4096); assert_eq!(instances.max_instances, 3); - assert_eq!( - &*instances - .free_list - .lock() - .map_err(|_| "failed to lock".to_string())?, - &[0, 1, 2], - ); + assert_eq!(&*instances.free_list.lock().unwrap(), &[0, 1, 2],); let mut handles = Vec::new(); let module = Arc::new(Module::default()); @@ -1409,13 +1397,7 @@ mod test { ); } - assert_eq!( - &*instances - .free_list - .lock() - .map_err(|_| "failed to lock".to_string())?, - &[], - ); + assert_eq!(&*instances.free_list.lock().unwrap(), &[],); match instances.allocate( PoolingAllocationStrategy::NextAvailable, @@ -1443,20 +1425,14 @@ mod test { instances.deallocate(&handle); } - assert_eq!( - &*instances - .free_list - .lock() - .map_err(|_| "failed to lock".to_string())?, - &[2, 1, 0], - ); + assert_eq!(&*instances.free_list.lock().unwrap(), &[2, 1, 0],); Ok(()) } #[cfg(target_pointer_width = "64")] #[test] - fn test_memory_pool() -> Result<(), String> { + fn test_memory_pool() -> Result<()> { let pool = MemoryPool::new( &ModuleLimits { imported_functions: 0, @@ -1502,7 +1478,7 @@ mod test { #[cfg(target_pointer_width = "64")] #[test] - fn test_table_pool() -> Result<(), String> { + fn test_table_pool() -> Result<()> { let pool = TablePool::new( &ModuleLimits { imported_functions: 0, @@ -1549,7 +1525,7 @@ mod test { #[cfg(all(unix, target_pointer_width = "64"))] #[test] - fn test_stack_pool() -> Result<(), String> { + fn test_stack_pool() -> Result<()> { let pool = StackPool::new( &InstanceLimits { count: 10, @@ -1563,10 +1539,7 @@ mod test { assert_eq!(pool.page_size, 4096); assert_eq!( - &*pool - .free_list - .lock() - .map_err(|_| "failed to lock".to_string())?, + &*pool.free_list.lock().unwrap(), &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9], ); @@ -1581,13 +1554,7 @@ mod test { stacks.push(stack); } - assert_eq!( - &*pool - .free_list - .lock() - .map_err(|_| "failed to lock".to_string())?, - &[], - ); + assert_eq!(&*pool.free_list.lock().unwrap(), &[],); match pool .allocate(PoolingAllocationStrategy::NextAvailable) @@ -1602,10 +1569,7 @@ mod test { } assert_eq!( - &*pool - .free_list - .lock() - .map_err(|_| "failed to lock".to_string())?, + &*pool.free_list.lock().unwrap(), &[9, 8, 7, 6, 5, 4, 3, 2, 1, 0], ); @@ -1624,6 +1588,7 @@ mod test { }, 4096 ) + .map_err(|e| e.to_string()) .expect_err("expected a failure constructing instance allocator"), "the instance count limit cannot be zero" ); @@ -1644,6 +1609,7 @@ mod test { }, 4096 ) + .map_err(|e| e.to_string()) .expect_err("expected a failure constructing instance allocator"), "module memory page limit of 65537 exceeds the maximum of 65536" ); @@ -1664,6 +1630,7 @@ mod test { }, 4096, ) + .map_err(|e| e.to_string()) .expect_err("expected a failure constructing instance allocator"), "module memory page limit of 2 pages exeeds the memory reservation size limit of 65536 bytes" ); @@ -1672,7 +1639,7 @@ mod test { #[cfg_attr(target_arch = "aarch64", ignore)] // https://github.com/bytecodealliance/wasmtime/pull/2518#issuecomment-747280133 #[cfg(all(unix, target_pointer_width = "64"))] #[test] - fn test_stack_zeroed() -> Result<(), String> { + fn test_stack_zeroed() -> Result<()> { let allocator = PoolingInstanceAllocator::new( PoolingAllocationStrategy::NextAvailable, ModuleLimits { @@ -1695,9 +1662,7 @@ mod test { unsafe { for _ in 0..10 { - let stack = allocator - .allocate_fiber_stack() - .map_err(|e| format!("failed to allocate stack: {}", e))?; + let stack = allocator.allocate_fiber_stack()?; // The stack pointer is at the top, so decerement it first let addr = stack.sub(1); diff --git a/crates/runtime/src/instance/allocator/pooling/linux.rs b/crates/runtime/src/instance/allocator/pooling/linux.rs index 4a8a43f08f03..bd62f37dfbbe 100644 --- a/crates/runtime/src/instance/allocator/pooling/linux.rs +++ b/crates/runtime/src/instance/allocator/pooling/linux.rs @@ -1,4 +1,5 @@ use crate::Mmap; +use anyhow::{anyhow, Result}; pub unsafe fn make_accessible(addr: *mut u8, len: usize) -> bool { region::protect(addr, len, region::Protection::READ_WRITE).is_ok() @@ -16,7 +17,7 @@ pub unsafe fn decommit(addr: *mut u8, len: usize) { ); } -pub fn create_memory_map(accessible_size: usize, mapping_size: usize) -> Result { +pub fn create_memory_map(accessible_size: usize, mapping_size: usize) -> Result { Mmap::accessible_reserved(accessible_size, mapping_size) - .map_err(|e| format!("failed to allocate pool memory: {}", e)) + .map_err(|e| anyhow!("failed to allocate pool memory: {}", e)) } diff --git a/crates/runtime/src/instance/allocator/pooling/uffd.rs b/crates/runtime/src/instance/allocator/pooling/uffd.rs index 793b63522fa4..b140a8d2a9ba 100644 --- a/crates/runtime/src/instance/allocator/pooling/uffd.rs +++ b/crates/runtime/src/instance/allocator/pooling/uffd.rs @@ -1,23 +1,40 @@ -//! Implements user space page fault handling with the `userfaultfd` ("uffd") system call on Linux. +//! This module implements user space page fault handling with the `userfaultfd` ("uffd") system call on Linux. //! //! Handling page faults for memory accesses in regions relating to WebAssembly instances -//! enables the implementation of protecting guard pages in user space rather than kernel space. +//! enables the runtime to protect guard pages in user space rather than kernel space (i.e. without `mprotect`). //! -//! This reduces the number of system calls and kernel locks needed to provide correct -//! WebAssembly memory semantics. +//! Additionally, linear memories can be lazy-initialized upon first access. //! -//! Additionally, linear memories can be lazy-initialized upon access. +//! Handling faults in user space is slower than handling faults in the kernel. However, +//! in use cases where there is a high number of concurrently executing instances, handling the faults +//! in user space requires rarely changing memory protection levels. This can improve concurrency +//! by not taking kernel memory manager locks and may decrease TLB shootdowns as fewer page table entries need +//! to continually change. +//! +//! Here's how the `uffd` feature works: +//! +//! 1. A user fault file descriptor is created to monitor specific areas of the address space. +//! 2. A thread is spawned to continually read events from the user fault file descriptor. +//! 3. When a page fault event is received, the handler thread calculates where the fault occurred: +//! a) If the fault occurs on a table page, it is handled by zeroing the page. +//! b) If the fault occurs on a linear memory page, it is handled by either copying the page from +//! initialization data or zeroing it. +//! c) If the fault occurs on a guard page, the protection level of the guard page is changed to +//! force the kernel to signal SIGSEV on the next retry. The faulting page is recorded so the +//! protection level can be reset in the future. +//! 4. Faults to address space relating to an instance may occur from both Wasmtime (e.g. instance +//! initialization) or from WebAssembly code (e.g. reading from or writing to linear memory), +//! therefore the user fault handling must do as little work as possible to handle the fault. +//! 5. When the pooling allocator is dropped, it will drop the memory mappings relating to the pool; this +//! generates unmap events for the fault handling thread, which responds by decrementing the mapping +//! count. When the count reaches zero, the user fault handling thread will gracefully terminate. //! //! This feature requires a Linux kernel 4.11 or newer to use. -use super::{InstancePool, StackPool}; +use super::InstancePool; use crate::{instance::Instance, Mmap}; -use std::convert::TryInto; +use anyhow::{bail, Context, Result}; use std::ptr; -use std::sync::{ - atomic::{AtomicBool, Ordering}, - Arc, -}; use std::thread; use userfaultfd::{Event, FeatureFlags, IoctlFlags, Uffd, UffdBuilder}; use wasmtime_environ::{entity::EntityRef, wasm::DefinedMemoryIndex, MemoryInitialization}; @@ -45,11 +62,11 @@ pub unsafe fn decommit(addr: *mut u8, len: usize) { ); } -pub fn create_memory_map(_accessible_size: usize, mapping_size: usize) -> Result { +pub fn create_memory_map(_accessible_size: usize, mapping_size: usize) -> Result { // Allocate a single read-write region at once // As writable pages need to count towards commit charge, use MAP_NORESERVE to override. - // This implies that the kernel is configured to allow overcommit or else - // this allocation will almost certainly fail without a plethora of physical memory to back the allocation. + // This implies that the kernel is configured to allow overcommit or else this allocation + // will almost certainly fail without a plethora of physical memory to back the allocation. // The consequence of not reserving is that our process may segfault on any write to a memory // page that cannot be backed (i.e. out of memory conditions). @@ -68,10 +85,10 @@ pub fn create_memory_map(_accessible_size: usize, mapping_size: usize) -> Result ); if ptr as isize == -1_isize { - return Err(format!( - "failed to allocate pool memory: {}", + bail!( + "failed to allocate pool memory: mmap failed with {}", std::io::Error::last_os_error() - )); + ); } Ok(Mmap::from_raw(ptr as usize, mapping_size)) @@ -98,22 +115,10 @@ enum AddressLocation<'a> { /// The instance related to the memory page that was accessed. instance: &'a Instance, /// The index of the memory that was accessed. - memory_index: usize, + memory_index: DefinedMemoryIndex, /// The Wasm page index to initialize if the access was not a guard page. page_index: Option, }, - /// The address location is in an execution stack. - /// The fault handler will zero the page. - StackPage { - /// The address of the page being accessed. - page_addr: *mut u8, - /// The length of the page being accessed. - len: usize, - /// The index of the stack that was accessed. - index: usize, - /// Whether or not the access was to a guard page. - guard_page: bool, - }, } /// Used to resolve fault addresses to address locations. @@ -132,22 +137,16 @@ struct AddressLocator { tables_start: usize, tables_end: usize, table_size: usize, - stacks_start: usize, - stacks_end: usize, - stack_size: usize, page_size: usize, } impl AddressLocator { - fn new(instances: &InstancePool, stacks: &StackPool) -> Self { + fn new(instances: &InstancePool) -> Self { let instances_start = instances.mapping.as_ptr() as usize; let memories_start = instances.memories.mapping.as_ptr() as usize; let memories_end = memories_start + instances.memories.mapping.len(); let tables_start = instances.tables.mapping.as_ptr() as usize; let tables_end = tables_start + instances.tables.mapping.len(); - let stacks_start = stacks.mapping.as_ptr() as usize; - let stacks_end = stacks_start + stacks.mapping.len(); - let stack_size = stacks.stack_size; // Should always have instances debug_assert!(instances_start != 0); @@ -163,9 +162,6 @@ impl AddressLocator { tables_start, tables_end, table_size: instances.tables.table_size, - stacks_start, - stacks_end, - stack_size, page_size: instances.tables.page_size, } } @@ -191,25 +187,18 @@ impl AddressLocator { // Check for a memory location if addr >= self.memories_start && addr < self.memories_end { let index = (addr - self.memories_start) / self.memory_size; - let memory_index = index % self.max_memories; + let memory_index = DefinedMemoryIndex::new(index % self.max_memories); let memory_start = self.memories_start + (index * self.memory_size); let page_index = (addr - memory_start) / WASM_PAGE_SIZE; let instance = self.get_instance(index / self.max_memories); - let init_page_index = instance - .memories - .get( - DefinedMemoryIndex::from_u32(memory_index as u32) - .try_into() - .unwrap(), - ) - .and_then(|m| { - if page_index < m.size() as usize { - Some(page_index) - } else { - None - } - }); + let init_page_index = instance.memories.get(memory_index).and_then(|m| { + if page_index < m.size() as usize { + Some(page_index) + } else { + None + } + }); return Some(AddressLocation::MemoryPage { page_addr: (memory_start + page_index * WASM_PAGE_SIZE) as _, @@ -233,128 +222,125 @@ impl AddressLocator { }); } - // Check for a stack location - if addr >= self.stacks_start && addr < self.stacks_end { - let index = (addr - self.stacks_start) / self.stack_size; - let stack_start = self.stacks_start + (index * self.stack_size); - let stack_offset = addr - stack_start; - let page_offset = (stack_offset / self.page_size) * self.page_size; - - return Some(AddressLocation::StackPage { - page_addr: (stack_start + page_offset) as _, - len: self.page_size, - index, - guard_page: stack_offset < self.page_size, - }); - } - None } } -unsafe fn wake_guard_page_access( - uffd: &Uffd, - page_addr: *const u8, - len: usize, -) -> Result<(), String> { - // Set the page to NONE to induce a SIGSEV for the access on the next retry +/// This is called following a fault on a guard page. +/// +/// Because the region being monitored is protected read-write, this needs to set the +/// protection level to `NONE` before waking the page. +/// +/// This will cause the kernel to raise a SIGSEGV when retrying the fault. +unsafe fn wake_guard_page_access(uffd: &Uffd, page_addr: *const u8, len: usize) -> Result<()> { + // Set the page to NONE to induce a SIGSEGV for the access on the next retry region::protect(page_addr, len, region::Protection::NONE) - .map_err(|e| format!("failed to change guard page protection: {}", e))?; + .context("failed to change guard page protection")?; - uffd.wake(page_addr as _, len).map_err(|e| { - format!( - "failed to wake page at {:p} with length {}: {}", - page_addr, len, e - ) - })?; + uffd.wake(page_addr as _, len) + .context("failed to wake guard page access")?; Ok(()) } +/// This is called to initialize a linear memory page (64 KiB). +/// +/// If paged initialization is used for the module, then we can instruct the kernel to back the page with +/// what is already stored in the initialization data; if the page isn't in the initialization data, +/// it will be zeroed instead. +/// +/// If paged initialization isn't being used, we zero the page. Initialization happens +/// at module instantiation in this case and the segment data will be then copied to the zeroed page. unsafe fn initialize_wasm_page( uffd: &Uffd, instance: &Instance, page_addr: *const u8, - memory_index: usize, + memory_index: DefinedMemoryIndex, page_index: usize, -) -> Result<(), String> { - if let Some(MemoryInitialization::Paged { page_size, map }) = - &instance.module.memory_initialization - { - let memory_index = DefinedMemoryIndex::new(memory_index); - let memory = instance.memory(memory_index); +) -> Result<()> { + // Check for paged initialization and copy the page if present in the initialization data + if let MemoryInitialization::Paged { map, .. } = &instance.module.memory_initialization { let pages = &map[memory_index]; - debug_assert_eq!(WASM_PAGE_SIZE % page_size, 0); - - let count = WASM_PAGE_SIZE / page_size; - let start = page_index * count; - - for i in start..start + count { - let dst = memory.base.add(i * page_size); - - match pages.get(i) { - Some(Some(data)) => { - log::trace!( - "copying page initialization data from {:p} to {:p} with length {}", - data, - dst, - page_size - ); - - // Copy the page data without waking - uffd.copy(data.as_ptr() as _, dst as _, *page_size, false) - .map_err(|e| { - format!( - "failed to copy page from {:p} to {:p} with length {}: {}", - data, dst, page_size, e - ) - })?; - } - _ => { - log::trace!("zeroing page at {:p} with length {}", dst, page_size); - - // No data, zero the page without waking - uffd.zeropage(dst as _, *page_size, false).map_err(|e| { - format!( - "failed to zero page at {:p} with length {}: {}", - dst, page_size, e - ) - })?; - } - } + + if let Some(Some(data)) = pages.get(page_index) { + debug_assert_eq!(data.len(), WASM_PAGE_SIZE); + + log::trace!( + "copying linear memory page from {:p} to {:p}", + data.as_ptr(), + page_addr + ); + + uffd.copy(data.as_ptr() as _, page_addr as _, WASM_PAGE_SIZE, true) + .context("failed to copy linear memory page")?; + + return Ok(()); } + } - // Finally wake the entire wasm page - uffd.wake(page_addr as _, WASM_PAGE_SIZE).map_err(|e| { - format!( - "failed to wake page at {:p} with length {}: {}", - page_addr, WASM_PAGE_SIZE, e - ) - }) - } else { - log::trace!( - "initialization data is not paged; zeroing Wasm page at {:p}", - page_addr - ); + log::trace!("zeroing linear memory page at {:p}", page_addr); + + uffd.zeropage(page_addr as _, WASM_PAGE_SIZE, true) + .context("failed to zero linear memory page")?; - uffd.zeropage(page_addr as _, WASM_PAGE_SIZE, true) - .map_err(|e| { - format!( - "failed to zero page at {:p} with length {}: {}", - page_addr, WASM_PAGE_SIZE, e - ) - })?; + Ok(()) +} - Ok(()) +unsafe fn handle_page_fault( + uffd: &Uffd, + locator: &AddressLocator, + addr: *mut std::ffi::c_void, +) -> Result<()> { + match locator.get_location(addr as usize) { + Some(AddressLocation::TablePage { page_addr, len }) => { + log::trace!( + "handling fault in table at address {:p} on page {:p}", + addr, + page_addr, + ); + + // Tables are always initialized upon instantiation, so zero the page + uffd.zeropage(page_addr as _, len, true) + .context("failed to zero table page")?; + } + Some(AddressLocation::MemoryPage { + page_addr, + len, + instance, + memory_index, + page_index, + }) => { + log::trace!( + "handling fault in linear memory at address {:p} on page {:p}", + addr, + page_addr + ); + + match page_index { + Some(page_index) => { + initialize_wasm_page(&uffd, instance, page_addr, memory_index, page_index)?; + } + None => { + log::trace!("out of bounds memory access at {:p}", addr); + + // Record the guard page fault with the instance so it can be reset later. + instance.record_guard_page_fault(page_addr, len, reset_guard_page); + wake_guard_page_access(&uffd, page_addr, len)?; + } + } + } + None => { + bail!( + "failed to locate fault address {:p} in registered memory regions", + addr + ); + } } + + Ok(()) } -fn handler_thread( - uffd: Uffd, - locator: AddressLocator, - mut registrations: usize, - faulted_stack_guard_pages: Arc<[AtomicBool]>, -) -> Result<(), String> { +fn handler_thread(uffd: Uffd, locator: AddressLocator, mut registrations: usize) -> Result<()> { loop { match uffd.read_event().expect("failed to read event") { Some(Event::Unmap { start, end }) => { @@ -364,7 +350,6 @@ fn handler_thread( if (start == locator.memories_start && end == locator.memories_end) || (start == locator.tables_start && end == locator.tables_end) - || (start == locator.stacks_start && end == locator.stacks_end) { registrations -= 1; if registrations == 0 { @@ -374,104 +359,11 @@ fn handler_thread( panic!("unexpected memory region unmapped"); } } - Some(Event::Pagefault { - addr: access_addr, .. - }) => { - unsafe { - match locator.get_location(access_addr as usize) { - Some(AddressLocation::TablePage { page_addr, len }) => { - log::trace!( - "handling fault in table at address {:p} on page {:p}", - access_addr, - page_addr, - ); - - // Tables are always initialized upon instantiation, so zero the page - uffd.zeropage(page_addr as _, len, true).map_err(|e| { - format!( - "failed to zero page at {:p} with length {}: {}", - page_addr, len, e - ) - })?; - } - Some(AddressLocation::MemoryPage { - page_addr, - len, - instance, - memory_index, - page_index, - }) => { - log::trace!( - "handling fault in linear memory at address {:p} on page {:p}", - access_addr, - page_addr - ); - - match page_index { - Some(page_index) => { - initialize_wasm_page( - &uffd, - instance, - page_addr, - memory_index, - page_index, - )?; - } - None => { - log::trace!("out of bounds memory access at {:p}", access_addr); - - // Record the guard page fault with the instance so it can be reset later. - instance.record_guard_page_fault( - page_addr, - len, - reset_guard_page, - ); - wake_guard_page_access(&uffd, page_addr, len)?; - } - } - } - Some(AddressLocation::StackPage { - page_addr, - len, - index, - guard_page, - }) => { - log::trace!( - "handling fault in stack {} at address {:p}", - index, - access_addr, - ); - - if guard_page { - // Logging as trace as stack guard pages might be a trap condition in the future - log::trace!("stack overflow fault at {:p}", access_addr); - - // Mark the stack as having a faulted guard page - // The next time the stack is used the guard page will be reset - faulted_stack_guard_pages[index].store(true, Ordering::SeqCst); - wake_guard_page_access(&uffd, page_addr, len)?; - continue; - } - - // Always zero stack pages - uffd.zeropage(page_addr as _, len, true).map_err(|e| { - format!( - "failed to zero page at {:p} with length {}: {}", - page_addr, len, e - ) - })?; - } - None => { - return Err(format!( - "failed to locate fault address {:p} in registered memory regions", - access_addr - )); - } - } - } - } + Some(Event::Pagefault { addr, .. }) => unsafe { + handle_page_fault(&uffd, &locator, addr as _)? + }, Some(_) => continue, - None => break, + None => bail!("no event was read from the user fault descriptor"), } } @@ -482,16 +374,16 @@ fn handler_thread( #[derive(Debug)] pub struct PageFaultHandler { - thread: Option>>, + thread: Option>>, } impl PageFaultHandler { - pub(super) fn new(instances: &InstancePool, stacks: &StackPool) -> Result { + pub(super) fn new(instances: &InstancePool) -> Result { let uffd = UffdBuilder::new() .close_on_exec(true) .require_features(FeatureFlags::EVENT_UNMAP) .create() - .map_err(|e| format!("failed to create user fault descriptor: {}", e))?; + .context("failed to create user fault descriptor")?; // Register the ranges with the userfault fd let mut registrations = 0; @@ -504,7 +396,6 @@ impl PageFaultHandler { instances.tables.mapping.as_ptr() as usize, instances.tables.mapping.len(), ), - (stacks.mapping.as_ptr() as usize, stacks.mapping.len()), ] { if *start == 0 || *len == 0 { continue; @@ -512,13 +403,13 @@ impl PageFaultHandler { let ioctls = uffd .register(*start as _, *len) - .map_err(|e| format!("failed to register user fault range: {}", e))?; + .context("failed to register user fault range")?; if !ioctls.contains(IoctlFlags::WAKE | IoctlFlags::COPY | IoctlFlags::ZEROPAGE) { - return Err(format!( + bail!( "required user fault ioctls not supported; found: {:?}", ioctls, - )); + ); } registrations += 1; @@ -533,17 +424,13 @@ impl PageFaultHandler { registrations ); - let locator = AddressLocator::new(&instances, &stacks); - - let faulted_stack_guard_pages = stacks.faulted_guard_pages.clone(); + let locator = AddressLocator::new(&instances); Some( thread::Builder::new() .name("page fault handler".into()) - .spawn(move || { - handler_thread(uffd, locator, registrations, faulted_stack_guard_pages) - }) - .map_err(|e| format!("failed to spawn page fault handler thread: {}", e))?, + .spawn(move || handler_thread(uffd, locator, registrations)) + .context("failed to spawn page fault handler thread")?, ) }; @@ -553,6 +440,9 @@ impl PageFaultHandler { impl Drop for PageFaultHandler { fn drop(&mut self) { + // The handler thread should terminate once all monitored regions of memory are unmapped. + // The pooling instance allocator ensures that the regions are unmapped prior to dropping + // the user fault handler. if let Some(thread) = self.thread.take() { thread .join() @@ -569,6 +459,7 @@ mod test { table::max_table_element_size, Imports, InstanceAllocationRequest, InstanceLimits, ModuleLimits, PoolingAllocationStrategy, VMSharedSignatureIndex, }; + use std::sync::Arc; use wasmtime_environ::{ entity::PrimaryMap, wasm::{Memory, Table, TableElementType, WasmType}, @@ -598,9 +489,8 @@ mod test { let instances = InstancePool::new(&module_limits, &instance_limits).expect("should allocate"); - let stacks = StackPool::new(&instance_limits, 8192).expect("should allocate"); - let locator = AddressLocator::new(&instances, &stacks); + let locator = AddressLocator::new(&instances); assert_eq!(locator.instances_start, instances.mapping.as_ptr() as usize); assert_eq!(locator.instance_size, 4096); @@ -625,20 +515,10 @@ mod test { ); assert_eq!(locator.table_size, 8192); - assert_eq!(locator.stacks_start, stacks.mapping.as_ptr() as usize); - assert_eq!( - locator.stacks_end, - locator.stacks_start + stacks.mapping.len() - ); - assert_eq!(locator.stack_size, 12288); - unsafe { assert!(locator.get_location(0).is_none()); assert!(locator - .get_location(std::cmp::max( - locator.memories_end, - std::cmp::max(locator.tables_end, locator.stacks_end) - )) + .get_location(std::cmp::max(locator.memories_end, locator.tables_end)) .is_none()); let mut module = Module::new(); @@ -667,9 +547,7 @@ mod test { }); } - module_limits - .validate_module(&module) - .expect("should validate"); + module_limits.validate(&module).expect("should validate"); let mut handles = Vec::new(); let module = Arc::new(module); @@ -719,7 +597,7 @@ mod test { }) => { assert_eq!(page_addr, memory_start as _); assert_eq!(len, WASM_PAGE_SIZE); - assert_eq!(mem_index, memory_index); + assert_eq!(mem_index, DefinedMemoryIndex::new(memory_index)); assert_eq!(page_index, Some(0)); } _ => panic!("expected a memory page location"), @@ -736,7 +614,7 @@ mod test { }) => { assert_eq!(page_addr, (memory_start + WASM_PAGE_SIZE) as _); assert_eq!(len, WASM_PAGE_SIZE); - assert_eq!(mem_index, memory_index); + assert_eq!(mem_index, DefinedMemoryIndex::new(memory_index)); assert_eq!(page_index, Some(1)); } _ => panic!("expected a memory page location"), @@ -753,7 +631,7 @@ mod test { }) => { assert_eq!(page_addr, (memory_start + (9 * WASM_PAGE_SIZE)) as _); assert_eq!(len, WASM_PAGE_SIZE); - assert_eq!(mem_index, memory_index); + assert_eq!(mem_index, DefinedMemoryIndex::new(memory_index)); assert_eq!(page_index, None); } _ => panic!("expected a memory page location"), @@ -788,43 +666,6 @@ mod test { } } - // Validate stack locations - for stack_index in 0..instances.max_instances { - let stack_start = locator.stacks_start + (stack_index * locator.stack_size); - - // Check for stack page location - match locator.get_location(stack_start + locator.page_size * 2) { - Some(AddressLocation::StackPage { - page_addr, - len, - index, - guard_page, - }) => { - assert_eq!(page_addr, (stack_start + locator.page_size * 2) as _); - assert_eq!(len, locator.page_size); - assert_eq!(index, stack_index); - assert!(!guard_page); - } - _ => panic!("expected a stack page location"), - } - - // Check for guard page - match locator.get_location(stack_start) { - Some(AddressLocation::StackPage { - page_addr, - len, - index, - guard_page, - }) => { - assert_eq!(page_addr, stack_start as _); - assert_eq!(len, locator.page_size); - assert_eq!(index, stack_index); - assert!(guard_page); - } - _ => panic!("expected a stack page location"), - } - } - for handle in handles.drain(..) { instances.deallocate(&handle); } diff --git a/crates/runtime/src/instance/allocator/pooling/unix.rs b/crates/runtime/src/instance/allocator/pooling/unix.rs index 900e73d1747e..9cc68b3361e7 100644 --- a/crates/runtime/src/instance/allocator/pooling/unix.rs +++ b/crates/runtime/src/instance/allocator/pooling/unix.rs @@ -1,4 +1,5 @@ use crate::Mmap; +use anyhow::{anyhow, Result}; pub unsafe fn make_accessible(addr: *mut u8, len: usize) -> bool { region::protect(addr, len, region::Protection::READ_WRITE).is_ok() @@ -20,7 +21,7 @@ pub unsafe fn decommit(addr: *mut u8, len: usize) { ); } -pub fn create_memory_map(accessible_size: usize, mapping_size: usize) -> Result { +pub fn create_memory_map(accessible_size: usize, mapping_size: usize) -> Result { Mmap::accessible_reserved(accessible_size, mapping_size) - .map_err(|e| format!("failed to allocate pool memory: {}", e)) + .map_err(|e| anyhow!("failed to allocate pool memory: {}", e)) } diff --git a/crates/runtime/src/instance/allocator/pooling/windows.rs b/crates/runtime/src/instance/allocator/pooling/windows.rs index fe8566558ae0..159f00b63fa7 100644 --- a/crates/runtime/src/instance/allocator/pooling/windows.rs +++ b/crates/runtime/src/instance/allocator/pooling/windows.rs @@ -1,4 +1,5 @@ use crate::Mmap; +use anyhow::{anyhow, Result}; use winapi::um::memoryapi::{VirtualAlloc, VirtualFree}; use winapi::um::winnt::{MEM_COMMIT, MEM_DECOMMIT, PAGE_READWRITE}; @@ -15,7 +16,7 @@ pub unsafe fn decommit(addr: *mut u8, len: usize) { ); } -pub fn create_memory_map(accessible_size: usize, mapping_size: usize) -> Result { +pub fn create_memory_map(accessible_size: usize, mapping_size: usize) -> Result { Mmap::accessible_reserved(accessible_size, mapping_size) - .map_err(|e| format!("failed to allocate pool memory: {}", e)) + .map_err(|e| anyhow!("failed to allocate pool memory: {}", e)) } diff --git a/crates/wasmtime/src/config.rs b/crates/wasmtime/src/config.rs index 77d5c52be789..30e63546e867 100644 --- a/crates/wasmtime/src/config.rs +++ b/crates/wasmtime/src/config.rs @@ -627,15 +627,12 @@ impl Config { #[cfg(not(feature = "async"))] let stack_size = 0; - Some(Arc::new( - PoolingInstanceAllocator::new( - strategy, - module_limits, - instance_limits, - stack_size, - ) - .map_err(|e| anyhow::anyhow!(e))?, - )) + Some(Arc::new(PoolingInstanceAllocator::new( + strategy, + module_limits, + instance_limits, + stack_size, + )?)) } }; Ok(self) diff --git a/crates/wasmtime/src/module.rs b/crates/wasmtime/src/module.rs index a9d32ee1d17d..f87b28f6fe49 100644 --- a/crates/wasmtime/src/module.rs +++ b/crates/wasmtime/src/module.rs @@ -307,22 +307,36 @@ impl Module { /// # } /// ``` pub fn from_binary(engine: &Engine, binary: &[u8]) -> Result { - // Check with the instance allocator to see if the given module is supported - let allocator = engine.config().instance_allocator(); - - #[cfg(feature = "cache")] - let (main_module, artifacts, types) = ModuleCacheEntry::new( - "wasmtime", - engine.cache_config(), - ) - .get_data((engine.compiler(), binary), |(compiler, binary)| { - CompilationArtifacts::build(compiler, binary, |m| allocator.validate_module(m)) - })?; - #[cfg(not(feature = "cache"))] - let (main_module, artifacts, types) = - CompilationArtifacts::build(engine.compiler(), binary, |m| { - allocator.validate_module(m) - })?; + cfg_if::cfg_if! { + if #[cfg(feature = "cache")] { + let (main_module, artifacts, types) = ModuleCacheEntry::new( + "wasmtime", + engine.cache_config(), + ) + .get_data((engine.compiler(), binary), |(compiler, binary)| { + cfg_if::cfg_if! { + if #[cfg(all(feature = "uffd", target_os = "linux"))] { + let use_paged_mem_init = true; + } else { + let use_paged_mem_init = false; + } + }; + + CompilationArtifacts::build(compiler, binary, use_paged_mem_init) + })?; + } else { + cfg_if::cfg_if! { + if #[cfg(all(feature = "uffd", target_os = "linux"))] { + let use_paged_mem_init = true; + } else { + let use_paged_mem_init = false; + } + }; + + let (main_module, artifacts, types) = + CompilationArtifacts::build(engine.compiler(), binary, use_paged_mem_init)?; + } + }; let mut modules = CompiledModule::from_artifacts_list( artifacts, @@ -331,6 +345,12 @@ impl Module { )?; let module = modules.remove(main_module); + // Validate the module can be used with the current allocator + engine + .config() + .instance_allocator() + .validate(module.module())?; + Ok(Module { inner: Arc::new(ModuleInner { engine: engine.clone(), diff --git a/tests/all/pooling_allocator.rs b/tests/all/pooling_allocator.rs index d7b7101fee49..bdad287aa90e 100644 --- a/tests/all/pooling_allocator.rs +++ b/tests/all/pooling_allocator.rs @@ -48,7 +48,10 @@ fn memory_limit() -> Result<()> { // Module should fail to validate because the minimum is greater than the configured limit match Module::new(&engine, r#"(module (memory 4))"#) { Ok(_) => panic!("module compilation should fail"), - Err(e) => assert_eq!(e.to_string(), "Validation error: memory index 0 has a minimum page size of 4 which exceeds the limit of 3") + Err(e) => assert_eq!( + e.to_string(), + "memory index 0 has a minimum page size of 4 which exceeds the limit of 3" + ), } let module = Module::new( @@ -243,7 +246,10 @@ fn table_limit() -> Result<()> { // Module should fail to validate because the minimum is greater than the configured limit match Module::new(&engine, r#"(module (table 31 funcref))"#) { Ok(_) => panic!("module compilation should fail"), - Err(e) => assert_eq!(e.to_string(), "Validation error: table index 0 has a minimum element size of 31 which exceeds the limit of 10") + Err(e) => assert_eq!( + e.to_string(), + "table index 0 has a minimum element size of 31 which exceeds the limit of 10" + ), } let module = Module::new( From ff840b3d3b3a53d9b9fc20f1d3f54c9763eebdde Mon Sep 17 00:00:00 2001 From: Peter Huene Date: Thu, 4 Mar 2021 14:01:42 -0800 Subject: [PATCH 24/33] More PR feedback changes. * More use of `anyhow`. * Change `make_accessible` into `protect_linear_memory` to better demonstrate what it is used for; this will make the uffd implementation make a little more sense. * Remove `create_memory_map` in favor of just creating the `Mmap` instances in the pooling allocator. This also removes the need for `MAP_NORESERVE` in the uffd implementation. * Moar comments. * Remove `BasePointerIterator` in favor of `impl Iterator`. * The uffd implementation now only monitors linear memory pages and will only receive faults on pages that could potentially be accessible and never on a statically known guard page. * Stop allocating memory or table pools if the maximum limit of the memory or table is 0. --- crates/jit/src/code_memory.rs | 2 +- crates/runtime/src/instance.rs | 14 +- crates/runtime/src/instance/allocator.rs | 6 +- .../runtime/src/instance/allocator/pooling.rs | 350 ++++++++---------- .../src/instance/allocator/pooling/linux.rs | 67 +++- .../src/instance/allocator/pooling/uffd.rs | 340 +++++++---------- .../src/instance/allocator/pooling/unix.rs | 69 +++- .../src/instance/allocator/pooling/windows.rs | 61 ++- crates/runtime/src/memory.rs | 30 +- crates/runtime/src/mmap.rs | 34 +- crates/wasmtime/src/trampoline/memory.rs | 5 +- 11 files changed, 481 insertions(+), 497 deletions(-) diff --git a/crates/jit/src/code_memory.rs b/crates/jit/src/code_memory.rs index f4ba5c30175d..49a2d0ecd4c6 100644 --- a/crates/jit/src/code_memory.rs +++ b/crates/jit/src/code_memory.rs @@ -25,7 +25,7 @@ struct CodeMemoryEntry { impl CodeMemoryEntry { fn with_capacity(cap: usize) -> Result { - let mmap = ManuallyDrop::new(Mmap::with_at_least(cap)?); + let mmap = ManuallyDrop::new(Mmap::with_at_least(cap).map_err(|e| e.to_string())?); let registry = ManuallyDrop::new(UnwindRegistry::new(mmap.as_ptr() as usize)); Ok(Self { mmap, diff --git a/crates/runtime/src/instance.rs b/crates/runtime/src/instance.rs index 313642a077ab..7c06c6121193 100644 --- a/crates/runtime/src/instance.rs +++ b/crates/runtime/src/instance.rs @@ -67,10 +67,10 @@ pub(crate) struct Instance { /// Hosts can store arbitrary per-instance information here. host_state: Box, - /// Stores guard page faults in memory relating to the instance. - /// This is used for the pooling allocator with uffd enabled on Linux. + /// Stores linear memory guard page faults for the pooling allocator with uffd enabled. + /// These pages need to be reset after the signal handler generates the out-of-bounds trap. #[cfg(all(feature = "uffd", target_os = "linux"))] - guard_page_faults: RefCell bool)>>, + guard_page_faults: RefCell anyhow::Result<()>)>>, /// Additional context used by compiled wasm code. This field is last, and /// represents a dynamically-sized array that extends beyond the nominal @@ -821,7 +821,7 @@ impl Instance { &self, page_addr: *mut u8, size: usize, - reset: unsafe fn(*mut u8, usize) -> bool, + reset: fn(*mut u8, usize) -> anyhow::Result<()>, ) { self.guard_page_faults .borrow_mut() @@ -837,11 +837,7 @@ impl Instance { pub(crate) fn reset_guard_pages(&self) -> anyhow::Result<()> { let mut faults = self.guard_page_faults.borrow_mut(); for (addr, len, reset) in faults.drain(..) { - unsafe { - if !reset(addr, len) { - anyhow::bail!("failed to reset previously faulted memory guard page"); - } - } + reset(addr, len)?; } Ok(()) diff --git a/crates/runtime/src/instance/allocator.rs b/crates/runtime/src/instance/allocator.rs index c835c144216a..c70105dfb528 100644 --- a/crates/runtime/src/instance/allocator.rs +++ b/crates/runtime/src/instance/allocator.rs @@ -549,8 +549,10 @@ impl OnDemandInstanceAllocator { let mut memories: PrimaryMap = PrimaryMap::with_capacity(module.memory_plans.len() - num_imports); for plan in &module.memory_plans.values().as_slice()[num_imports..] { - memories - .push(Memory::new_dynamic(plan, creator).map_err(InstantiationError::Resource)?); + memories.push( + Memory::new_dynamic(plan, creator) + .map_err(|e| InstantiationError::Resource(e.to_string()))?, + ); } Ok(memories) } diff --git a/crates/runtime/src/instance/allocator/pooling.rs b/crates/runtime/src/instance/allocator/pooling.rs index 5029d90667b9..c9e054cbc1ae 100644 --- a/crates/runtime/src/instance/allocator/pooling.rs +++ b/crates/runtime/src/instance/allocator/pooling.rs @@ -31,9 +31,7 @@ cfg_if::cfg_if! { } else if #[cfg(all(feature = "uffd", target_os = "linux"))] { mod uffd; use uffd as imp; - use imp::PageFaultHandler; - use super::{check_init_bounds, initialize_tables}; - use wasmtime_environ::MemoryInitialization; + use imp::initialize_memory_pool; } else if #[cfg(target_os = "linux")] { mod linux; use linux as imp; @@ -43,7 +41,10 @@ cfg_if::cfg_if! { } } -use imp::{create_memory_map, decommit, make_accessible}; +use imp::{ + commit_memory_pages, commit_stack_pages, commit_table_pages, decommit_memory_pages, + decommit_stack_pages, decommit_table_pages, +}; fn round_up_to_pow2(n: usize, to: usize) -> usize { debug_assert!(to > 0); @@ -272,9 +273,9 @@ impl Default for InstanceLimits { Self { count: 1000, #[cfg(target_pointer_width = "32")] - memory_reservation_size: 0xA00000, + memory_reservation_size: 10 * (1 << 20), // 10 MiB, #[cfg(target_pointer_width = "64")] - memory_reservation_size: 0x180000000, + memory_reservation_size: 6 * (1 << 30), // 6 GiB, } } } @@ -305,45 +306,6 @@ impl Default for PoolingAllocationStrategy { } } -// Used to iterate the base address of instance memories and tables. -struct BasePointerIterator { - base: *mut u8, - current: usize, - num: usize, - size: usize, -} - -impl BasePointerIterator { - fn new(base: *mut u8, num: usize, size: usize) -> Self { - Self { - base, - current: 0, - num, - size, - } - } -} - -impl Iterator for BasePointerIterator { - type Item = *mut u8; - - fn next(&mut self) -> Option { - let current = self.current; - if current == self.num { - return None; - } - - self.current += 1; - - Some(unsafe { self.base.add(current * self.size) }) - } - - fn size_hint(&self) -> (usize, Option) { - let remaining = self.num - self.current; - (remaining, Some(remaining)) - } -} - /// Represents a pool of maximal `Instance` structures. /// /// Each index in the pool provides enough space for a maximal `Instance` @@ -395,8 +357,11 @@ impl InstancePool { .checked_mul(max_instances) .ok_or_else(|| anyhow!("total size of instance data exceeds addressable memory"))?; + let mapping = Mmap::accessible_reserved(allocation_size, allocation_size) + .context("failed to create instance pool mapping")?; + let pool = Self { - mapping: create_memory_map(allocation_size, allocation_size)?, + mapping, offsets, instance_size, max_instances, @@ -414,15 +379,18 @@ impl InstancePool { Ok(pool) } - fn initialize(&self, index: usize, module: &Arc) { + unsafe fn instance(&self, index: usize) -> &mut Instance { debug_assert!(index < self.max_instances); + &mut *(self.mapping.as_mut_ptr().add(index * self.instance_size) as *mut Instance) + } + fn initialize(&self, index: usize, module: &Arc) { unsafe { - let instance_ptr = self.mapping.as_mut_ptr().add(index * self.instance_size); + let instance = self.instance(index); // Write a default instance with preallocated memory/table map storage to the ptr std::ptr::write( - instance_ptr as _, + instance as _, Instance { module: module.clone(), offsets: self.offsets, @@ -456,9 +424,7 @@ impl InstancePool { let host_state = std::mem::replace(&mut req.host_state, Box::new(())); unsafe { - debug_assert!(index < self.max_instances); - let instance = - &mut *(self.mapping.as_mut_ptr().add(index * self.instance_size) as *mut Instance); + let instance = self.instance(index); instance.module = req.module.clone(); instance.offsets = VMOffsets::new( @@ -490,47 +456,40 @@ impl InstancePool { let index = (addr - base) / self.instance_size; debug_assert!(index < self.max_instances); - unsafe { - // Decommit any linear memories that were used - for (mem, base) in (*handle.instance) - .memories - .values() - .zip(self.memories.get(index)) - { - let size = (mem.size() * WASM_PAGE_SIZE) as usize; - if size > 0 { - decommit(base, size); - } - } + let instance = unsafe { &mut *handle.instance }; - // Decommit any tables that were used - let table_element_size = max_table_element_size(); - for (table, base) in (*handle.instance) - .tables - .values() - .zip(self.tables.get(index)) - { - let size = round_up_to_pow2( - table.size() as usize * table_element_size, - self.tables.page_size, - ); - if size > 0 { - decommit(base, size); - } - } - - // Drop the host state - (*handle.instance).host_state = Box::new(()); + // Decommit any linear memories that were used + for (mem, base) in instance.memories.values().zip(self.memories.get(index)) { + let size = (mem.size() * WASM_PAGE_SIZE) as usize; + decommit_memory_pages(base, size).unwrap(); } - { - self.free_list.lock().unwrap().push(index); + instance.memories.clear(); + instance.dropped_data.borrow_mut().clear(); + + // Decommit any tables that were used + let table_element_size = max_table_element_size(); + for (table, base) in instance.tables.values().zip(self.tables.get(index)) { + let size = round_up_to_pow2( + table.size() as usize * table_element_size, + self.tables.page_size, + ); + + decommit_table_pages(base, size).unwrap(); } + + instance.tables.clear(); + instance.dropped_elements.borrow_mut().clear(); + + // Drop any host state + instance.host_state = Box::new(()); + + self.free_list.lock().unwrap().push(index); } fn set_instance_memories( instance: &mut Instance, - mut memories: BasePointerIterator, + mut memories: impl Iterator, max_pages: u32, ) -> Result<(), InstantiationError> { let module = instance.module.as_ref(); @@ -541,19 +500,24 @@ impl InstancePool { .reset_guard_pages() .map_err(|e| InstantiationError::Resource(e.to_string()))?; - instance.memories.clear(); + debug_assert!(instance.memories.is_empty()); for plan in (&module.memory_plans.values().as_slice()[module.num_imported_memories..]).iter() { instance.memories.push( - Memory::new_static(plan, memories.next().unwrap(), max_pages, make_accessible) - .map_err(InstantiationError::Resource)?, + Memory::new_static( + plan, + memories.next().unwrap(), + max_pages, + commit_memory_pages, + ) + .map_err(|e| InstantiationError::Resource(e.to_string()))?, ); } let mut dropped_data = instance.dropped_data.borrow_mut(); - dropped_data.clear(); + debug_assert!(dropped_data.is_empty()); dropped_data.resize(module.passive_data.len()); Ok(()) @@ -561,22 +525,18 @@ impl InstancePool { fn set_instance_tables( instance: &mut Instance, - mut tables: BasePointerIterator, + mut tables: impl Iterator, max_elements: u32, ) -> Result<(), InstantiationError> { let module = instance.module.as_ref(); - instance.tables.clear(); + debug_assert!(instance.tables.is_empty()); for plan in (&module.table_plans.values().as_slice()[module.num_imported_tables..]).iter() { let base = tables.next().unwrap(); - // Make the table data accessible - if unsafe { !make_accessible(base, max_elements as usize * max_table_element_size()) } { - return Err(InstantiationError::Resource( - "failed to make instance memory accessible".into(), - )); - } + commit_table_pages(base, max_elements as usize * max_table_element_size()) + .map_err(|e| InstantiationError::Resource(e.to_string()))?; instance .tables @@ -584,7 +544,7 @@ impl InstancePool { } let mut dropped_elements = instance.dropped_elements.borrow_mut(); - dropped_elements.clear(); + debug_assert!(dropped_elements.is_empty()); dropped_elements.resize(module.passive_elements.len()); Ok(()) @@ -623,8 +583,31 @@ struct MemoryPool { impl MemoryPool { fn new(module_limits: &ModuleLimits, instance_limits: &InstanceLimits) -> Result { - let memory_size = usize::try_from(instance_limits.memory_reservation_size) - .map_err(|_| anyhow!("memory reservation size exceeds addressable memory"))?; + // The maximum module memory page count cannot exceed 65536 pages + if module_limits.memory_pages > 0x10000 { + bail!( + "module memory page limit of {} exceeds the maximum of 65536", + module_limits.memory_pages + ); + } + + // The maximum module memory page count cannot exceed the memory reservation size + if (module_limits.memory_pages * WASM_PAGE_SIZE) as u64 + > instance_limits.memory_reservation_size + { + bail!( + "module memory page limit of {} pages exceeds the memory reservation size limit of {} bytes", + module_limits.memory_pages, + instance_limits.memory_reservation_size + ); + } + + let memory_size = if module_limits.memory_pages > 0 { + usize::try_from(instance_limits.memory_reservation_size) + .map_err(|_| anyhow!("memory reservation size exceeds addressable memory"))? + } else { + 0 + }; debug_assert!( memory_size % region::page::size() == 0, @@ -642,25 +625,36 @@ impl MemoryPool { anyhow!("total size of memory reservation exceeds addressable memory") })?; - Ok(Self { - mapping: create_memory_map(0, allocation_size)?, + // Create a completely inaccessible region to start + let mapping = Mmap::accessible_reserved(0, allocation_size) + .context("failed to create memory pool mapping")?; + + let pool = Self { + mapping, memory_size, max_memories, max_instances, max_wasm_pages: module_limits.memory_pages, - }) + }; + + // uffd support requires some special setup for the memory pool + #[cfg(all(feature = "uffd", target_os = "linux"))] + initialize_memory_pool(&pool)?; + + Ok(pool) } - fn get(&self, instance_index: usize) -> BasePointerIterator { + fn get(&self, instance_index: usize) -> impl Iterator { debug_assert!(instance_index < self.max_instances); - let base = unsafe { + let base: *mut u8 = unsafe { self.mapping .as_mut_ptr() .add(instance_index * self.memory_size * self.max_memories) as _ }; - BasePointerIterator::new(base, self.max_memories, self.memory_size) + let size = self.memory_size; + (0..self.max_memories).map(move |i| unsafe { base.add(i * size) }) } } @@ -668,9 +662,6 @@ impl MemoryPool { /// /// Each instance index into the pool returns an iterator over the base addresses /// of the instance's tables. -/// -/// The userfault handler relies on how tables are stored in the mapping, -/// so make sure the uffd implementation is kept up-to-date. #[derive(Debug)] struct TablePool { mapping: Mmap, @@ -685,12 +676,16 @@ impl TablePool { fn new(module_limits: &ModuleLimits, instance_limits: &InstanceLimits) -> Result { let page_size = region::page::size(); - let table_size = round_up_to_pow2( - max_table_element_size() - .checked_mul(module_limits.table_elements as usize) - .ok_or_else(|| anyhow!("table size exceeds addressable memory"))?, - page_size, - ); + let table_size = if module_limits.table_elements > 0 { + round_up_to_pow2( + max_table_element_size() + .checked_mul(module_limits.table_elements as usize) + .ok_or_else(|| anyhow!("table size exceeds addressable memory"))?, + page_size, + ) + } else { + 0 + }; let max_instances = instance_limits.count as usize; let max_tables = module_limits.tables as usize; @@ -700,26 +695,30 @@ impl TablePool { .and_then(|c| c.checked_mul(max_instances)) .ok_or_else(|| anyhow!("total size of instance tables exceeds addressable memory"))?; + let mapping = Mmap::accessible_reserved(allocation_size, allocation_size) + .context("failed to create table pool mapping")?; + Ok(Self { - mapping: create_memory_map(0, allocation_size)?, + mapping, table_size, max_tables, max_instances, - page_size: region::page::size(), + page_size, max_elements: module_limits.table_elements, }) } - fn get(&self, instance_index: usize) -> BasePointerIterator { + fn get(&self, instance_index: usize) -> impl Iterator { debug_assert!(instance_index < self.max_instances); - let base = unsafe { + let base: *mut u8 = unsafe { self.mapping .as_mut_ptr() .add(instance_index * self.table_size * self.max_tables) as _ }; - BasePointerIterator::new(base, self.max_tables, self.table_size) + let size = self.table_size; + (0..self.max_tables).map(move |i| unsafe { base.add(i * size) }) } } @@ -733,9 +732,6 @@ impl TablePool { /// /// The top of the stack (starting stack pointer) is returned when a stack is allocated /// from the pool. -/// -/// The userfault handler relies on how stacks are stored in the mapping, -/// so make sure the uffd implementation is kept up-to-date. #[derive(Debug)] struct StackPool { mapping: Mmap, @@ -765,15 +761,18 @@ impl StackPool { .checked_mul(max_instances) .ok_or_else(|| anyhow!("total size of execution stacks exceeds addressable memory"))?; - let mapping = create_memory_map(allocation_size, allocation_size)?; + let mapping = Mmap::accessible_reserved(allocation_size, allocation_size) + .context("failed to create stack pool mapping")?; // Set up the stack guard pages - unsafe { - for i in 0..max_instances { - // Make the stack guard page inaccessible - let bottom_of_stack = mapping.as_mut_ptr().add(i * stack_size); - region::protect(bottom_of_stack, page_size, region::Protection::NONE) - .context("failed to protect stack guard page")?; + if allocation_size > 0 { + unsafe { + for i in 0..max_instances { + // Make the stack guard page inaccessible + let bottom_of_stack = mapping.as_mut_ptr().add(i * stack_size); + region::protect(bottom_of_stack, page_size, region::Protection::NONE) + .context("failed to protect stack guard page")?; + } } } @@ -804,8 +803,19 @@ impl StackPool { debug_assert!(index < self.max_instances); unsafe { - // The top (end) of the stack should be returned - Ok(self.mapping.as_mut_ptr().add((index + 1) * self.stack_size)) + // Remove the guard page from the size + let size_without_guard = self.stack_size - self.page_size; + + let bottom_of_stack = self + .mapping + .as_mut_ptr() + .add((index * self.stack_size) + self.page_size); + + commit_stack_pages(bottom_of_stack, size_without_guard) + .map_err(|e| FiberStackError::Resource(e.to_string()))?; + + // The top of the stack should be returned + Ok(bottom_of_stack.add(size_without_guard)) } } @@ -826,18 +836,16 @@ impl StackPool { let index = (start_of_stack - base) / self.stack_size; debug_assert!(index < self.max_instances); - decommit(bottom_of_stack, stack_size); + decommit_stack_pages(bottom_of_stack, stack_size).unwrap(); - { - self.free_list.lock().unwrap().push(index); - } + self.free_list.lock().unwrap().push(index); } } } /// Implements the pooling instance allocator. /// -/// This allocator interinally maintains pools of instances, memories, tables, and stacks. +/// This allocator internally maintains pools of instances, memories, tables, and stacks. /// /// Note: the resource pools are manually dropped so that the fault handler terminates correctly. #[derive(Debug)] @@ -845,10 +853,11 @@ pub struct PoolingInstanceAllocator { strategy: PoolingAllocationStrategy, module_limits: ModuleLimits, instance_limits: InstanceLimits, + // This is manually drop so that the pools unmap their memory before the page fault handler drops. instances: mem::ManuallyDrop, - stacks: mem::ManuallyDrop, + stacks: StackPool, #[cfg(all(feature = "uffd", target_os = "linux"))] - _fault_handler: PageFaultHandler, + _fault_handler: imp::PageFaultHandler, } impl PoolingInstanceAllocator { @@ -874,37 +883,18 @@ impl PoolingInstanceAllocator { instance_limits.memory_reservation_size = min(instance_limits.memory_reservation_size, 0x200000000); - // The maximum module memory page count cannot exceed 65536 pages - if module_limits.memory_pages > 0x10000 { - bail!( - "module memory page limit of {} exceeds the maximum of 65536", - module_limits.memory_pages - ); - } - - // The maximum module memory page count cannot exceed the memory reservation size - if (module_limits.memory_pages * WASM_PAGE_SIZE) as u64 - > instance_limits.memory_reservation_size - { - bail!( - "module memory page limit of {} pages exeeds the memory reservation size limit of {} bytes", - module_limits.memory_pages, - instance_limits.memory_reservation_size - ); - } - let instances = InstancePool::new(&module_limits, &instance_limits)?; let stacks = StackPool::new(&instance_limits, stack_size)?; #[cfg(all(feature = "uffd", target_os = "linux"))] - let _fault_handler = PageFaultHandler::new(&instances)?; + let _fault_handler = imp::PageFaultHandler::new(&instances)?; Ok(Self { strategy, module_limits, instance_limits, instances: mem::ManuallyDrop::new(instances), - stacks: mem::ManuallyDrop::new(stacks), + stacks, #[cfg(all(feature = "uffd", target_os = "linux"))] _fault_handler, }) @@ -917,7 +907,6 @@ impl Drop for PoolingInstanceAllocator { // This ensures that any fault handler thread monitoring the pool memory terminates unsafe { mem::ManuallyDrop::drop(&mut self.instances); - mem::ManuallyDrop::drop(&mut self.stacks); } } } @@ -963,13 +952,13 @@ unsafe impl InstanceAllocator for PoolingInstanceAllocator { cfg_if::cfg_if! { if #[cfg(all(feature = "uffd", target_os = "linux"))] { match &instance.module.memory_initialization { - MemoryInitialization::Paged{ out_of_bounds, .. } => { + wasmtime_environ::MemoryInitialization::Paged{ out_of_bounds, .. } => { if !is_bulk_memory { - check_init_bounds(instance)?; + super::check_init_bounds(instance)?; } // Initialize the tables - initialize_tables(instance)?; + super::initialize_tables(instance)?; // Don't initialize the memory; the fault handler will back the pages when accessed @@ -1312,21 +1301,6 @@ mod test { assert_eq!(strat.next(1), 0); } - #[test] - fn test_base_pointer_iterator() { - let mut iter = BasePointerIterator::new(std::ptr::null_mut(), 5, 3); - - assert_eq!(iter.next(), Some(0usize as _)); - assert_eq!(iter.next(), Some(3usize as _)); - assert_eq!(iter.next(), Some(6usize as _)); - assert_eq!(iter.next(), Some(9usize as _)); - assert_eq!(iter.next(), Some(12usize as _)); - assert_eq!(iter.next(), None); - - let mut iter = BasePointerIterator::new(std::ptr::null_mut(), 0, 10); - assert_eq!(iter.next(), None); - } - #[cfg(target_pointer_width = "64")] #[test] fn test_instance_pool() -> Result<()> { @@ -1341,11 +1315,11 @@ mod test { memories: 1, globals: 0, table_elements: 10, - memory_pages: 10, + memory_pages: 1, }; let instance_limits = InstanceLimits { count: 3, - memory_reservation_size: 4096, + memory_reservation_size: WASM_PAGE_SIZE as u64, }; let instances = InstancePool::new(&module_limits, &instance_limits)?; @@ -1366,7 +1340,7 @@ mod test { assert_eq!(instances.instance_size, 4096); assert_eq!(instances.max_instances, 3); - assert_eq!(&*instances.free_list.lock().unwrap(), &[0, 1, 2],); + assert_eq!(&*instances.free_list.lock().unwrap(), &[0, 1, 2]); let mut handles = Vec::new(); let module = Arc::new(Module::default()); @@ -1397,7 +1371,7 @@ mod test { ); } - assert_eq!(&*instances.free_list.lock().unwrap(), &[],); + assert_eq!(&*instances.free_list.lock().unwrap(), &[]); match instances.allocate( PoolingAllocationStrategy::NextAvailable, @@ -1425,7 +1399,7 @@ mod test { instances.deallocate(&handle); } - assert_eq!(&*instances.free_list.lock().unwrap(), &[2, 1, 0],); + assert_eq!(&*instances.free_list.lock().unwrap(), &[2, 1, 0]); Ok(()) } @@ -1445,7 +1419,7 @@ mod test { memories: 3, globals: 0, table_elements: 0, - memory_pages: 10, + memory_pages: 1, }, &InstanceLimits { count: 5, @@ -1456,7 +1430,7 @@ mod test { assert_eq!(pool.memory_size, WASM_PAGE_SIZE as usize); assert_eq!(pool.max_memories, 3); assert_eq!(pool.max_instances, 5); - assert_eq!(pool.max_wasm_pages, 10); + assert_eq!(pool.max_wasm_pages, 1); let base = pool.mapping.as_ptr() as usize; @@ -1554,7 +1528,7 @@ mod test { stacks.push(stack); } - assert_eq!(&*pool.free_list.lock().unwrap(), &[],); + assert_eq!(&*pool.free_list.lock().unwrap(), &[]); match pool .allocate(PoolingAllocationStrategy::NextAvailable) @@ -1632,7 +1606,7 @@ mod test { ) .map_err(|e| e.to_string()) .expect_err("expected a failure constructing instance allocator"), - "module memory page limit of 2 pages exeeds the memory reservation size limit of 65536 bytes" + "module memory page limit of 2 pages exceeds the memory reservation size limit of 65536 bytes" ); } diff --git a/crates/runtime/src/instance/allocator/pooling/linux.rs b/crates/runtime/src/instance/allocator/pooling/linux.rs index bd62f37dfbbe..324200efe42b 100644 --- a/crates/runtime/src/instance/allocator/pooling/linux.rs +++ b/crates/runtime/src/instance/allocator/pooling/linux.rs @@ -1,23 +1,58 @@ -use crate::Mmap; -use anyhow::{anyhow, Result}; +use anyhow::{bail, Context, Result}; -pub unsafe fn make_accessible(addr: *mut u8, len: usize) -> bool { - region::protect(addr, len, region::Protection::READ_WRITE).is_ok() +fn decommit(addr: *mut u8, len: usize, protect: bool) -> Result<()> { + if len == 0 { + return Ok(()); + } + + unsafe { + if protect { + region::protect(addr, len, region::Protection::NONE) + .context("failed to protect memory pages")?; + } + + // On Linux, this is enough to cause the kernel to initialize the pages to 0 on next access + if libc::madvise(addr as _, len, libc::MADV_DONTNEED) != 0 { + bail!( + "madvise failed to decommit: {}", + std::io::Error::last_os_error() + ); + } + } + + Ok(()) +} + +pub fn commit_memory_pages(addr: *mut u8, len: usize) -> Result<()> { + if len == 0 { + return Ok(()); + } + + // Just change the protection level to READ|WRITE + unsafe { + region::protect(addr, len, region::Protection::READ_WRITE) + .context("failed to make linear memory pages read/write") + } } -pub unsafe fn decommit(addr: *mut u8, len: usize) { - region::protect(addr, len, region::Protection::NONE).unwrap(); +pub fn decommit_memory_pages(addr: *mut u8, len: usize) -> Result<()> { + decommit(addr, len, true) +} + +pub fn commit_table_pages(_addr: *mut u8, _len: usize) -> Result<()> { + // A no-op as table pages remain READ|WRITE + Ok(()) +} + +pub fn decommit_table_pages(addr: *mut u8, len: usize) -> Result<()> { + decommit(addr, len, false) +} - // On Linux, this is enough to cause the kernel to initialize the pages to 0 on next access - assert_eq!( - libc::madvise(addr as _, len, libc::MADV_DONTNEED), - 0, - "madvise failed to mark pages as missing: {}", - std::io::Error::last_os_error() - ); +pub fn commit_stack_pages(_addr: *mut u8, _len: usize) -> Result<()> { + // A no-op as stack pages remain READ|WRITE + Ok(()) } -pub fn create_memory_map(accessible_size: usize, mapping_size: usize) -> Result { - Mmap::accessible_reserved(accessible_size, mapping_size) - .map_err(|e| anyhow!("failed to allocate pool memory: {}", e)) +pub fn decommit_stack_pages(addr: *mut u8, len: usize) -> Result<()> { + decommit(addr, len, false) } diff --git a/crates/runtime/src/instance/allocator/pooling/uffd.rs b/crates/runtime/src/instance/allocator/pooling/uffd.rs index b140a8d2a9ba..18e9eb3c2c82 100644 --- a/crates/runtime/src/instance/allocator/pooling/uffd.rs +++ b/crates/runtime/src/instance/allocator/pooling/uffd.rs @@ -16,11 +16,10 @@ //! 1. A user fault file descriptor is created to monitor specific areas of the address space. //! 2. A thread is spawned to continually read events from the user fault file descriptor. //! 3. When a page fault event is received, the handler thread calculates where the fault occurred: -//! a) If the fault occurs on a table page, it is handled by zeroing the page. -//! b) If the fault occurs on a linear memory page, it is handled by either copying the page from +//! a) If the fault occurs on a linear memory page, it is handled by either copying the page from //! initialization data or zeroing it. -//! c) If the fault occurs on a guard page, the protection level of the guard page is changed to -//! force the kernel to signal SIGSEV on the next retry. The faulting page is recorded so the +//! b) If the fault occurs on a guard page, the protection level of the guard page is changed to +//! force the kernel to signal SIGBUS on the next retry. The faulting page is recorded so the //! protection level can be reset in the future. //! 4. Faults to address space relating to an instance may occur from both Wasmtime (e.g. instance //! initialization) or from WebAssembly code (e.g. reading from or writing to linear memory), @@ -31,80 +30,103 @@ //! //! This feature requires a Linux kernel 4.11 or newer to use. -use super::InstancePool; -use crate::{instance::Instance, Mmap}; +use super::{InstancePool, MemoryPool}; +use crate::instance::Instance; use anyhow::{bail, Context, Result}; -use std::ptr; use std::thread; use userfaultfd::{Event, FeatureFlags, IoctlFlags, Uffd, UffdBuilder}; use wasmtime_environ::{entity::EntityRef, wasm::DefinedMemoryIndex, MemoryInitialization}; const WASM_PAGE_SIZE: usize = wasmtime_environ::WASM_PAGE_SIZE as usize; -pub unsafe fn make_accessible(_addr: *mut u8, _len: usize) -> bool { - // A no-op when userfaultfd is used - true +fn decommit(addr: *mut u8, len: usize) -> Result<()> { + if len == 0 { + return Ok(()); + } + + unsafe { + // On Linux, this is enough to cause the kernel to initialize the pages to 0 on next access + if libc::madvise(addr as _, len, libc::MADV_DONTNEED) != 0 { + bail!( + "madvise failed to decommit: {}", + std::io::Error::last_os_error() + ); + } + } + + Ok(()) } -pub unsafe fn reset_guard_page(addr: *mut u8, len: usize) -> bool { - // Guard pages are READ_WRITE with uffd until faulted - region::protect(addr, len, region::Protection::READ_WRITE).is_ok() +pub fn commit_memory_pages(_addr: *mut u8, _len: usize) -> Result<()> { + // A no-op as memory pages remain READ|WRITE with uffd + Ok(()) } -pub unsafe fn decommit(addr: *mut u8, len: usize) { - // Use MADV_DONTNEED to mark the pages as missing - // This will cause a missing page fault for next access on any page in the given range - assert_eq!( - libc::madvise(addr as _, len, libc::MADV_DONTNEED), - 0, - "madvise failed to mark pages as missing: {}", - std::io::Error::last_os_error() - ); +pub fn decommit_memory_pages(addr: *mut u8, len: usize) -> Result<()> { + decommit(addr, len) } -pub fn create_memory_map(_accessible_size: usize, mapping_size: usize) -> Result { - // Allocate a single read-write region at once - // As writable pages need to count towards commit charge, use MAP_NORESERVE to override. - // This implies that the kernel is configured to allow overcommit or else this allocation - // will almost certainly fail without a plethora of physical memory to back the allocation. - // The consequence of not reserving is that our process may segfault on any write to a memory - // page that cannot be backed (i.e. out of memory conditions). +pub fn commit_table_pages(_addr: *mut u8, _len: usize) -> Result<()> { + // A no-op as table pages remain READ|WRITE + Ok(()) +} - if mapping_size == 0 { - return Ok(Mmap::new()); - } +pub fn decommit_table_pages(addr: *mut u8, len: usize) -> Result<()> { + decommit(addr, len) +} - unsafe { - let ptr = libc::mmap( - ptr::null_mut(), - mapping_size, - libc::PROT_READ | libc::PROT_WRITE, - libc::MAP_PRIVATE | libc::MAP_ANON | libc::MAP_NORESERVE, - -1, - 0, - ); +pub fn commit_stack_pages(_addr: *mut u8, _len: usize) -> Result<()> { + // A no-op as stack pages remain READ|WRITE + Ok(()) +} - if ptr as isize == -1_isize { - bail!( - "failed to allocate pool memory: mmap failed with {}", - std::io::Error::last_os_error() - ); +pub fn decommit_stack_pages(addr: *mut u8, len: usize) -> Result<()> { + decommit(addr, len) +} + +/// This is used to initialize the memory pool when uffd is enabled. +/// +/// Without uffd, all of the memory pool's pages are initially protected with `NONE` to treat the entire +/// range as guard pages. When an instance is created, the initial pages of the memory are +/// changed to `READ_WRITE`. +/// +/// With uffd, however, the potentially accessible pages of the each linear memory are made `READ_WRITE` and +/// the page fault handler will detect an out of bounds access and treat the page, temporarily, +/// as a guard page. +/// +/// This me +pub(super) fn initialize_memory_pool(pool: &MemoryPool) -> Result<()> { + if pool.memory_size == 0 || pool.max_wasm_pages == 0 { + return Ok(()); + } + + for i in 0..pool.max_instances { + for base in pool.get(i) { + unsafe { + region::protect( + base as _, + pool.max_wasm_pages as usize * WASM_PAGE_SIZE, + region::Protection::READ_WRITE, + ) + .context("failed to initialize memory pool for uffd")?; + } } + } + + Ok(()) +} - Ok(Mmap::from_raw(ptr as usize, mapping_size)) +/// This is used to reset a linear memory's guard page back to read-write as the page might be accessible +/// again in the future depending on how the linear memory grows. +fn reset_guard_page(addr: *mut u8, len: usize) -> Result<()> { + unsafe { + region::protect(addr, len, region::Protection::READ_WRITE) + .context("failed to reset guard page") } } /// Represents a location of a page fault within monitored regions of memory. -enum AddressLocation<'a> { - /// The address location is in a WebAssembly table page. - /// The fault handler will zero the page as tables are initialized at instantiation-time. - TablePage { - /// The address of the page being accessed. - page_addr: *mut u8, - /// The length of the page being accessed. - len: usize, - }, +enum FaultLocation<'a> { /// The address location is in a WebAssembly linear memory page. /// The fault handler will copy the pages from initialization data if necessary. MemoryPage { @@ -121,12 +143,12 @@ enum AddressLocation<'a> { }, } -/// Used to resolve fault addresses to address locations. +/// Used to resolve fault addresses to a location. /// -/// This implementation relies heavily on how the various resource pools utilize their memory. +/// This implementation relies heavily on how the linear memory pool organizes its memory. /// /// `usize` is used here instead of pointers to keep this `Send` as it gets sent to the handler thread. -struct AddressLocator { +struct FaultLocator { instances_start: usize, instance_size: usize, max_instances: usize, @@ -134,19 +156,13 @@ struct AddressLocator { memories_end: usize, memory_size: usize, max_memories: usize, - tables_start: usize, - tables_end: usize, - table_size: usize, - page_size: usize, } -impl AddressLocator { +impl FaultLocator { fn new(instances: &InstancePool) -> Self { let instances_start = instances.mapping.as_ptr() as usize; let memories_start = instances.memories.mapping.as_ptr() as usize; let memories_end = memories_start + instances.memories.mapping.len(); - let tables_start = instances.tables.mapping.as_ptr() as usize; - let tables_end = tables_start + instances.tables.mapping.len(); // Should always have instances debug_assert!(instances_start != 0); @@ -159,10 +175,6 @@ impl AddressLocator { memories_end, memory_size: instances.memories.memory_size, max_memories: instances.memories.max_memories, - tables_start, - tables_end, - table_size: instances.tables.table_size, - page_size: instances.tables.page_size, } } @@ -174,7 +186,7 @@ impl AddressLocator { /// /// Of course a stray faulting memory access from a thread that does not own /// the instance might introduce a race, but this implementation considers - /// such to be a serious bug. + /// such to be a serious soundness bug not originating in this code. /// /// If the assumption holds true, accessing the instance data from the handler thread /// should, in theory, be safe. @@ -183,8 +195,8 @@ impl AddressLocator { &*((self.instances_start + (index * self.instance_size)) as *const Instance) } - unsafe fn get_location(&self, addr: usize) -> Option { - // Check for a memory location + unsafe fn locate(&self, addr: usize) -> Option { + // Check for a linear memory location if addr >= self.memories_start && addr < self.memories_end { let index = (addr - self.memories_start) / self.memory_size; let memory_index = DefinedMemoryIndex::new(index % self.max_memories); @@ -200,7 +212,7 @@ impl AddressLocator { } }); - return Some(AddressLocation::MemoryPage { + return Some(FaultLocation::MemoryPage { page_addr: (memory_start + page_index * WASM_PAGE_SIZE) as _, len: WASM_PAGE_SIZE, instance, @@ -209,19 +221,6 @@ impl AddressLocator { }); } - // Check for a table location - if addr >= self.tables_start && addr < self.tables_end { - let index = (addr - self.tables_start) / self.table_size; - let table_start = self.tables_start + (index * self.table_size); - let table_offset = addr - table_start; - let page_index = table_offset / self.page_size; - - return Some(AddressLocation::TablePage { - page_addr: (table_start + (page_index * self.page_size)) as _, - len: self.page_size, - }); - } - None } } @@ -231,9 +230,9 @@ impl AddressLocator { /// Because the region being monitored is protected read-write, this needs to set the /// protection level to `NONE` before waking the page. /// -/// This will cause the kernel to raise a SIGSEGV when retrying the fault. +/// This will cause the kernel to raise a SIGBUS when retrying the fault. unsafe fn wake_guard_page_access(uffd: &Uffd, page_addr: *const u8, len: usize) -> Result<()> { - // Set the page to NONE to induce a SIGSEGV for the access on the next retry + // Set the page to NONE to induce a SIGBUS for the access on the next retry region::protect(page_addr, len, region::Protection::NONE) .context("failed to change guard page protection")?; @@ -288,22 +287,11 @@ unsafe fn initialize_wasm_page( unsafe fn handle_page_fault( uffd: &Uffd, - locator: &AddressLocator, + locator: &FaultLocator, addr: *mut std::ffi::c_void, ) -> Result<()> { - match locator.get_location(addr as usize) { - Some(AddressLocation::TablePage { page_addr, len }) => { - log::trace!( - "handling fault in table at address {:p} on page {:p}", - addr, - page_addr, - ); - - // Tables are always initialized upon instantiation, so zero the page - uffd.zeropage(page_addr as _, len, true) - .context("failed to zero table page")?; - } - Some(AddressLocation::MemoryPage { + match locator.locate(addr as usize) { + Some(FaultLocation::MemoryPage { page_addr, len, instance, @@ -340,7 +328,7 @@ unsafe fn handle_page_fault( Ok(()) } -fn handler_thread(uffd: Uffd, locator: AddressLocator, mut registrations: usize) -> Result<()> { +fn fault_handler_thread(uffd: Uffd, locator: FaultLocator) -> Result<()> { loop { match uffd.read_event().expect("failed to read event") { Some(Event::Unmap { start, end }) => { @@ -348,13 +336,8 @@ fn handler_thread(uffd: Uffd, locator: AddressLocator, mut registrations: usize) let (start, end) = (start as usize, end as usize); - if (start == locator.memories_start && end == locator.memories_end) - || (start == locator.tables_start && end == locator.tables_end) - { - registrations -= 1; - if registrations == 0 { - break; - } + if start == locator.memories_start && end == locator.memories_end { + break; } else { panic!("unexpected memory region unmapped"); } @@ -385,53 +368,39 @@ impl PageFaultHandler { .create() .context("failed to create user fault descriptor")?; - // Register the ranges with the userfault fd - let mut registrations = 0; - for (start, len) in &[ - ( - instances.memories.mapping.as_ptr() as usize, - instances.memories.mapping.len(), - ), - ( - instances.tables.mapping.as_ptr() as usize, - instances.tables.mapping.len(), - ), - ] { - if *start == 0 || *len == 0 { - continue; - } + // Register the linear memory pool with the userfault fd + let start = instances.memories.mapping.as_ptr(); + let len = instances.memories.mapping.len(); + let thread = if !start.is_null() && len > 0 { let ioctls = uffd - .register(*start as _, *len) + .register(start as _, len) .context("failed to register user fault range")?; if !ioctls.contains(IoctlFlags::WAKE | IoctlFlags::COPY | IoctlFlags::ZEROPAGE) { bail!( - "required user fault ioctls not supported; found: {:?}", + "required user fault ioctls not supported by the kernel; found: {:?}", ioctls, ); } - registrations += 1; - } - - let thread = if registrations == 0 { - log::trace!("user fault handling disabled as there are no regions to monitor"); - None - } else { log::trace!( - "user fault handling enabled on {} memory regions", - registrations + "user fault handling enabled on linear memory pool at {:p} with size {}", + start, + len ); - let locator = AddressLocator::new(&instances); + let locator = FaultLocator::new(&instances); Some( thread::Builder::new() .name("page fault handler".into()) - .spawn(move || handler_thread(uffd, locator, registrations)) + .spawn(move || fault_handler_thread(uffd, locator)) .context("failed to spawn page fault handler thread")?, ) + } else { + log::trace!("user fault handling disabled as there is no linear memory pool"); + None }; Ok(Self { thread }) @@ -442,7 +411,7 @@ impl Drop for PageFaultHandler { fn drop(&mut self) { // The handler thread should terminate once all monitored regions of memory are unmapped. // The pooling instance allocator ensures that the regions are unmapped prior to dropping - // the user fault handler. + // the page fault handler. if let Some(thread) = self.thread.take() { thread .join() @@ -456,15 +425,12 @@ impl Drop for PageFaultHandler { mod test { use super::*; use crate::{ - table::max_table_element_size, Imports, InstanceAllocationRequest, InstanceLimits, - ModuleLimits, PoolingAllocationStrategy, VMSharedSignatureIndex, + Imports, InstanceAllocationRequest, InstanceLimits, ModuleLimits, + PoolingAllocationStrategy, VMSharedSignatureIndex, }; + use std::ptr; use std::sync::Arc; - use wasmtime_environ::{ - entity::PrimaryMap, - wasm::{Memory, Table, TableElementType, WasmType}, - MemoryPlan, MemoryStyle, Module, TablePlan, TableStyle, - }; + use wasmtime_environ::{entity::PrimaryMap, wasm::Memory, MemoryPlan, MemoryStyle, Module}; #[cfg(target_pointer_width = "64")] #[test] @@ -476,10 +442,10 @@ mod test { imported_globals: 0, types: 0, functions: 0, - tables: 3, + tables: 0, memories: 2, globals: 0, - table_elements: 1000, + table_elements: 0, memory_pages: 2, }; let instance_limits = InstanceLimits { @@ -490,7 +456,7 @@ mod test { let instances = InstancePool::new(&module_limits, &instance_limits).expect("should allocate"); - let locator = AddressLocator::new(&instances); + let locator = FaultLocator::new(&instances); assert_eq!(locator.instances_start, instances.mapping.as_ptr() as usize); assert_eq!(locator.instance_size, 4096); @@ -505,21 +471,10 @@ mod test { ); assert_eq!(locator.memory_size, WASM_PAGE_SIZE * 10); assert_eq!(locator.max_memories, 2); - assert_eq!( - locator.tables_start, - instances.tables.mapping.as_ptr() as usize - ); - assert_eq!( - locator.tables_end, - locator.tables_start + instances.tables.mapping.len() - ); - assert_eq!(locator.table_size, 8192); unsafe { - assert!(locator.get_location(0).is_none()); - assert!(locator - .get_location(std::cmp::max(locator.memories_end, locator.tables_end)) - .is_none()); + assert!(locator.locate(0).is_none()); + assert!(locator.locate(locator.memories_end).is_none()); let mut module = Module::new(); @@ -535,25 +490,13 @@ mod test { }); } - for _ in 0..module_limits.tables { - module.table_plans.push(TablePlan { - table: Table { - wasm_ty: WasmType::FuncRef, - ty: TableElementType::Func, - minimum: 800, - maximum: Some(900), - }, - style: TableStyle::CallerChecksSignature, - }); - } - module_limits.validate(&module).expect("should validate"); let mut handles = Vec::new(); let module = Arc::new(module); let finished_functions = &PrimaryMap::new(); - // Allocate the maximum number of instances with the maxmimum number of memories and tables + // Allocate the maximum number of instances with the maximum number of memories for _ in 0..instances.max_instances { handles.push( instances @@ -570,9 +513,9 @@ mod test { }, lookup_shared_signature: &|_| VMSharedSignatureIndex::default(), host_state: Box::new(()), - interrupts: std::ptr::null(), - externref_activations_table: std::ptr::null_mut(), - stack_map_registry: std::ptr::null_mut(), + interrupts: ptr::null(), + externref_activations_table: ptr::null_mut(), + stack_map_registry: ptr::null_mut(), }, ) .expect("instance should allocate"), @@ -587,8 +530,8 @@ mod test { + (memory_index * locator.memory_size); // Test for access to first page - match locator.get_location(memory_start + 10000) { - Some(AddressLocation::MemoryPage { + match locator.locate(memory_start + 10000) { + Some(FaultLocation::MemoryPage { page_addr, len, instance: _, @@ -604,8 +547,8 @@ mod test { } // Test for access to second page - match locator.get_location(memory_start + 1024 + WASM_PAGE_SIZE) { - Some(AddressLocation::MemoryPage { + match locator.locate(memory_start + 1024 + WASM_PAGE_SIZE) { + Some(FaultLocation::MemoryPage { page_addr, len, instance: _, @@ -621,8 +564,8 @@ mod test { } // Test for guard page - match locator.get_location(memory_start + 10 + 9 * WASM_PAGE_SIZE) { - Some(AddressLocation::MemoryPage { + match locator.locate(memory_start + 10 + 9 * WASM_PAGE_SIZE) { + Some(FaultLocation::MemoryPage { page_addr, len, instance: _, @@ -639,33 +582,6 @@ mod test { } } - // Validate table locations - for instance_index in 0..instances.max_instances { - for table_index in 0..instances.tables.max_tables { - let table_start = locator.tables_start - + (instance_index * locator.table_size * instances.tables.max_tables) - + (table_index * locator.table_size); - - // Check for an access of index 107 (first page) - match locator.get_location(table_start + (107 * max_table_element_size())) { - Some(AddressLocation::TablePage { page_addr, len }) => { - assert_eq!(page_addr, table_start as _); - assert_eq!(len, locator.page_size); - } - _ => panic!("expected a table page location"), - } - - // Check for an access of index 799 (second page) - match locator.get_location(table_start + (799 * max_table_element_size())) { - Some(AddressLocation::TablePage { page_addr, len }) => { - assert_eq!(page_addr, (table_start + locator.page_size) as _); - assert_eq!(len, locator.page_size); - } - _ => panic!("expected a table page location"), - } - } - } - for handle in handles.drain(..) { instances.deallocate(&handle); } diff --git a/crates/runtime/src/instance/allocator/pooling/unix.rs b/crates/runtime/src/instance/allocator/pooling/unix.rs index 9cc68b3361e7..957aea8e1b4b 100644 --- a/crates/runtime/src/instance/allocator/pooling/unix.rs +++ b/crates/runtime/src/instance/allocator/pooling/unix.rs @@ -1,27 +1,64 @@ -use crate::Mmap; -use anyhow::{anyhow, Result}; +use anyhow::{bail, Context, Result}; -pub unsafe fn make_accessible(addr: *mut u8, len: usize) -> bool { - region::protect(addr, len, region::Protection::READ_WRITE).is_ok() -} +fn decommit(addr: *mut u8, len: usize, protect: bool) -> Result<()> { + if len == 0 { + return Ok(()); + } -pub unsafe fn decommit(addr: *mut u8, len: usize) { - assert_eq!( + if unsafe { libc::mmap( addr as _, len, - libc::PROT_NONE, + if protect { + libc::PROT_NONE + } else { + libc::PROT_READ | libc::PROT_WRITE + }, libc::MAP_PRIVATE | libc::MAP_ANON | libc::MAP_FIXED, -1, 0, - ) as *mut u8, - addr, - "mmap failed to remap pages: {}", - std::io::Error::last_os_error() - ); + ) as *mut u8 + } != addr + { + bail!( + "mmap failed to remap pages: {}", + std::io::Error::last_os_error() + ); + } + + Ok(()) +} + +pub fn commit_memory_pages(addr: *mut u8, len: usize) -> Result<()> { + if len == 0 { + return Ok(()); + } + + // Just change the protection level to READ|WRITE + unsafe { + region::protect(addr, len, region::Protection::READ_WRITE) + .context("failed to make linear memory pages read/write") + } +} + +pub fn decommit_memory_pages(addr: *mut u8, len: usize) -> Result<()> { + decommit(addr, len, true) +} + +pub fn commit_table_pages(_addr: *mut u8, _len: usize) -> Result<()> { + // A no-op as table pages remain READ|WRITE + Ok(()) +} + +pub fn decommit_table_pages(addr: *mut u8, len: usize) -> Result<()> { + decommit(addr, len, false) +} + +pub fn commit_stack_pages(_addr: *mut u8, _len: usize) -> Result<()> { + // A no-op as stack pages remain READ|WRITE + Ok(()) } -pub fn create_memory_map(accessible_size: usize, mapping_size: usize) -> Result { - Mmap::accessible_reserved(accessible_size, mapping_size) - .map_err(|e| anyhow!("failed to allocate pool memory: {}", e)) +pub fn decommit_stack_pages(addr: *mut u8, len: usize) -> Result<()> { + decommit(addr, len, false) } diff --git a/crates/runtime/src/instance/allocator/pooling/windows.rs b/crates/runtime/src/instance/allocator/pooling/windows.rs index 159f00b63fa7..286cd459fe29 100644 --- a/crates/runtime/src/instance/allocator/pooling/windows.rs +++ b/crates/runtime/src/instance/allocator/pooling/windows.rs @@ -1,22 +1,55 @@ -use crate::Mmap; -use anyhow::{anyhow, Result}; +use anyhow::{bail, Result}; use winapi::um::memoryapi::{VirtualAlloc, VirtualFree}; use winapi::um::winnt::{MEM_COMMIT, MEM_DECOMMIT, PAGE_READWRITE}; -pub unsafe fn make_accessible(addr: *mut u8, len: usize) -> bool { - // This doesn't use the `region` crate because the memory needs to be committed - !VirtualAlloc(addr as _, len, MEM_COMMIT, PAGE_READWRITE).is_null() +pub fn commit(addr: *mut u8, len: usize) -> Result<()> { + if len == 0 { + return Ok(()); + } + + // Memory needs to be committed, so don't use the `region` crate + if unsafe { VirtualAlloc(addr as _, len, MEM_COMMIT, PAGE_READWRITE).is_null() } { + bail!("failed to commit memory as read/write"); + } + + Ok(()) +} + +pub fn decommit(addr: *mut u8, len: usize) -> Result<()> { + if len == 0 { + return Ok(()); + } + + if unsafe { VirtualFree(addr as _, len, MEM_DECOMMIT) } == 0 { + bail!( + "failed to decommit memory pages: {}", + std::io::Error::last_os_error() + ); + } + + Ok(()) +} + +pub fn commit_memory_pages(addr: *mut u8, len: usize) -> Result<()> { + commit(addr, len) +} + +pub fn decommit_memory_pages(addr: *mut u8, len: usize) -> Result<()> { + decommit(addr, len) +} + +pub fn commit_table_pages(addr: *mut u8, len: usize) -> Result<()> { + commit(addr, len) +} + +pub fn decommit_table_pages(addr: *mut u8, len: usize) -> Result<()> { + decommit(addr, len) } -pub unsafe fn decommit(addr: *mut u8, len: usize) { - assert!( - VirtualFree(addr as _, len, MEM_DECOMMIT) != 0, - "failed to decommit memory pages: {}", - std::io::Error::last_os_error() - ); +pub fn commit_stack_pages(addr: *mut u8, len: usize) -> Result<()> { + commit(addr, len) } -pub fn create_memory_map(accessible_size: usize, mapping_size: usize) -> Result { - Mmap::accessible_reserved(accessible_size, mapping_size) - .map_err(|e| anyhow!("failed to allocate pool memory: {}", e)) +pub fn decommit_stack_pages(addr: *mut u8, len: usize) -> Result<()> { + decommit(addr, len) } diff --git a/crates/runtime/src/memory.rs b/crates/runtime/src/memory.rs index 7d248b136189..ba7bab470034 100644 --- a/crates/runtime/src/memory.rs +++ b/crates/runtime/src/memory.rs @@ -4,6 +4,7 @@ use crate::mmap::Mmap; use crate::vmcontext::VMMemoryDefinition; +use anyhow::Result; use more_asserts::{assert_ge, assert_le}; use std::cell::{Cell, RefCell}; use std::cmp::min; @@ -13,7 +14,7 @@ use wasmtime_environ::{MemoryPlan, MemoryStyle, WASM_MAX_PAGES, WASM_PAGE_SIZE}; /// A memory allocator pub trait RuntimeMemoryCreator: Send + Sync { /// Create new RuntimeLinearMemory - fn new_memory(&self, plan: &MemoryPlan) -> Result, String>; + fn new_memory(&self, plan: &MemoryPlan) -> Result>; } /// A default memory allocator used by Wasmtime @@ -21,8 +22,8 @@ pub struct DefaultMemoryCreator; impl RuntimeMemoryCreator for DefaultMemoryCreator { /// Create new MmapMemory - fn new_memory(&self, plan: &MemoryPlan) -> Result, String> { - Ok(Box::new(MmapMemory::new(plan)?) as Box) + fn new_memory(&self, plan: &MemoryPlan) -> Result> { + Ok(Box::new(MmapMemory::new(plan)?) as _) } } @@ -65,7 +66,7 @@ struct WasmMmap { impl MmapMemory { /// Create a new linear memory instance with specified minimum and maximum number of wasm pages. - pub fn new(plan: &MemoryPlan) -> Result { + pub fn new(plan: &MemoryPlan) -> Result { // `maximum` cannot be set to more than `65536` pages. assert_le!(plan.memory.minimum, WASM_MAX_PAGES); assert!(plan.memory.maximum.is_none() || plan.memory.maximum.unwrap() <= WASM_MAX_PAGES); @@ -177,7 +178,7 @@ enum MemoryStorage { base: *mut u8, size: Cell, maximum: u32, - make_accessible: unsafe fn(*mut u8, usize) -> bool, + make_accessible: fn(*mut u8, usize) -> Result<()>, }, Dynamic(Box), } @@ -189,10 +190,7 @@ pub struct Memory { impl Memory { /// Create a new dynamic (movable) memory instance for the specified plan. - pub fn new_dynamic( - plan: &MemoryPlan, - creator: &dyn RuntimeMemoryCreator, - ) -> Result { + pub fn new_dynamic(plan: &MemoryPlan, creator: &dyn RuntimeMemoryCreator) -> Result { Ok(Self { storage: MemoryStorage::Dynamic(creator.new_memory(plan)?), }) @@ -203,14 +201,10 @@ impl Memory { plan: &MemoryPlan, base: *mut u8, maximum: u32, - make_accessible: unsafe fn(*mut u8, usize) -> bool, - ) -> Result { + make_accessible: fn(*mut u8, usize) -> Result<()>, + ) -> Result { if plan.memory.minimum > 0 { - if unsafe { - !make_accessible(base, plan.memory.minimum as usize * WASM_PAGE_SIZE as usize) - } { - return Err("memory cannot be made accessible".into()); - } + make_accessible(base, plan.memory.minimum as usize * WASM_PAGE_SIZE as usize)?; } Ok(Self { @@ -258,9 +252,7 @@ impl Memory { let start = usize::try_from(old_size).unwrap() * WASM_PAGE_SIZE as usize; let len = usize::try_from(delta).unwrap() * WASM_PAGE_SIZE as usize; - if unsafe { !make_accessible(base.add(start), len) } { - return None; - } + make_accessible(unsafe { base.add(start) }, len).ok()?; size.set(new_size); diff --git a/crates/runtime/src/mmap.rs b/crates/runtime/src/mmap.rs index e60e87e3783a..b0612f91c96c 100644 --- a/crates/runtime/src/mmap.rs +++ b/crates/runtime/src/mmap.rs @@ -1,6 +1,7 @@ //! Low-level abstraction for allocating and managing zero-filled pages //! of memory. +use anyhow::{bail, Result}; use more_asserts::assert_le; use more_asserts::assert_lt; use std::io; @@ -38,7 +39,7 @@ impl Mmap { } /// Create a new `Mmap` pointing to at least `size` bytes of page-aligned accessible memory. - pub fn with_at_least(size: usize) -> Result { + pub fn with_at_least(size: usize) -> Result { let page_size = region::page::size(); let rounded_size = round_up_to_page_size(size, page_size); Self::accessible_reserved(rounded_size, rounded_size) @@ -48,10 +49,7 @@ impl Mmap { /// within a reserved mapping of `mapping_size` bytes. `accessible_size` and `mapping_size` /// must be native page-size multiples. #[cfg(not(target_os = "windows"))] - pub fn accessible_reserved( - accessible_size: usize, - mapping_size: usize, - ) -> Result { + pub fn accessible_reserved(accessible_size: usize, mapping_size: usize) -> Result { let page_size = region::page::size(); assert_le!(accessible_size, mapping_size); assert_eq!(mapping_size & (page_size - 1), 0); @@ -76,7 +74,7 @@ impl Mmap { ) }; if ptr as isize == -1_isize { - return Err(io::Error::last_os_error().to_string()); + bail!("mmap failed: {}", io::Error::last_os_error()); } Self { @@ -96,7 +94,7 @@ impl Mmap { ) }; if ptr as isize == -1_isize { - return Err(io::Error::last_os_error().to_string()); + bail!("mmap failed: {}", io::Error::last_os_error()); } let mut result = Self { @@ -117,10 +115,7 @@ impl Mmap { /// within a reserved mapping of `mapping_size` bytes. `accessible_size` and `mapping_size` /// must be native page-size multiples. #[cfg(target_os = "windows")] - pub fn accessible_reserved( - accessible_size: usize, - mapping_size: usize, - ) -> Result { + pub fn accessible_reserved(accessible_size: usize, mapping_size: usize) -> Result { use winapi::um::memoryapi::VirtualAlloc; use winapi::um::winnt::{MEM_COMMIT, MEM_RESERVE, PAGE_NOACCESS, PAGE_READWRITE}; @@ -144,7 +139,7 @@ impl Mmap { ) }; if ptr.is_null() { - return Err(io::Error::last_os_error().to_string()); + bail!("VirtualAlloc failed: {}", io::Error::last_os_error()); } Self { @@ -156,7 +151,7 @@ impl Mmap { let ptr = unsafe { VirtualAlloc(ptr::null_mut(), mapping_size, MEM_RESERVE, PAGE_NOACCESS) }; if ptr.is_null() { - return Err(io::Error::last_os_error().to_string()); + bail!("VirtualAlloc failed: {}", io::Error::last_os_error()); } let mut result = Self { @@ -177,7 +172,7 @@ impl Mmap { /// `start` and `len` must be native page-size multiples and describe a range within /// `self`'s reserved memory. #[cfg(not(target_os = "windows"))] - pub fn make_accessible(&mut self, start: usize, len: usize) -> Result<(), String> { + pub fn make_accessible(&mut self, start: usize, len: usize) -> Result<()> { let page_size = region::page::size(); assert_eq!(start & (page_size - 1), 0); assert_eq!(len & (page_size - 1), 0); @@ -186,15 +181,18 @@ impl Mmap { // Commit the accessible size. let ptr = self.ptr as *const u8; - unsafe { region::protect(ptr.add(start), len, region::Protection::READ_WRITE) } - .map_err(|e| e.to_string()) + unsafe { + region::protect(ptr.add(start), len, region::Protection::READ_WRITE)?; + } + + Ok(()) } /// Make the memory starting at `start` and extending for `len` bytes accessible. /// `start` and `len` must be native page-size multiples and describe a range within /// `self`'s reserved memory. #[cfg(target_os = "windows")] - pub fn make_accessible(&mut self, start: usize, len: usize) -> Result<(), String> { + pub fn make_accessible(&mut self, start: usize, len: usize) -> Result<()> { use winapi::ctypes::c_void; use winapi::um::memoryapi::VirtualAlloc; use winapi::um::winnt::{MEM_COMMIT, PAGE_READWRITE}; @@ -216,7 +214,7 @@ impl Mmap { } .is_null() { - return Err(io::Error::last_os_error().to_string()); + bail!("VirtualAlloc failed: {}", io::Error::last_os_error()); } Ok(()) diff --git a/crates/wasmtime/src/trampoline/memory.rs b/crates/wasmtime/src/trampoline/memory.rs index af2b9547920b..37c14c51f6aa 100644 --- a/crates/wasmtime/src/trampoline/memory.rs +++ b/crates/wasmtime/src/trampoline/memory.rs @@ -3,7 +3,7 @@ use crate::memory::{LinearMemory, MemoryCreator}; use crate::trampoline::StoreInstanceHandle; use crate::Store; use crate::{Limits, MemoryType}; -use anyhow::Result; +use anyhow::{anyhow, Result}; use wasmtime_environ::entity::PrimaryMap; use wasmtime_environ::{wasm, MemoryPlan, MemoryStyle, Module, WASM_PAGE_SIZE}; use wasmtime_runtime::{RuntimeLinearMemory, RuntimeMemoryCreator, VMMemoryDefinition}; @@ -57,7 +57,7 @@ impl RuntimeLinearMemory for LinearMemoryProxy { pub(crate) struct MemoryCreatorProxy(pub Arc); impl RuntimeMemoryCreator for MemoryCreatorProxy { - fn new_memory(&self, plan: &MemoryPlan) -> Result, String> { + fn new_memory(&self, plan: &MemoryPlan) -> Result> { let ty = MemoryType::new(Limits::new(plan.memory.minimum, plan.memory.maximum)); let reserved_size_in_bytes = match plan.style { MemoryStyle::Static { bound } => Some(bound as u64 * WASM_PAGE_SIZE as u64), @@ -66,5 +66,6 @@ impl RuntimeMemoryCreator for MemoryCreatorProxy { self.0 .new_memory(ty, reserved_size_in_bytes, plan.offset_guard_size) .map(|mem| Box::new(LinearMemoryProxy { mem }) as Box) + .map_err(|e| anyhow!(e)) } } From a4084db09660d9cf51bc84d3dc625f1e9b228c4c Mon Sep 17 00:00:00 2001 From: Peter Huene Date: Thu, 4 Mar 2021 22:27:27 -0800 Subject: [PATCH 25/33] More feedback changes. * Don't reexport types from `wasmtime_runtime` from the `wasmtime` crate. * Add more comments. --- crates/runtime/src/instance.rs | 6 +- .../runtime/src/instance/allocator/pooling.rs | 60 ++--- crates/wasmtime/src/config.rs | 228 +++++++++++++++++- 3 files changed, 241 insertions(+), 53 deletions(-) diff --git a/crates/runtime/src/instance.rs b/crates/runtime/src/instance.rs index 7c06c6121193..9475e4751745 100644 --- a/crates/runtime/src/instance.rs +++ b/crates/runtime/src/instance.rs @@ -381,7 +381,11 @@ impl Instance { /// Returns `None` if memory can't be grown by the specified amount /// of pages. pub(crate) fn memory_grow(&self, memory_index: DefinedMemoryIndex, delta: u32) -> Option { - // Reset all guard pages before growing any memory + // Reset all guard pages before growing any memory when using the uffd feature. + // The uffd feature induces a trap when a fault on a linear memory page is determined to be out-of-bounds. + // It does this by temporarily setting the protection level to `NONE` to cause the kernel to signal SIGBUS. + // Because instances might still be used after a trap, this resets the page back to the expected protection + // level (READ_WRITE) for the uffd implementation. #[cfg(all(feature = "uffd", target_os = "linux"))] self.reset_guard_pages().ok()?; diff --git a/crates/runtime/src/instance/allocator/pooling.rs b/crates/runtime/src/instance/allocator/pooling.rs index c9e054cbc1ae..18ddf2bc4211 100644 --- a/crates/runtime/src/instance/allocator/pooling.rs +++ b/crates/runtime/src/instance/allocator/pooling.rs @@ -55,53 +55,37 @@ fn round_up_to_pow2(n: usize, to: usize) -> usize { /// Represents the limits placed on a module for compiling with the pooling instance allocator. #[derive(Debug, Copy, Clone)] pub struct ModuleLimits { - /// The maximum number of imported functions for a module (default is 1000). + /// The maximum number of imported functions for a module. pub imported_functions: u32, - /// The maximum number of imported tables for a module (default is 0). + /// The maximum number of imported tables for a module. pub imported_tables: u32, - /// The maximum number of imported linear memories for a module (default is 0). + /// The maximum number of imported linear memories for a module. pub imported_memories: u32, - /// The maximum number of imported globals for a module (default is 0). + /// The maximum number of imported globals for a module. pub imported_globals: u32, - /// The maximum number of defined types for a module (default is 100). + /// The maximum number of defined types for a module. pub types: u32, - /// The maximum number of defined functions for a module (default is 10000). + /// The maximum number of defined functions for a module. pub functions: u32, - /// The maximum number of defined tables for a module (default is 1). + /// The maximum number of defined tables for a module. pub tables: u32, - /// The maximum number of defined linear memories for a module (default is 1). + /// The maximum number of defined linear memories for a module. pub memories: u32, - /// The maximum number of defined globals for a module (default is 10). + /// The maximum number of defined globals for a module. pub globals: u32, - /// The maximum table elements for any table defined in a module (default is 10000). - /// - /// If a table's minimum element limit is greater than this value, the module will - /// fail to compile. - /// - /// If a table's maximum element limit is unbounded or greater than this value, - /// the maximum will be `table_elements` for the purpose of any `table.grow` instruction. + /// The maximum table elements for any table defined in a module. pub table_elements: u32, - /// The maximum number of pages for any linear memory defined in a module (default is 160). - /// - /// The default of 160 means at most 10 MiB of host memory may be committed for each instance. - /// - /// If a memory's minimum page limit is greater than this value, the module will - /// fail to compile. - /// - /// If a memory's maximum page limit is unbounded or greater than this value, - /// the maximum will be `memory_pages` for the purpose of any `memory.grow` instruction. - /// - /// This value cannot exceed any memory reservation size limits placed on instances. + /// The maximum number of pages for any linear memory defined in a module. pub memory_pages: u32, } @@ -224,7 +208,7 @@ impl ModuleLimits { impl Default for ModuleLimits { fn default() -> Self { - // See doc comments for `ModuleLimits` for these default values + // See doc comments for `wasmtime::ModuleLimits` for these default values Self { imported_functions: 1000, imported_tables: 0, @@ -244,32 +228,16 @@ impl Default for ModuleLimits { /// Represents the limits placed on instances by the pooling instance allocator. #[derive(Debug, Copy, Clone)] pub struct InstanceLimits { - /// The maximum number of concurrent instances supported (default is 1000). + /// The maximum number of concurrent instances supported. pub count: u32, /// The maximum size, in bytes, of host address space to reserve for each linear memory of an instance. - /// - /// Note: this value has important performance ramifications. - /// - /// On 64-bit platforms, the default for this value will be 6 GiB. A value of less than 4 GiB will - /// force runtime bounds checking for memory accesses and thus will negatively impact performance. - /// Any value above 4 GiB will start eliding bounds checks provided the `offset` of the memory access is - /// less than (`memory_reservation_size` - 4 GiB). A value of 8 GiB will completely elide *all* bounds - /// checks; consequently, 8 GiB will be the maximum supported value. The default of 6 GiB reserves - /// less host address space for each instance, but a memory access with an offet above 2 GiB will incur - /// runtime bounds checks. - /// - /// On 32-bit platforms, the default for this value will be 10 MiB. A 32-bit host has very limited address - /// space to reserve for a lot of concurrent instances. As a result, runtime bounds checking will be used - /// for all memory accesses. For better runtime performance, a 64-bit host is recommended. - /// - /// This value will be rounded up by the WebAssembly page size (64 KiB). pub memory_reservation_size: u64, } impl Default for InstanceLimits { fn default() -> Self { - // See doc comments for `InstanceLimits` for these default values + // See doc comments for `wasmtime::InstanceLimits` for these default values Self { count: 1000, #[cfg(target_pointer_width = "32")] diff --git a/crates/wasmtime/src/config.rs b/crates/wasmtime/src/config.rs index 30e63546e867..fdce7b9202f7 100644 --- a/crates/wasmtime/src/config.rs +++ b/crates/wasmtime/src/config.rs @@ -16,8 +16,213 @@ use wasmtime_jit::{native, CompilationStrategy, Compiler}; use wasmtime_profiling::{JitDumpAgent, NullProfilerAgent, ProfilingAgent, VTuneAgent}; use wasmtime_runtime::{InstanceAllocator, OnDemandInstanceAllocator, PoolingInstanceAllocator}; -// Re-export the limit structures for the pooling allocator -pub use wasmtime_runtime::{InstanceLimits, ModuleLimits, PoolingAllocationStrategy}; +/// Represents the limits placed on a module for compiling with the pooling instance allocation strategy. +#[derive(Debug, Copy, Clone)] +pub struct ModuleLimits { + /// The maximum number of imported functions for a module (default is 1000). + pub imported_functions: u32, + + /// The maximum number of imported tables for a module (default is 0). + pub imported_tables: u32, + + /// The maximum number of imported linear memories for a module (default is 0). + pub imported_memories: u32, + + /// The maximum number of imported globals for a module (default is 0). + pub imported_globals: u32, + + /// The maximum number of defined types for a module (default is 100). + pub types: u32, + + /// The maximum number of defined functions for a module (default is 10000). + pub functions: u32, + + /// The maximum number of defined tables for a module (default is 1). + pub tables: u32, + + /// The maximum number of defined linear memories for a module (default is 1). + pub memories: u32, + + /// The maximum number of defined globals for a module (default is 10). + pub globals: u32, + + /// The maximum table elements for any table defined in a module (default is 10000). + /// + /// If a table's minimum element limit is greater than this value, the module will + /// fail to compile. + /// + /// If a table's maximum element limit is unbounded or greater than this value, + /// the maximum will be `table_elements` for the purpose of any `table.grow` instruction. + pub table_elements: u32, + + /// The maximum number of pages for any linear memory defined in a module (default is 160). + /// + /// The default of 160 means at most 10 MiB of host memory may be committed for each instance. + /// + /// If a memory's minimum page limit is greater than this value, the module will + /// fail to compile. + /// + /// If a memory's maximum page limit is unbounded or greater than this value, + /// the maximum will be `memory_pages` for the purpose of any `memory.grow` instruction. + /// + /// This value cannot exceed any memory reservation size limits placed on instances. + pub memory_pages: u32, +} + +impl Default for ModuleLimits { + fn default() -> Self { + // Use the defaults from the runtime + let wasmtime_runtime::ModuleLimits { + imported_functions, + imported_tables, + imported_memories, + imported_globals, + types, + functions, + tables, + memories, + globals, + table_elements, + memory_pages, + } = wasmtime_runtime::ModuleLimits::default(); + + Self { + imported_functions, + imported_tables, + imported_memories, + imported_globals, + types, + functions, + tables, + memories, + globals, + table_elements, + memory_pages, + } + } +} + +// This exists so we can convert between the public Wasmtime API and the runtime representation +// without having to export runtime types from the Wasmtime API. +#[doc(hidden)] +impl Into for ModuleLimits { + fn into(self) -> wasmtime_runtime::ModuleLimits { + let Self { + imported_functions, + imported_tables, + imported_memories, + imported_globals, + types, + functions, + tables, + memories, + globals, + table_elements, + memory_pages, + } = self; + + wasmtime_runtime::ModuleLimits { + imported_functions, + imported_tables, + imported_memories, + imported_globals, + types, + functions, + tables, + memories, + globals, + table_elements, + memory_pages, + } + } +} + +/// Represents the limits placed on instances by the pooling instance allocation strategy. +#[derive(Debug, Copy, Clone)] +pub struct InstanceLimits { + /// The maximum number of concurrent instances supported (default is 1000). + pub count: u32, + + /// The maximum size, in bytes, of host address space to reserve for each linear memory of an instance. + /// + /// Note: this value has important performance ramifications. + /// + /// On 64-bit platforms, the default for this value will be 6 GiB. A value of less than 4 GiB will + /// force runtime bounds checking for memory accesses and thus will negatively impact performance. + /// Any value above 4 GiB will start eliding bounds checks provided the `offset` of the memory access is + /// less than (`memory_reservation_size` - 4 GiB). A value of 8 GiB will completely elide *all* bounds + /// checks; consequently, 8 GiB will be the maximum supported value. The default of 6 GiB reserves + /// less host address space for each instance, but a memory access with an offset above 2 GiB will incur + /// runtime bounds checks. + /// + /// On 32-bit platforms, the default for this value will be 10 MiB. A 32-bit host has very limited address + /// space to reserve for a lot of concurrent instances. As a result, runtime bounds checking will be used + /// for all memory accesses. For better runtime performance, a 64-bit host is recommended. + /// + /// This value will be rounded up by the WebAssembly page size (64 KiB). + pub memory_reservation_size: u64, +} + +impl Default for InstanceLimits { + fn default() -> Self { + let wasmtime_runtime::InstanceLimits { + count, + memory_reservation_size, + } = wasmtime_runtime::InstanceLimits::default(); + + Self { + count, + memory_reservation_size, + } + } +} + +// This exists so we can convert between the public Wasmtime API and the runtime representation +// without having to export runtime types from the Wasmtime API. +#[doc(hidden)] +impl Into for InstanceLimits { + fn into(self) -> wasmtime_runtime::InstanceLimits { + let Self { + count, + memory_reservation_size, + } = self; + + wasmtime_runtime::InstanceLimits { + count, + memory_reservation_size, + } + } +} + +/// The allocation strategy to use for the pooling instance allocation strategy. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PoolingAllocationStrategy { + /// Allocate from the next available instance. + NextAvailable, + /// Allocate from a random available instance. + Random, +} + +impl Default for PoolingAllocationStrategy { + fn default() -> Self { + match wasmtime_runtime::PoolingAllocationStrategy::default() { + wasmtime_runtime::PoolingAllocationStrategy::NextAvailable => Self::NextAvailable, + wasmtime_runtime::PoolingAllocationStrategy::Random => Self::Random, + } + } +} + +// This exists so we can convert between the public Wasmtime API and the runtime representation +// without having to export runtime types from the Wasmtime API. +#[doc(hidden)] +impl Into for PoolingAllocationStrategy { + fn into(self) -> wasmtime_runtime::PoolingAllocationStrategy { + match self { + Self::NextAvailable => wasmtime_runtime::PoolingAllocationStrategy::NextAvailable, + Self::Random => wasmtime_runtime::PoolingAllocationStrategy::Random, + } + } +} /// Represents the module instance allocation strategy to use. #[derive(Clone)] @@ -44,6 +249,17 @@ pub enum InstanceAllocationStrategy { }, } +impl InstanceAllocationStrategy { + /// The default pooling instance allocation strategy. + pub fn pooling() -> Self { + Self::Pooling { + strategy: PoolingAllocationStrategy::default(), + module_limits: ModuleLimits::default(), + instance_limits: InstanceLimits::default(), + } + } +} + impl Default for InstanceAllocationStrategy { fn default() -> Self { Self::OnDemand @@ -66,7 +282,7 @@ pub struct Config { pub(crate) profiler: Arc, pub(crate) instance_allocator: Option>, // The default instance allocator is used for instantiating host objects - // and for module instatiation when `instance_allocator` is None + // and for module instantiation when `instance_allocator` is None pub(crate) default_instance_allocator: OnDemandInstanceAllocator, pub(crate) max_wasm_stack: usize, pub(crate) features: WasmFeatures, @@ -628,9 +844,9 @@ impl Config { let stack_size = 0; Some(Arc::new(PoolingInstanceAllocator::new( - strategy, - module_limits, - instance_limits, + strategy.into(), + module_limits.into(), + instance_limits.into(), stack_size, )?)) } From a7190764e12ad0e69a06a2296d2e8171edba8d58 Mon Sep 17 00:00:00 2001 From: Peter Huene Date: Fri, 5 Mar 2021 00:47:49 -0800 Subject: [PATCH 26/33] More code review changes. * Add more overflow checks in table/memory initialization. * Comment for `with_allocation_strategy` to explain ignored `Config` options. * Fix Wasmtime `Table` to not panic for type mismatches in `fill`/`copy`. * Add tests for that fix. --- crates/runtime/src/instance/allocator.rs | 176 ++++++++++-------- crates/wasmtime/src/config.rs | 6 + crates/wasmtime/src/externals.rs | 10 +- .../wasmtime/src/trampoline/create_handle.rs | 2 + tests/all/table.rs | 39 ++++ 5 files changed, 154 insertions(+), 79 deletions(-) diff --git a/crates/runtime/src/instance/allocator.rs b/crates/runtime/src/instance/allocator.rs index c70105dfb528..31ced83f5b38 100644 --- a/crates/runtime/src/instance/allocator.rs +++ b/crates/runtime/src/instance/allocator.rs @@ -168,33 +168,45 @@ pub unsafe trait InstanceAllocator: Send + Sync { unsafe fn deallocate_fiber_stack(&self, stack: *mut u8); } -fn get_table_init_start(init: &TableInitializer, instance: &Instance) -> usize { - let mut start = init.offset; - - if let Some(base) = init.base { - let val = unsafe { - if let Some(def_index) = instance.module.defined_global_index(base) { - *instance.global(def_index).as_u32() - } else { - *(*instance.imported_global(base).from).as_u32() - } - }; - start += usize::try_from(val).unwrap(); - } +fn get_table_init_start( + init: &TableInitializer, + instance: &Instance, +) -> Result { + match init.base { + Some(base) => { + let val = unsafe { + if let Some(def_index) = instance.module.defined_global_index(base) { + *instance.global(def_index).as_u32() + } else { + *(*instance.imported_global(base).from).as_u32() + } + }; - start + init.offset.checked_add(val as usize).ok_or_else(|| { + InstantiationError::Link(LinkError( + "element segment global base overflows".to_owned(), + )) + }) + } + None => Ok(init.offset), + } } fn check_table_init_bounds(instance: &Instance) -> Result<(), InstantiationError> { for init in &instance.module.table_initializers { - let start = get_table_init_start(init, instance); let table = instance.get_table(init.table_index); + let start = get_table_init_start(init, instance)?; + let end = start.checked_add(init.elements.len()); - let size = usize::try_from(table.size()).unwrap(); - if size < start + init.elements.len() { - return Err(InstantiationError::Link(LinkError( - "table out of bounds: elements segment does not fit".to_owned(), - ))); + match end { + Some(end) if end <= table.size() as usize => { + // Initializer is in bounds + } + _ => { + return Err(InstantiationError::Link(LinkError( + "table out of bounds: elements segment does not fit".to_owned(), + ))) + } } } @@ -203,53 +215,59 @@ fn check_table_init_bounds(instance: &Instance) -> Result<(), InstantiationError fn initialize_tables(instance: &Instance) -> Result<(), InstantiationError> { for init in &instance.module.table_initializers { - let start = get_table_init_start(init, instance); let table = instance.get_table(init.table_index); - - if start - .checked_add(init.elements.len()) - .map_or(true, |end| end > table.size() as usize) - { - return Err(InstantiationError::Trap(Trap::wasm( - ir::TrapCode::TableOutOfBounds, - ))); - } - - for (i, func_idx) in init.elements.iter().enumerate() { - let item = match table.element_type() { - TableElementType::Func => instance - .get_caller_checked_anyfunc(*func_idx) - .map_or(ptr::null_mut(), |f: &VMCallerCheckedAnyfunc| { - f as *const VMCallerCheckedAnyfunc as *mut VMCallerCheckedAnyfunc - }) - .into(), - TableElementType::Val(_) => { - assert!(*func_idx == FuncIndex::reserved_value()); - TableElement::ExternRef(None) + let start = get_table_init_start(init, instance)?; + let end = start.checked_add(init.elements.len()); + + match end { + Some(end) if end <= table.size() as usize => { + for (i, func_idx) in init.elements.iter().enumerate() { + let item = match table.element_type() { + TableElementType::Func => instance + .get_caller_checked_anyfunc(*func_idx) + .map_or(ptr::null_mut(), |f: &VMCallerCheckedAnyfunc| { + f as *const VMCallerCheckedAnyfunc as *mut VMCallerCheckedAnyfunc + }) + .into(), + TableElementType::Val(_) => { + assert!(*func_idx == FuncIndex::reserved_value()); + TableElement::ExternRef(None) + } + }; + table.set(u32::try_from(start + i).unwrap(), item).unwrap(); } - }; - table.set(u32::try_from(start + i).unwrap(), item).unwrap(); + } + _ => { + return Err(InstantiationError::Trap(Trap::wasm( + ir::TrapCode::TableOutOfBounds, + ))) + } } } Ok(()) } -fn get_memory_init_start(init: &MemoryInitializer, instance: &Instance) -> usize { - let mut start = init.offset; +fn get_memory_init_start( + init: &MemoryInitializer, + instance: &Instance, +) -> Result { + match init.base { + Some(base) => { + let val = unsafe { + if let Some(def_index) = instance.module.defined_global_index(base) { + *instance.global(def_index).as_u32() + } else { + *(*instance.imported_global(base).from).as_u32() + } + }; - if let Some(base) = init.base { - let val = unsafe { - if let Some(def_index) = instance.module.defined_global_index(base) { - *instance.global(def_index).as_u32() - } else { - *(*instance.imported_global(base).from).as_u32() - } - }; - start += usize::try_from(val).unwrap(); + init.offset.checked_add(val as usize).ok_or_else(|| { + InstantiationError::Link(LinkError("data segment global base overflows".to_owned())) + }) + } + None => Ok(init.offset), } - - start } unsafe fn get_memory_slice<'instance>( @@ -267,7 +285,7 @@ unsafe fn get_memory_slice<'instance>( let foreign_index = foreign_instance.memory_index(foreign_memory); foreign_instance.memory(foreign_index) }; - slice::from_raw_parts_mut(memory.base, memory.current_length) + &mut *ptr::slice_from_raw_parts_mut(memory.base, memory.current_length) } fn check_memory_init_bounds( @@ -275,13 +293,18 @@ fn check_memory_init_bounds( initializers: &[MemoryInitializer], ) -> Result<(), InstantiationError> { for init in initializers { - let start = get_memory_init_start(init, instance); - unsafe { - let mem_slice = get_memory_slice(init, instance); - if mem_slice.get_mut(start..start + init.data.len()).is_none() { + let memory = instance.get_memory(init.memory_index); + let start = get_memory_init_start(init, instance)?; + let end = start.checked_add(init.data.len()); + + match end { + Some(end) if end <= memory.current_length => { + // Initializer is in bounds + } + _ => { return Err(InstantiationError::Link(LinkError( "memory out of bounds: data segment does not fit".into(), - ))); + ))) } } } @@ -295,22 +318,19 @@ fn initialize_memories( ) -> Result<(), InstantiationError> { for init in initializers { let memory = instance.get_memory(init.memory_index); + let start = get_memory_init_start(init, instance)?; + let end = start.checked_add(init.data.len()); - let start = get_memory_init_start(init, instance); - if start - .checked_add(init.data.len()) - .map_or(true, |end| end > memory.current_length) - { - return Err(InstantiationError::Trap(Trap::wasm( - ir::TrapCode::HeapOutOfBounds, - ))); - } - - unsafe { - let mem_slice = get_memory_slice(init, instance); - let end = start + init.data.len(); - let to_init = &mut mem_slice[start..end]; - to_init.copy_from_slice(&init.data); + match end { + Some(end) if end <= memory.current_length => { + let mem_slice = unsafe { get_memory_slice(init, instance) }; + mem_slice[start..end].copy_from_slice(&init.data); + } + _ => { + return Err(InstantiationError::Trap(Trap::wasm( + ir::TrapCode::HeapOutOfBounds, + ))) + } } } diff --git a/crates/wasmtime/src/config.rs b/crates/wasmtime/src/config.rs index fdce7b9202f7..f3135dd6c8ed 100644 --- a/crates/wasmtime/src/config.rs +++ b/crates/wasmtime/src/config.rs @@ -826,6 +826,12 @@ impl Config { } /// Sets the instance allocation strategy to use. + /// + /// When using the pooling instance allocation strategy, all linear memories will be created as "static". + /// + /// This means the [`Config::static_memory_maximum_size`] and [`Config::static_memory_guard_size`] options + /// will be ignored in favor of [`InstanceLimits::memory_reservation_size`] when the pooling instance + /// allocation strategy is used. pub fn with_allocation_strategy( &mut self, strategy: InstanceAllocationStrategy, diff --git a/crates/wasmtime/src/externals.rs b/crates/wasmtime/src/externals.rs index e71fb7d75c4b..665acf9f02cb 100644 --- a/crates/wasmtime/src/externals.rs +++ b/crates/wasmtime/src/externals.rs @@ -547,10 +547,13 @@ impl Table { bail!("cross-`Store` table copies are not supported"); } + if dst_table.ty() != src_table.ty() { + bail!("tables do not have the same element type"); + } + // NB: We must use the `dst_table`'s `wasmtime_handle` for the // `dst_table_index` and vice versa for `src_table` since each table can // come from different modules. - let dst_table_index = dst_table.wasmtime_table_index(); let dst_table_index = dst_table.instance.get_defined_table(dst_table_index); @@ -579,6 +582,11 @@ impl Table { bail!("cross-`Store` table fills are not supported"); } + // Ensure the fill value is the correct type + if self.ty().element() != &val.ty() { + bail!("mismatched element fill type"); + } + let table_index = self.wasmtime_table_index(); self.instance .handle diff --git a/crates/wasmtime/src/trampoline/create_handle.rs b/crates/wasmtime/src/trampoline/create_handle.rs index 71595ff7296a..4ecf261b1e56 100644 --- a/crates/wasmtime/src/trampoline/create_handle.rs +++ b/crates/wasmtime/src/trampoline/create_handle.rs @@ -27,6 +27,8 @@ pub(crate) fn create_handle( unsafe { // Use the default allocator when creating handles associated with host objects + // The configured instance allocator should only be used when creating module instances + // as we don't want host objects to count towards instance limits. let handle = store .engine() .config() diff --git a/tests/all/table.rs b/tests/all/table.rs index 26830c4ea9bf..abdddccc4acc 100644 --- a/tests/all/table.rs +++ b/tests/all/table.rs @@ -11,3 +11,42 @@ fn get_none() { } assert!(table.get(1).is_none()); } + +#[test] +fn fill_wrong() { + let store = Store::default(); + let ty = TableType::new(ValType::FuncRef, Limits::new(1, None)); + let table = Table::new(&store, ty, Val::FuncRef(None)).unwrap(); + assert_eq!( + table + .fill(0, Val::ExternRef(None), 1) + .map_err(|e| e.to_string()) + .unwrap_err(), + "mismatched element fill type" + ); + + let ty = TableType::new(ValType::ExternRef, Limits::new(1, None)); + let table = Table::new(&store, ty, Val::ExternRef(None)).unwrap(); + assert_eq!( + table + .fill(0, Val::FuncRef(None), 1) + .map_err(|e| e.to_string()) + .unwrap_err(), + "mismatched element fill type" + ); +} + +#[test] +fn copy_wrong() { + let store = Store::default(); + let ty = TableType::new(ValType::FuncRef, Limits::new(1, None)); + let table1 = Table::new(&store, ty, Val::FuncRef(None)).unwrap(); + let ty = TableType::new(ValType::ExternRef, Limits::new(1, None)); + let table2 = Table::new(&store, ty, Val::ExternRef(None)).unwrap(); + assert_eq!( + Table::copy(&table1, 0, &table2, 0, 1) + .map_err(|e| e.to_string()) + .unwrap_err(), + "tables do not have the same element type" + ); +} From 1a0493946dc936e501c8091ab0fb996db6f85354 Mon Sep 17 00:00:00 2001 From: Peter Huene Date: Fri, 5 Mar 2021 17:25:04 -0800 Subject: [PATCH 27/33] Make the storage of `wasmtime_runtime::Table` consistent. This change makes the storage of `Table` more internally consistent. Elements are stored as raw pointers for both static and dynamic table storage. Explicitly storing elements as pointers removes assumptions being made by the pooling allocator in terms of the size and default representation of the elements. However, care must be made to properly clone externrefs for table operations. --- crates/runtime/src/externref.rs | 29 +- .../runtime/src/instance/allocator/pooling.rs | 25 +- crates/runtime/src/memory.rs | 54 +- crates/runtime/src/table.rs | 505 +++++++++--------- 4 files changed, 341 insertions(+), 272 deletions(-) diff --git a/crates/runtime/src/externref.rs b/crates/runtime/src/externref.rs index b5a8ef30bb9d..7fd964b8e3d7 100644 --- a/crates/runtime/src/externref.rs +++ b/crates/runtime/src/externref.rs @@ -351,8 +351,35 @@ impl VMExternRef { ptr } + /// Consume this `VMExternRef` into a raw, untyped pointer. + /// + /// # Safety + /// + /// This method forgets self, so it is possible to create a leak of the + /// underlying reference counted data if not used carefully. + /// + /// Use `from_raw` to recreate the `VMExternRef`. + pub unsafe fn into_raw(self) -> *mut u8 { + let ptr = self.0.cast::().as_ptr(); + std::mem::forget(self); + ptr + } + + /// Recreate a `VMExternRef` from a pointer returned from a previous call to + /// `as_raw`. + /// + /// # Safety + /// + /// Unlike `clone_from_raw`, this does not increment the reference count of the + /// underlying data. It is not safe to continue to use the pointer passed to this + /// function. + pub unsafe fn from_raw(ptr: *mut u8) -> Self { + debug_assert!(!ptr.is_null()); + VMExternRef(NonNull::new_unchecked(ptr).cast()) + } + /// Recreate a `VMExternRef` from a pointer returned from a previous call to - /// `VMExternRef::as_raw`. + /// `as_raw`. /// /// # Safety /// diff --git a/crates/runtime/src/instance/allocator/pooling.rs b/crates/runtime/src/instance/allocator/pooling.rs index 18ddf2bc4211..277bea2b707c 100644 --- a/crates/runtime/src/instance/allocator/pooling.rs +++ b/crates/runtime/src/instance/allocator/pooling.rs @@ -11,7 +11,7 @@ use super::{ initialize_instance, initialize_vmcontext, FiberStackError, InstanceAllocationRequest, InstanceAllocator, InstanceHandle, InstantiationError, }; -use crate::{instance::Instance, table::max_table_element_size, Memory, Mmap, Table, VMContext}; +use crate::{instance::Instance, Memory, Mmap, Table, VMContext}; use anyhow::{anyhow, bail, Context, Result}; use rand::Rng; use std::cell::RefCell; @@ -427,8 +427,12 @@ impl InstancePool { let instance = unsafe { &mut *handle.instance }; // Decommit any linear memories that were used - for (mem, base) in instance.memories.values().zip(self.memories.get(index)) { - let size = (mem.size() * WASM_PAGE_SIZE) as usize; + for (memory, base) in instance.memories.values_mut().zip(self.memories.get(index)) { + let memory = mem::take(memory); + debug_assert!(memory.is_static()); + + let size = (memory.size() * WASM_PAGE_SIZE) as usize; + drop(memory); decommit_memory_pages(base, size).unwrap(); } @@ -436,13 +440,16 @@ impl InstancePool { instance.dropped_data.borrow_mut().clear(); // Decommit any tables that were used - let table_element_size = max_table_element_size(); - for (table, base) in instance.tables.values().zip(self.tables.get(index)) { + for (table, base) in instance.tables.values_mut().zip(self.tables.get(index)) { + let table = mem::take(table); + debug_assert!(table.is_static()); + let size = round_up_to_pow2( - table.size() as usize * table_element_size, + table.size() as usize * mem::size_of::<*mut u8>(), self.tables.page_size, ); + drop(table); decommit_table_pages(base, size).unwrap(); } @@ -503,12 +510,12 @@ impl InstancePool { for plan in (&module.table_plans.values().as_slice()[module.num_imported_tables..]).iter() { let base = tables.next().unwrap(); - commit_table_pages(base, max_elements as usize * max_table_element_size()) + commit_table_pages(base, max_elements as usize * mem::size_of::<*mut u8>()) .map_err(|e| InstantiationError::Resource(e.to_string()))?; instance .tables - .push(Table::new_static(plan, base, max_elements)); + .push(Table::new_static(plan, base as _, max_elements)); } let mut dropped_elements = instance.dropped_elements.borrow_mut(); @@ -646,7 +653,7 @@ impl TablePool { let table_size = if module_limits.table_elements > 0 { round_up_to_pow2( - max_table_element_size() + mem::size_of::<*mut u8>() .checked_mul(module_limits.table_elements as usize) .ok_or_else(|| anyhow!("table size exceeds addressable memory"))?, page_size, diff --git a/crates/runtime/src/memory.rs b/crates/runtime/src/memory.rs index ba7bab470034..945cb5796731 100644 --- a/crates/runtime/src/memory.rs +++ b/crates/runtime/src/memory.rs @@ -9,6 +9,7 @@ use more_asserts::{assert_ge, assert_le}; use std::cell::{Cell, RefCell}; use std::cmp::min; use std::convert::TryFrom; +use std::ptr; use wasmtime_environ::{MemoryPlan, MemoryStyle, WASM_MAX_PAGES, WASM_PAGE_SIZE}; /// A memory allocator @@ -184,16 +185,12 @@ enum MemoryStorage { } /// Represents an instantiation of a WebAssembly memory. -pub struct Memory { - storage: MemoryStorage, -} +pub struct Memory(MemoryStorage); impl Memory { /// Create a new dynamic (movable) memory instance for the specified plan. pub fn new_dynamic(plan: &MemoryPlan, creator: &dyn RuntimeMemoryCreator) -> Result { - Ok(Self { - storage: MemoryStorage::Dynamic(creator.new_memory(plan)?), - }) + Ok(Self(MemoryStorage::Dynamic(creator.new_memory(plan)?))) } /// Create a new static (immovable) memory instance for the specified plan. @@ -207,30 +204,37 @@ impl Memory { make_accessible(base, plan.memory.minimum as usize * WASM_PAGE_SIZE as usize)?; } - Ok(Self { - storage: MemoryStorage::Static { - base, - size: Cell::new(plan.memory.minimum), - maximum: min(plan.memory.maximum.unwrap_or(maximum), maximum), - make_accessible, - }, - }) + Ok(Self(MemoryStorage::Static { + base, + size: Cell::new(plan.memory.minimum), + maximum: min(plan.memory.maximum.unwrap_or(maximum), maximum), + make_accessible, + })) } /// Returns the number of allocated wasm pages. pub fn size(&self) -> u32 { - match &self.storage { + match &self.0 { MemoryStorage::Static { size, .. } => size.get(), MemoryStorage::Dynamic(mem) => mem.size(), } } + /// Returns whether or not the underlying storage of the memory is "static". + pub(crate) fn is_static(&self) -> bool { + if let MemoryStorage::Static { .. } = &self.0 { + true + } else { + false + } + } + /// Grow memory by the specified amount of wasm pages. /// /// Returns `None` if memory can't be grown by the specified amount /// of wasm pages. pub fn grow(&self, delta: u32) -> Option { - match &self.storage { + match &self.0 { MemoryStorage::Static { base, size, @@ -264,7 +268,7 @@ impl Memory { /// Return a `VMMemoryDefinition` for exposing the memory to compiled wasm code. pub fn vmmemory(&self) -> VMMemoryDefinition { - match &self.storage { + match &self.0 { MemoryStorage::Static { base, size, .. } => VMMemoryDefinition { base: *base, current_length: size.get() as usize * WASM_PAGE_SIZE as usize, @@ -273,3 +277,19 @@ impl Memory { } } } + +// The default memory representation is an empty memory that cannot grow. +impl Default for Memory { + fn default() -> Self { + fn make_accessible(_ptr: *mut u8, _len: usize) -> Result<()> { + unreachable!() + } + + Self(MemoryStorage::Static { + base: ptr::null_mut(), + size: Cell::new(0), + maximum: 0, + make_accessible, + }) + } +} diff --git a/crates/runtime/src/table.rs b/crates/runtime/src/table.rs index 0469c4ff3981..35f489b778c9 100644 --- a/crates/runtime/src/table.rs +++ b/crates/runtime/src/table.rs @@ -6,12 +6,15 @@ use crate::vmcontext::{VMCallerCheckedAnyfunc, VMTableDefinition}; use crate::{Trap, VMExternRef}; use std::cell::{Cell, RefCell}; use std::cmp::min; -use std::convert::{TryFrom, TryInto}; +use std::convert::TryInto; +use std::ops::Range; use std::ptr; use wasmtime_environ::wasm::TableElementType; -use wasmtime_environ::{ir, TablePlan, TableStyle}; +use wasmtime_environ::{ir, TablePlan}; /// An element going into or coming out of a table. +/// +/// Table elements are stored as pointers and are default-initialized with `ptr::null_mut`. #[derive(Clone, Debug)] pub enum TableElement { /// A `funcref`. @@ -20,24 +23,53 @@ pub enum TableElement { ExternRef(Option), } -impl TryFrom for *mut VMCallerCheckedAnyfunc { - type Error = (); - - fn try_from(e: TableElement) -> Result { - match e { - TableElement::FuncRef(f) => Ok(f), - _ => Err(()), +impl TableElement { + /// Consumes the given raw pointer into a table element. + /// + /// # Safety + /// + /// This is unsafe as it will *not* clone any externref, leaving the reference count unchanged. + /// + /// This should only be used if the raw pointer is no longer in use. + unsafe fn from_raw(ty: TableElementType, ptr: *mut u8) -> Self { + match ty { + TableElementType::Func => Self::FuncRef(ptr as _), + TableElementType::Val(_) => Self::ExternRef(if ptr.is_null() { + None + } else { + Some(VMExternRef::from_raw(ptr)) + }), } } -} -impl TryFrom for Option { - type Error = (); + /// Clones a table element from the underlying raw pointer. + /// + /// # Safety + /// + /// This is unsafe as it will clone any externref, incrementing the reference count. + unsafe fn clone_from_raw(ty: TableElementType, ptr: *mut u8) -> Self { + match ty { + TableElementType::Func => Self::FuncRef(ptr as _), + TableElementType::Val(_) => Self::ExternRef(if ptr.is_null() { + None + } else { + Some(VMExternRef::clone_from_raw(ptr)) + }), + } + } - fn try_from(e: TableElement) -> Result { - match e { - TableElement::ExternRef(x) => Ok(x), - _ => Err(()), + /// Consumes a table element into a raw pointer. + /// + /// # Safety + /// + /// This is unsafe as it will consume any underlying externref into a raw pointer without modifying + /// the reference count. + /// + /// Use `from_raw` to properly drop any table elements stored as raw pointers. + unsafe fn into_raw(self) -> *mut u8 { + match self { + Self::FuncRef(e) => e as _, + Self::ExternRef(e) => e.map(|e| e.into_raw()).unwrap_or(ptr::null_mut()), } } } @@ -60,107 +92,79 @@ impl From for TableElement { } } -#[derive(Debug)] -enum TableElements { - FuncRefs(Vec<*mut VMCallerCheckedAnyfunc>), - ExternRefs(Vec>), -} - -// Ideally this should be static assertion that table elements are pointer-sized -#[inline(always)] -pub(crate) fn max_table_element_size() -> usize { - debug_assert_eq!( - std::mem::size_of::<*mut VMCallerCheckedAnyfunc>(), - std::mem::size_of::<*const ()>() - ); - debug_assert_eq!( - std::mem::size_of::>(), - std::mem::size_of::<*const ()>() - ); - std::mem::size_of::<*const ()>() -} - #[derive(Debug)] enum TableStorage { Static { - data: *mut u8, + data: *mut *mut u8, size: Cell, ty: TableElementType, maximum: u32, }, Dynamic { - elements: RefCell, + elements: RefCell>, + ty: TableElementType, maximum: Option, }, } /// Represents an instance's table. #[derive(Debug)] -pub struct Table { - storage: TableStorage, -} +pub struct Table(TableStorage); impl Table { /// Create a new dynamic (movable) table instance for the specified table plan. pub fn new_dynamic(plan: &TablePlan) -> Self { - let min = usize::try_from(plan.table.minimum).unwrap(); - let elements = RefCell::new(match plan.table.ty { - TableElementType::Func => TableElements::FuncRefs(vec![ptr::null_mut(); min]), - TableElementType::Val(ty) => { - debug_assert_eq!(ty, crate::ref_type()); - TableElements::ExternRefs(vec![None; min]) - } - }); - - match plan.style { - TableStyle::CallerChecksSignature => Self { - storage: TableStorage::Dynamic { - elements, - maximum: plan.table.maximum, - }, - }, - } + let elements = RefCell::new(vec![ptr::null_mut(); plan.table.minimum as usize]); + let ty = plan.table.ty.clone(); + let maximum = plan.table.maximum; + Self(TableStorage::Dynamic { + elements, + ty, + maximum, + }) } /// Create a new static (immovable) table instance for the specified table plan. - pub fn new_static(plan: &TablePlan, data: *mut u8, maximum: u32) -> Self { - match plan.style { - TableStyle::CallerChecksSignature => Self { - storage: TableStorage::Static { - data, - size: Cell::new(plan.table.minimum), - ty: plan.table.ty.clone(), - maximum: min(plan.table.maximum.unwrap_or(maximum), maximum), - }, - }, - } + pub fn new_static(plan: &TablePlan, data: *mut *mut u8, maximum: u32) -> Self { + let size = Cell::new(plan.table.minimum); + let ty = plan.table.ty.clone(); + let maximum = min(plan.table.maximum.unwrap_or(maximum), maximum); + Self(TableStorage::Static { + data, + size, + ty, + maximum, + }) } /// Returns the type of the elements in this table. pub fn element_type(&self) -> TableElementType { - match &self.storage { + match &self.0 { TableStorage::Static { ty, .. } => *ty, - TableStorage::Dynamic { elements, .. } => match &*elements.borrow() { - TableElements::FuncRefs(_) => TableElementType::Func, - TableElements::ExternRefs(_) => TableElementType::Val(crate::ref_type()), - }, + TableStorage::Dynamic { ty, .. } => *ty, + } + } + + /// Returns whether or not the underlying storage of the table is "static". + pub(crate) fn is_static(&self) -> bool { + if let TableStorage::Static { .. } = &self.0 { + true + } else { + false } } /// Returns the number of allocated elements. pub fn size(&self) -> u32 { - match &self.storage { + match &self.0 { TableStorage::Static { size, .. } => size.get(), - TableStorage::Dynamic { elements, .. } => match &*elements.borrow() { - TableElements::FuncRefs(x) => x.len().try_into().unwrap(), - TableElements::ExternRefs(x) => x.len().try_into().unwrap(), - }, + TableStorage::Dynamic { elements, .. } => elements.borrow().len().try_into().unwrap(), } } /// Returns the maximum number of elements. pub fn maximum(&self) -> Option { - match &self.storage { + match &self.0 { TableStorage::Static { maximum, .. } => Some(*maximum), TableStorage::Dynamic { maximum, .. } => maximum.clone(), } @@ -170,31 +174,30 @@ impl Table { /// /// Returns a trap error on out-of-bounds accesses. pub fn fill(&self, dst: u32, val: TableElement, len: u32) -> Result<(), Trap> { - let start = dst; + let start = dst as usize; let end = start - .checked_add(len) + .checked_add(len as usize) .ok_or_else(|| Trap::wasm(ir::TrapCode::TableOutOfBounds))?; - if end > self.size() { + if end > self.size() as usize { return Err(Trap::wasm(ir::TrapCode::TableOutOfBounds)); } - match val { - TableElement::FuncRef(r) => unsafe { - self.with_funcrefs_mut(move |elements| { - let elements = elements.unwrap(); - elements[start as usize..end as usize].fill(r); - }); - }, - TableElement::ExternRef(r) => unsafe { - self.with_externrefs_mut(move |elements| { - let elements = elements.unwrap(); - elements[start as usize..end as usize].fill(r); - }); - }, - } + debug_assert!(self.type_matches(&val)); - Ok(()) + self.with_elements_mut(|elements| { + if let Some((last, elements)) = elements[start..end].split_last_mut() { + let ty = self.element_type(); + + for e in elements { + Self::set_raw(ty, e, val.clone()); + } + + Self::set_raw(self.element_type(), last, val); + } + + Ok(()) + }) } /// Grow table by the specified amount of elements. @@ -223,41 +226,34 @@ impl Table { } } - match &self.storage { + debug_assert!(self.type_matches(&init_value)); + + // First resize the storage and then fill with the init value + match &self.0 { TableStorage::Static { size, .. } => { size.set(new_size); - self.fill(old_size, init_value, delta) - .ok() - .map(|_| old_size) } TableStorage::Dynamic { elements, .. } => { - let new_len = usize::try_from(new_size).unwrap(); - - match &mut *elements.borrow_mut() { - TableElements::FuncRefs(x) => x.resize(new_len, init_value.try_into().ok()?), - TableElements::ExternRefs(x) => x.resize(new_len, init_value.try_into().ok()?), - } - - Some(old_size) + let mut elements = elements.borrow_mut(); + elements.resize(new_size as usize, ptr::null_mut()); } } + + self.fill(old_size, init_value, delta) + .expect("table should not be out of bounds"); + + Some(old_size) } /// Get reference to the specified element. /// /// Returns `None` if the index is out of bounds. pub fn get(&self, index: u32) -> Option { - unsafe { - match self.element_type() { - TableElementType::Func => self.with_funcrefs(|elements| { - elements.and_then(|e| e.get(index as usize).cloned().map(TableElement::FuncRef)) - }), - TableElementType::Val(_) => self.with_externrefs(|elements| { - elements - .and_then(|e| e.get(index as usize).cloned().map(TableElement::ExternRef)) - }), - } - } + self.with_elements(|elements| { + elements + .get(index as usize) + .map(|p| unsafe { TableElement::clone_from_raw(self.element_type(), *p) }) + }) } /// Set reference to the specified element. @@ -267,22 +263,15 @@ impl Table { /// Returns an error if `index` is out of bounds or if this table type does /// not match the element type. pub fn set(&self, index: u32, elem: TableElement) -> Result<(), ()> { - unsafe { - match self.element_type() { - TableElementType::Func => self.with_funcrefs_mut(move |elements| { - let elements = elements.ok_or(())?; - let e = elements.get_mut(index as usize).ok_or(())?; - *e = elem.try_into()?; - Ok(()) - }), - TableElementType::Val(_) => self.with_externrefs_mut(move |elements| { - let elements = elements.ok_or(())?; - let e = elements.get_mut(index as usize).ok_or(())?; - *e = elem.try_into()?; - Ok(()) - }), - } + if !self.type_matches(&elem) { + return Err(()); } + + self.with_elements_mut(|elements| { + let e = elements.get_mut(index as usize).ok_or(())?; + Self::set_raw(self.element_type(), e, elem); + Ok(()) + }) } /// Copy `len` elements from `src_table[src_index..]` into `dst_table[dst_index..]`. @@ -310,49 +299,19 @@ impl Table { return Err(Trap::wasm(ir::TrapCode::TableOutOfBounds)); } - // Check if the source and destination are the same table - // This ensures we don't `borrow` and `borrow_mut` the same underlying RefCell - let same_table = ptr::eq(dst_table, src_table); + debug_assert!( + dst_table.element_type() == src_table.element_type(), + "table element type mismatch" + ); let src_range = src_index as usize..src_index as usize + len as usize; let dst_range = dst_index as usize..dst_index as usize + len as usize; - unsafe { - match dst_table.element_type() { - TableElementType::Func => dst_table.with_funcrefs_mut(|dst| { - let dst = dst.unwrap(); - - if same_table { - dst.copy_within(src_range, dst_index as usize); - } else { - src_table.with_funcrefs(|src| { - let src = src.unwrap(); - dst[dst_range].copy_from_slice(&src[src_range]); - }) - } - }), - TableElementType::Val(_) => dst_table.with_externrefs_mut(|dst| { - let dst = dst.unwrap(); - - if same_table { - // As there's no `slice::clone_within` because cloning can't be done with memmove, use a loop - if dst_index <= src_index { - for (s, d) in (src_range).zip(dst_range) { - dst[d] = dst[s].clone(); - } - } else { - for (s, d) in src_range.rev().zip(dst_range.rev()) { - dst[d] = dst[s].clone(); - } - } - } else { - src_table.with_externrefs(|src| { - let src = src.unwrap(); - dst[dst_range].clone_from_slice(&src[src_range]); - }) - } - }), - } + // Check if the tables are the same as we cannot mutably borrow and also borrow the same `RefCell` + if ptr::eq(dst_table, src_table) { + Self::copy_elements_within(dst_table, dst_range, src_range); + } else { + Self::copy_elements(dst_table, src_table, dst_range, src_range); } Ok(()) @@ -360,97 +319,153 @@ impl Table { /// Return a `VMTableDefinition` for exposing the table to compiled wasm code. pub fn vmtable(&self) -> VMTableDefinition { - match &self.storage { + match &self.0 { TableStorage::Static { data, size, .. } => VMTableDefinition { - base: *data, + base: *data as _, current_elements: size.get(), }, - TableStorage::Dynamic { elements, .. } => match &*elements.borrow() { - TableElements::FuncRefs(x) => VMTableDefinition { - base: x.as_ptr() as *const u8 as _, - current_elements: x.len().try_into().unwrap(), - }, - TableElements::ExternRefs(x) => VMTableDefinition { - base: x.as_ptr() as *const u8 as _, - current_elements: x.len().try_into().unwrap(), - }, - }, + TableStorage::Dynamic { elements, .. } => { + let elements = elements.borrow(); + VMTableDefinition { + base: elements.as_ptr() as _, + current_elements: elements.len().try_into().unwrap(), + } + } } } - unsafe fn with_funcrefs(&self, with: F) -> R - where - F: FnOnce(Option<&[*mut VMCallerCheckedAnyfunc]>) -> R, - { - match &self.storage { - TableStorage::Static { data, size, ty, .. } => match ty { - TableElementType::Func => with(Some(std::slice::from_raw_parts( - *data as *const _, - size.get() as usize, - ))), - _ => with(None), - }, - TableStorage::Dynamic { elements, .. } => match &*elements.borrow() { - TableElements::FuncRefs(x) => with(Some(x.as_slice())), - _ => with(None), - }, + fn type_matches(&self, val: &TableElement) -> bool { + match (&val, self.element_type()) { + (TableElement::FuncRef(_), TableElementType::Func) => true, + (TableElement::ExternRef(_), TableElementType::Val(_)) => true, + _ => false, } } - unsafe fn with_funcrefs_mut(&self, with: F) -> R + fn with_elements(&self, f: F) -> R where - F: FnOnce(Option<&mut [*mut VMCallerCheckedAnyfunc]>) -> R, + F: FnOnce(&[*mut u8]) -> R, { - match &self.storage { - TableStorage::Static { data, size, ty, .. } => match ty { - TableElementType::Func => with(Some(std::slice::from_raw_parts_mut( - *data as *mut _, - size.get() as usize, - ))), - _ => with(None), - }, - TableStorage::Dynamic { elements, .. } => match &mut *elements.borrow_mut() { - TableElements::FuncRefs(x) => with(Some(x.as_mut_slice())), - _ => with(None), + match &self.0 { + TableStorage::Static { data, size, .. } => unsafe { + f(std::slice::from_raw_parts(*data, size.get() as usize)) }, + TableStorage::Dynamic { elements, .. } => { + let elements = elements.borrow(); + f(elements.as_slice()) + } } } - unsafe fn with_externrefs(&self, with: F) -> R + fn with_elements_mut(&self, f: F) -> R where - F: FnOnce(Option<&[Option]>) -> R, + F: FnOnce(&mut [*mut u8]) -> R, { - match &self.storage { - TableStorage::Static { data, size, ty, .. } => match ty { - TableElementType::Val(_) => with(Some(std::slice::from_raw_parts( - *data as *const _, - size.get() as usize, - ))), - _ => with(None), - }, - TableStorage::Dynamic { elements, .. } => match &*elements.borrow() { - TableElements::ExternRefs(x) => with(Some(x.as_slice())), - _ => with(None), + match &self.0 { + TableStorage::Static { data, size, .. } => unsafe { + f(std::slice::from_raw_parts_mut(*data, size.get() as usize)) }, + TableStorage::Dynamic { elements, .. } => { + let mut elements = elements.borrow_mut(); + f(elements.as_mut_slice()) + } } } - unsafe fn with_externrefs_mut(&self, with: F) -> R - where - F: FnOnce(Option<&mut [Option]>) -> R, - { - match &self.storage { - TableStorage::Static { data, size, ty, .. } => match ty { - TableElementType::Val(_) => with(Some(std::slice::from_raw_parts_mut( - *data as *mut _, - size.get() as usize, - ))), - _ => with(None), - }, - TableStorage::Dynamic { elements, .. } => match &mut *elements.borrow_mut() { - TableElements::ExternRefs(x) => with(Some(x.as_mut_slice())), - _ => with(None), - }, + fn set_raw(ty: TableElementType, e: &mut *mut u8, val: TableElement) { + unsafe { + // Drop the existing element + let _ = TableElement::from_raw(ty, *e); + *e = val.into_raw(); } } + + fn copy_elements( + dst_table: &Self, + src_table: &Self, + dst_range: Range, + src_range: Range, + ) { + // This can only be used when copying between different tables + debug_assert!(!ptr::eq(dst_table, src_table)); + + let ty = dst_table.element_type(); + + match ty { + TableElementType::Func => { + // `funcref` are `Copy`, so just do a mempcy + dst_table.with_elements_mut(|dst| { + src_table.with_elements(|src| dst[dst_range].copy_from_slice(&src[src_range])) + }); + } + TableElementType::Val(_) => { + // We need to clone each `externref` + dst_table.with_elements_mut(|dst| { + src_table.with_elements(|src| { + for (s, d) in src_range.zip(dst_range) { + let elem = unsafe { TableElement::clone_from_raw(ty, src[s]) }; + Self::set_raw(ty, &mut dst[d], elem); + } + }) + }); + } + } + } + + fn copy_elements_within(table: &Self, dst_range: Range, src_range: Range) { + let ty = table.element_type(); + + match ty { + TableElementType::Func => { + // `funcref` are `Copy`, so just do a memmove + table.with_elements_mut(|dst| dst.copy_within(src_range, dst_range.start)); + } + TableElementType::Val(_) => { + // We need to clone each `externref` while handling overlapping ranges + table.with_elements_mut(|dst| { + if dst_range.start <= src_range.start { + for (s, d) in src_range.zip(dst_range) { + let elem = unsafe { TableElement::clone_from_raw(ty, dst[s]) }; + Self::set_raw(ty, &mut dst[d], elem); + } + } else { + for (s, d) in src_range.rev().zip(dst_range.rev()) { + let elem = unsafe { TableElement::clone_from_raw(ty, dst[s]) }; + Self::set_raw(ty, &mut dst[d], elem); + } + } + }); + } + } + } +} + +impl Drop for Table { + fn drop(&mut self) { + let ty = self.element_type(); + + // funcref tables can skip this + if let TableElementType::Func = ty { + return; + } + + // Properly drop any table elements stored in the table + self.with_elements(|elements| { + for element in elements.iter() { + let _ = unsafe { TableElement::from_raw(ty, *element) }; + } + }); + } +} + +// The default table representation is an empty funcref table that cannot grow. +impl Default for Table { + fn default() -> Self { + Self(TableStorage::Static { + data: std::ptr::null_mut(), + size: Cell::new(0), + ty: TableElementType::Func, + maximum: 0, + }) + } } From 9801c681ce1bdceff9e7a2f1b6ef5a9fd4b94d43 Mon Sep 17 00:00:00 2001 From: Peter Huene Date: Fri, 5 Mar 2021 18:05:02 -0800 Subject: [PATCH 28/33] Fail module translation for segments with overflowing offset+length. This commit fails translation of modules that have an segment offset, when added to the data length, overflows. --- cranelift/wasm/src/sections_translator.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/cranelift/wasm/src/sections_translator.rs b/cranelift/wasm/src/sections_translator.rs index 3906c02393c9..a674658358cc 100644 --- a/cranelift/wasm/src/sections_translator.rs +++ b/cranelift/wasm/src/sections_translator.rs @@ -401,6 +401,12 @@ pub fn parse_element_section<'data>( )); } }; + // Check for offset + len overflow + if offset.checked_add(segments.len()).is_none() { + return Err(wasm_unsupported!( + "element segment offset and length overflows" + )); + } environ.declare_table_elements( TableIndex::from_u32(table_index), base, @@ -447,6 +453,12 @@ pub fn parse_data_section<'data>( )) } }; + // Check for offset + len overflow + if offset.checked_add(data.len()).is_none() { + return Err(wasm_unsupported!( + "data segment offset and length overflows" + )); + } environ.declare_data_initialization( MemoryIndex::from_u32(memory_index), base, From 57dfe99aa5b2b3673871c716c6a7a6c0010f8f19 Mon Sep 17 00:00:00 2001 From: Peter Huene Date: Fri, 5 Mar 2021 20:55:51 -0800 Subject: [PATCH 29/33] Run wast tests with both instance allocators. This commit adds a "pooling" variant to the wast tests that uses the pooling instance allocation strategy. This should help with the test coverage of the pooling instance allocator. --- .github/workflows/main.yml | 1 + build.rs | 20 ++++++++++++++++---- tests/all/wast.rs | 31 +++++++++++++++++++++++++++++-- 3 files changed, 46 insertions(+), 6 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 8b69ccfec2a1..47efbda923e5 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -305,6 +305,7 @@ jobs: - run: | cargo test --features uffd -p wasmtime-runtime instance::allocator::pooling cargo test --features uffd -p wasmtime-cli pooling_allocator + cargo test --features uffd -p wasmtime-cli wast::Cranelift if: matrix.os == 'ubuntu-latest' env: RUST_BACKTRACE: 1 diff --git a/build.rs b/build.rs index 53931669139e..a41bbf844b91 100644 --- a/build.rs +++ b/build.rs @@ -111,7 +111,8 @@ fn test_directory( let testsuite = &extract_name(path); for entry in dir_entries.iter() { - write_testsuite_tests(out, entry, testsuite, strategy)?; + write_testsuite_tests(out, entry, testsuite, strategy, false)?; + write_testsuite_tests(out, entry, testsuite, strategy, true)?; } Ok(dir_entries.len()) @@ -148,6 +149,7 @@ fn write_testsuite_tests( path: impl AsRef, testsuite: &str, strategy: &str, + pooling: bool, ) -> anyhow::Result<()> { let path = path.as_ref(); let testname = extract_name(path); @@ -160,14 +162,24 @@ fn write_testsuite_tests( )?; } else if ignore(testsuite, &testname, strategy) { writeln!(out, "#[ignore]")?; + } else if pooling { + // Ignore on aarch64 due to using QEMU for running tests (limited memory) + writeln!(out, r#"#[cfg_attr(target_arch = "aarch64", ignore)]"#)?; } - writeln!(out, "fn r#{}() {{", &testname)?; + + writeln!( + out, + "fn r#{}{}() {{", + &testname, + if pooling { "_pooling" } else { "" } + )?; writeln!(out, " let _ = env_logger::try_init();")?; writeln!( out, - " crate::wast::run_wast(r#\"{}\"#, crate::wast::Strategy::{}).unwrap();", + " crate::wast::run_wast(r#\"{}\"#, crate::wast::Strategy::{}, {}).unwrap();", path.display(), - strategy + strategy, + pooling )?; writeln!(out, "}}")?; writeln!(out)?; diff --git a/tests/all/wast.rs b/tests/all/wast.rs index 362dca274b0d..fd8a4f3a5b8a 100644 --- a/tests/all/wast.rs +++ b/tests/all/wast.rs @@ -1,5 +1,8 @@ use std::path::Path; -use wasmtime::{Config, Engine, Store, Strategy}; +use wasmtime::{ + Config, Engine, InstanceAllocationStrategy, InstanceLimits, ModuleLimits, + PoolingAllocationStrategy, Store, Strategy, +}; use wasmtime_wast::WastContext; include!(concat!(env!("OUT_DIR"), "/wast_testsuite_tests.rs")); @@ -7,7 +10,7 @@ include!(concat!(env!("OUT_DIR"), "/wast_testsuite_tests.rs")); // Each of the tests included from `wast_testsuite_tests` will call this // function which actually executes the `wast` test suite given the `strategy` // to compile it. -fn run_wast(wast: &str, strategy: Strategy) -> anyhow::Result<()> { +fn run_wast(wast: &str, strategy: Strategy, pooling: bool) -> anyhow::Result<()> { let wast = Path::new(wast); let simd = wast.iter().any(|s| s == "simd"); @@ -44,6 +47,30 @@ fn run_wast(wast: &str, strategy: Strategy) -> anyhow::Result<()> { cfg.static_memory_maximum_size(0); } + if pooling { + // The limits here are crafted such that the wast tests should pass. + // However, these limits may become insufficient in the future as the wast tests change. + // If a wast test fails because of a limit being "exceeded" or if memory/table + // fails to grow, the values here will need to be adjusted. + cfg.with_allocation_strategy(InstanceAllocationStrategy::Pooling { + strategy: PoolingAllocationStrategy::NextAvailable, + module_limits: ModuleLimits { + imported_memories: 2, + imported_tables: 2, + imported_globals: 11, + memories: 2, + tables: 4, + globals: 11, + memory_pages: 805, + ..Default::default() + }, + instance_limits: InstanceLimits { + count: 450, + ..Default::default() + }, + })?; + } + let store = Store::new(&Engine::new(&cfg)); let mut wast_context = WastContext::new(store); wast_context.register_spectest()?; From 8e51aefb2c842fb9861a4288dd78946b4fc2afc4 Mon Sep 17 00:00:00 2001 From: Peter Huene Date: Fri, 5 Mar 2021 21:15:19 -0800 Subject: [PATCH 30/33] Extract out finding a passive segment. This commit extracts out a common pattern of finding a passive element or data segment into a `find_passive_segment` method. --- crates/runtime/src/instance.rs | 62 +++++++++++++++++++--------------- 1 file changed, 34 insertions(+), 28 deletions(-) diff --git a/crates/runtime/src/instance.rs b/crates/runtime/src/instance.rs index 9475e4751745..60e9e662ad11 100644 --- a/crates/runtime/src/instance.rs +++ b/crates/runtime/src/instance.rs @@ -19,7 +19,9 @@ use more_asserts::assert_lt; use std::alloc::Layout; use std::any::Any; use std::cell::RefCell; +use std::collections::HashMap; use std::convert::TryFrom; +use std::hash::Hash; use std::ptr::NonNull; use std::rc::Rc; use std::sync::Arc; @@ -545,6 +547,22 @@ impl Instance { self.vmctx_plus_offset(self.offsets.vmctx_anyfunc(index)) } + fn find_passive_segment<'a, I, D, T>( + index: I, + index_map: &HashMap, + data: &'a Vec, + dropped: &RefCell>, + ) -> &'a [T] + where + D: AsRef<[T]>, + I: EntityRef + Hash, + { + match index_map.get(&index) { + Some(index) if !dropped.borrow().contains(I::new(*index)) => data[*index].as_ref(), + _ => &[], + } + } + /// The `table.init` operation: initializes a portion of a table with a /// passive element. /// @@ -563,25 +581,17 @@ impl Instance { // https://webassembly.github.io/bulk-memory-operations/core/exec/instructions.html#exec-table-init let table = self.get_table(table_index); - let elem_index = self.module.passive_elements_map.get(&elem_index); - let elem = match elem_index { - Some(index) => { - if self - .dropped_elements - .borrow() - .contains(ElemIndex::new(*index)) - { - &[] - } else { - self.module.passive_elements[*index].as_ref() - } - } - None => &[], - }; + + let elements = Self::find_passive_segment( + elem_index, + &self.module.passive_elements_map, + &self.module.passive_elements, + &self.dropped_elements, + ); if src .checked_add(len) - .map_or(true, |n| n as usize > elem.len()) + .map_or(true, |n| n as usize > elements.len()) || dst.checked_add(len).map_or(true, |m| m > table.size()) { return Err(Trap::wasm(ir::TrapCode::TableOutOfBounds)); @@ -590,7 +600,7 @@ impl Instance { // TODO(#983): investigate replacing this get/set loop with a `memcpy`. for (dst, src) in (dst..dst + len).zip(src..src + len) { let elem = self - .get_caller_checked_anyfunc(elem[src as usize]) + .get_caller_checked_anyfunc(elements[src as usize]) .map_or(ptr::null_mut(), |f: &VMCallerCheckedAnyfunc| { f as *const VMCallerCheckedAnyfunc as *mut _ }); @@ -733,17 +743,13 @@ impl Instance { // https://webassembly.github.io/bulk-memory-operations/core/exec/instructions.html#exec-memory-init let memory = self.get_memory(memory_index); - let data_index = self.module.passive_data_map.get(&data_index); - let data = match data_index { - Some(index) => { - if self.dropped_data.borrow().contains(DataIndex::new(*index)) { - &[] - } else { - self.module.passive_data[*index].as_ref() - } - } - None => &[], - }; + + let data = Self::find_passive_segment( + data_index, + &self.module.passive_data_map, + &self.module.passive_data, + &self.dropped_data, + ); if src .checked_add(len) From 7a93132ffa4fbb3ea38e3821dc4cb4a93d61f782 Mon Sep 17 00:00:00 2001 From: Peter Huene Date: Mon, 8 Mar 2021 09:04:13 -0800 Subject: [PATCH 31/33] Code review feedback. * Improve comments. * Drop old table element *after* updating the table. * Extract out the same `cfg_if!` to a single constant. --- .../src/instance/allocator/pooling/uffd.rs | 8 ++++--- .../src/instance/allocator/pooling/unix.rs | 4 ++++ crates/runtime/src/table.rs | 10 +++++---- crates/wasmtime/src/module.rs | 22 ++++--------------- 4 files changed, 19 insertions(+), 25 deletions(-) diff --git a/crates/runtime/src/instance/allocator/pooling/uffd.rs b/crates/runtime/src/instance/allocator/pooling/uffd.rs index 18e9eb3c2c82..93ffaf1504ef 100644 --- a/crates/runtime/src/instance/allocator/pooling/uffd.rs +++ b/crates/runtime/src/instance/allocator/pooling/uffd.rs @@ -45,7 +45,11 @@ fn decommit(addr: *mut u8, len: usize) -> Result<()> { } unsafe { - // On Linux, this is enough to cause the kernel to initialize the pages to 0 on next access + // On Linux, this tells the kernel to discard the backing of the pages in the range. + // If the discarded pages are part of a uffd region, then the next access will fault + // and the user fault handler will receive the event. + // If the pages are not monitored by uffd, the kernel will zero the page on next access, + // as if it were mmap'd for the first time. if libc::madvise(addr as _, len, libc::MADV_DONTNEED) != 0 { bail!( "madvise failed to decommit: {}", @@ -93,8 +97,6 @@ pub fn decommit_stack_pages(addr: *mut u8, len: usize) -> Result<()> { /// With uffd, however, the potentially accessible pages of the each linear memory are made `READ_WRITE` and /// the page fault handler will detect an out of bounds access and treat the page, temporarily, /// as a guard page. -/// -/// This me pub(super) fn initialize_memory_pool(pool: &MemoryPool) -> Result<()> { if pool.memory_size == 0 || pool.max_wasm_pages == 0 { return Ok(()); diff --git a/crates/runtime/src/instance/allocator/pooling/unix.rs b/crates/runtime/src/instance/allocator/pooling/unix.rs index 957aea8e1b4b..d172f411ebbf 100644 --- a/crates/runtime/src/instance/allocator/pooling/unix.rs +++ b/crates/runtime/src/instance/allocator/pooling/unix.rs @@ -5,6 +5,10 @@ fn decommit(addr: *mut u8, len: usize, protect: bool) -> Result<()> { return Ok(()); } + // By creating a new mapping at the same location, this will discard the + // mapping for the pages in the given range. + // The new mapping will be to the CoW zero page, so this effectively + // zeroes the pages. if unsafe { libc::mmap( addr as _, diff --git a/crates/runtime/src/table.rs b/crates/runtime/src/table.rs index 35f489b778c9..8c857add455a 100644 --- a/crates/runtime/src/table.rs +++ b/crates/runtime/src/table.rs @@ -372,11 +372,13 @@ impl Table { } } - fn set_raw(ty: TableElementType, e: &mut *mut u8, val: TableElement) { + fn set_raw(ty: TableElementType, elem: &mut *mut u8, val: TableElement) { unsafe { - // Drop the existing element - let _ = TableElement::from_raw(ty, *e); - *e = val.into_raw(); + let old = *elem; + *elem = val.into_raw(); + + // Drop the old element + let _ = TableElement::from_raw(ty, old); } } diff --git a/crates/wasmtime/src/module.rs b/crates/wasmtime/src/module.rs index f87b28f6fe49..e3c9213adc06 100644 --- a/crates/wasmtime/src/module.rs +++ b/crates/wasmtime/src/module.rs @@ -307,6 +307,8 @@ impl Module { /// # } /// ``` pub fn from_binary(engine: &Engine, binary: &[u8]) -> Result { + const USE_PAGED_MEM_INIT: bool = cfg!(all(feature = "uffd", target_os = "linux")); + cfg_if::cfg_if! { if #[cfg(feature = "cache")] { let (main_module, artifacts, types) = ModuleCacheEntry::new( @@ -314,27 +316,11 @@ impl Module { engine.cache_config(), ) .get_data((engine.compiler(), binary), |(compiler, binary)| { - cfg_if::cfg_if! { - if #[cfg(all(feature = "uffd", target_os = "linux"))] { - let use_paged_mem_init = true; - } else { - let use_paged_mem_init = false; - } - }; - - CompilationArtifacts::build(compiler, binary, use_paged_mem_init) + CompilationArtifacts::build(compiler, binary, USE_PAGED_MEM_INIT) })?; } else { - cfg_if::cfg_if! { - if #[cfg(all(feature = "uffd", target_os = "linux"))] { - let use_paged_mem_init = true; - } else { - let use_paged_mem_init = false; - } - }; - let (main_module, artifacts, types) = - CompilationArtifacts::build(engine.compiler(), binary, use_paged_mem_init)?; + CompilationArtifacts::build(engine.compiler(), binary, USE_PAGED_MEM_INIT)?; } }; From 5fa0f8d46992e89ccb1c041360e9417d24d62f0f Mon Sep 17 00:00:00 2001 From: Peter Huene Date: Mon, 8 Mar 2021 09:23:17 -0800 Subject: [PATCH 32/33] Move linear memory faulted guard page tracking into `Memory`. This commit moves the tracking for faulted guard pages in a linear memory into `Memory`. --- crates/runtime/src/instance.rs | 43 ------------- crates/runtime/src/instance/allocator.rs | 2 - .../runtime/src/instance/allocator/pooling.rs | 18 +++--- .../src/instance/allocator/pooling/uffd.rs | 8 ++- crates/runtime/src/memory.rs | 62 +++++++++++++++++++ 5 files changed, 76 insertions(+), 57 deletions(-) diff --git a/crates/runtime/src/instance.rs b/crates/runtime/src/instance.rs index 60e9e662ad11..9273870367d7 100644 --- a/crates/runtime/src/instance.rs +++ b/crates/runtime/src/instance.rs @@ -69,11 +69,6 @@ pub(crate) struct Instance { /// Hosts can store arbitrary per-instance information here. host_state: Box, - /// Stores linear memory guard page faults for the pooling allocator with uffd enabled. - /// These pages need to be reset after the signal handler generates the out-of-bounds trap. - #[cfg(all(feature = "uffd", target_os = "linux"))] - guard_page_faults: RefCell anyhow::Result<()>)>>, - /// Additional context used by compiled wasm code. This field is last, and /// represents a dynamically-sized array that extends beyond the nominal /// end of the struct (similar to a flexible array member). @@ -383,14 +378,6 @@ impl Instance { /// Returns `None` if memory can't be grown by the specified amount /// of pages. pub(crate) fn memory_grow(&self, memory_index: DefinedMemoryIndex, delta: u32) -> Option { - // Reset all guard pages before growing any memory when using the uffd feature. - // The uffd feature induces a trap when a fault on a linear memory page is determined to be out-of-bounds. - // It does this by temporarily setting the protection level to `NONE` to cause the kernel to signal SIGBUS. - // Because instances might still be used after a trap, this resets the page back to the expected protection - // level (READ_WRITE) for the uffd implementation. - #[cfg(all(feature = "uffd", target_os = "linux"))] - self.reset_guard_pages().ok()?; - let result = self .memories .get(memory_index) @@ -822,36 +809,6 @@ impl Instance { (foreign_table_index, foreign_instance) } } - - /// Records a faulted guard page. - /// - /// This is used to track faulted guard pages that need to be reset. - #[cfg(all(feature = "uffd", target_os = "linux"))] - pub(crate) fn record_guard_page_fault( - &self, - page_addr: *mut u8, - size: usize, - reset: fn(*mut u8, usize) -> anyhow::Result<()>, - ) { - self.guard_page_faults - .borrow_mut() - .push((page_addr, size, reset)); - } - - /// Resets previously faulted guard pages. - /// - /// This is used to reset the protection of any guard pages that were previously faulted. - /// - /// Resetting the guard pages is required before growing memory. - #[cfg(all(feature = "uffd", target_os = "linux"))] - pub(crate) fn reset_guard_pages(&self) -> anyhow::Result<()> { - let mut faults = self.guard_page_faults.borrow_mut(); - for (addr, len, reset) in faults.drain(..) { - reset(addr, len)?; - } - - Ok(()) - } } /// A handle holding an `Instance` of a WebAssembly module. diff --git a/crates/runtime/src/instance/allocator.rs b/crates/runtime/src/instance/allocator.rs index 31ced83f5b38..e855f4154317 100644 --- a/crates/runtime/src/instance/allocator.rs +++ b/crates/runtime/src/instance/allocator.rs @@ -602,8 +602,6 @@ unsafe impl InstanceAllocator for OnDemandInstanceAllocator { )), dropped_data: RefCell::new(EntitySet::with_capacity(req.module.passive_data.len())), host_state, - #[cfg(all(feature = "uffd", target_os = "linux"))] - guard_page_faults: RefCell::new(Vec::new()), vmctx: VMContext {}, }; let layout = instance.alloc_layout(); diff --git a/crates/runtime/src/instance/allocator/pooling.rs b/crates/runtime/src/instance/allocator/pooling.rs index 277bea2b707c..6b63d6de526f 100644 --- a/crates/runtime/src/instance/allocator/pooling.rs +++ b/crates/runtime/src/instance/allocator/pooling.rs @@ -367,8 +367,6 @@ impl InstancePool { dropped_elements: RefCell::new(EntitySet::new()), dropped_data: RefCell::new(EntitySet::new()), host_state: Box::new(()), - #[cfg(all(feature = "uffd", target_os = "linux"))] - guard_page_faults: RefCell::new(Vec::new()), vmctx: VMContext {}, }, ); @@ -431,9 +429,15 @@ impl InstancePool { let memory = mem::take(memory); debug_assert!(memory.is_static()); + // Reset any faulted guard pages as the physical memory may be reused for another instance in the future + #[cfg(all(feature = "uffd", target_os = "linux"))] + memory + .reset_guard_pages() + .expect("failed to reset guard pages"); + let size = (memory.size() * WASM_PAGE_SIZE) as usize; drop(memory); - decommit_memory_pages(base, size).unwrap(); + decommit_memory_pages(base, size).expect("failed to decommit linear memory pages"); } instance.memories.clear(); @@ -450,7 +454,7 @@ impl InstancePool { ); drop(table); - decommit_table_pages(base, size).unwrap(); + decommit_table_pages(base, size).expect("failed to decommit table pages"); } instance.tables.clear(); @@ -469,12 +473,6 @@ impl InstancePool { ) -> Result<(), InstantiationError> { let module = instance.module.as_ref(); - // Reset all guard pages before reusing the instance - #[cfg(all(feature = "uffd", target_os = "linux"))] - instance - .reset_guard_pages() - .map_err(|e| InstantiationError::Resource(e.to_string()))?; - debug_assert!(instance.memories.is_empty()); for plan in diff --git a/crates/runtime/src/instance/allocator/pooling/uffd.rs b/crates/runtime/src/instance/allocator/pooling/uffd.rs index 93ffaf1504ef..ebe5effbc348 100644 --- a/crates/runtime/src/instance/allocator/pooling/uffd.rs +++ b/crates/runtime/src/instance/allocator/pooling/uffd.rs @@ -313,8 +313,12 @@ unsafe fn handle_page_fault( None => { log::trace!("out of bounds memory access at {:p}", addr); - // Record the guard page fault with the instance so it can be reset later. - instance.record_guard_page_fault(page_addr, len, reset_guard_page); + // Record the guard page fault so the page protection level can be reset later + instance.memories[memory_index].record_guard_page_fault( + page_addr, + len, + reset_guard_page, + ); wake_guard_page_access(&uffd, page_addr, len)?; } } diff --git a/crates/runtime/src/memory.rs b/crates/runtime/src/memory.rs index 945cb5796731..024d90124604 100644 --- a/crates/runtime/src/memory.rs +++ b/crates/runtime/src/memory.rs @@ -180,6 +180,10 @@ enum MemoryStorage { size: Cell, maximum: u32, make_accessible: fn(*mut u8, usize) -> Result<()>, + /// Stores the pages in the linear memory that have faulted as guard pages when using the `uffd` feature. + /// These pages need their protection level reset before the memory can grow. + #[cfg(all(feature = "uffd", target_os = "linux"))] + guard_page_faults: RefCell Result<()>)>>, }, Dynamic(Box), } @@ -209,6 +213,8 @@ impl Memory { size: Cell::new(plan.memory.minimum), maximum: min(plan.memory.maximum.unwrap_or(maximum), maximum), make_accessible, + #[cfg(all(feature = "uffd", target_os = "linux"))] + guard_page_faults: RefCell::new(Vec::new()), })) } @@ -242,6 +248,10 @@ impl Memory { make_accessible, .. } => { + // Reset any faulted guard pages before growing the memory. + #[cfg(all(feature = "uffd", target_os = "linux"))] + self.reset_guard_pages().ok()?; + let old_size = size.get(); if delta == 0 { return Some(old_size); @@ -276,6 +286,56 @@ impl Memory { MemoryStorage::Dynamic(mem) => mem.vmmemory(), } } + + /// Records a faulted guard page in a static memory. + /// + /// This is used to track faulted guard pages that need to be reset for the uffd feature. + /// + /// This function will panic if called on a dynamic memory. + #[cfg(all(feature = "uffd", target_os = "linux"))] + pub(crate) fn record_guard_page_fault( + &self, + page_addr: *mut u8, + size: usize, + reset: fn(*mut u8, usize) -> Result<()>, + ) { + match &self.0 { + MemoryStorage::Static { + guard_page_faults, .. + } => { + guard_page_faults + .borrow_mut() + .push((page_addr, size, reset)); + } + MemoryStorage::Dynamic(_) => { + unreachable!("dynamic memories should not have guard page faults") + } + } + } + + /// Resets the previously faulted guard pages of a static memory. + /// + /// This is used to reset the protection of any guard pages that were previously faulted. + /// + /// This function will panic if called on a dynamic memory. + #[cfg(all(feature = "uffd", target_os = "linux"))] + pub(crate) fn reset_guard_pages(&self) -> Result<()> { + match &self.0 { + MemoryStorage::Static { + guard_page_faults, .. + } => { + let mut faults = guard_page_faults.borrow_mut(); + for (addr, len, reset) in faults.drain(..) { + reset(addr, len)?; + } + } + MemoryStorage::Dynamic(_) => { + unreachable!("dynamic memories should not have guard page faults") + } + } + + Ok(()) + } } // The default memory representation is an empty memory that cannot grow. @@ -290,6 +350,8 @@ impl Default for Memory { size: Cell::new(0), maximum: 0, make_accessible, + #[cfg(all(feature = "uffd", target_os = "linux"))] + guard_page_faults: RefCell::new(Vec::new()), }) } } From 623290d42e4c688d02549671ec39c6c1dff6dce1 Mon Sep 17 00:00:00 2001 From: Peter Huene Date: Mon, 8 Mar 2021 09:30:13 -0800 Subject: [PATCH 33/33] Use `anyhow::Error` in instantiation errors. This commit updates the error enums used in instantiation errors to encapsulate an `anyhow::Error` rather than a string. --- crates/jit/src/instantiate.rs | 8 ++++---- crates/jit/src/trampoline.rs | 4 +++- crates/runtime/src/instance/allocator.rs | 10 ++++------ crates/runtime/src/instance/allocator/pooling.rs | 6 +++--- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/crates/jit/src/instantiate.rs b/crates/jit/src/instantiate.rs index 250b83728190..df6a17fa24b4 100644 --- a/crates/jit/src/instantiate.rs +++ b/crates/jit/src/instantiate.rs @@ -133,9 +133,9 @@ impl CompilationArtifacts { } let obj = obj.write().map_err(|_| { - SetupError::Instantiate(InstantiationError::Resource( - "failed to create image memory".to_string(), - )) + SetupError::Instantiate(InstantiationError::Resource(anyhow::anyhow!( + "failed to create image memory" + ))) })?; Ok(CompilationArtifacts { @@ -236,7 +236,7 @@ impl CompiledModule { &artifacts.unwind_info, ) .map_err(|message| { - SetupError::Instantiate(InstantiationError::Resource(format!( + SetupError::Instantiate(InstantiationError::Resource(anyhow::anyhow!( "failed to build code memory for functions: {}", message ))) diff --git a/crates/jit/src/trampoline.rs b/crates/jit/src/trampoline.rs index aebc4f1bc862..bb470dbcb3fc 100644 --- a/crates/jit/src/trampoline.rs +++ b/crates/jit/src/trampoline.rs @@ -38,7 +38,9 @@ pub fn make_trampoline( assert!(compiled_function.relocations.is_empty()); let ptr = code_memory .allocate_for_function(&compiled_function) - .map_err(|message| SetupError::Instantiate(InstantiationError::Resource(message)))? + .map_err(|message| { + SetupError::Instantiate(InstantiationError::Resource(anyhow::anyhow!(message))) + })? .as_ptr(); Ok(unsafe { std::mem::transmute::<*const VMFunctionBody, VMTrampoline>(ptr) }) } diff --git a/crates/runtime/src/instance/allocator.rs b/crates/runtime/src/instance/allocator.rs index e855f4154317..c56499994e3c 100644 --- a/crates/runtime/src/instance/allocator.rs +++ b/crates/runtime/src/instance/allocator.rs @@ -71,7 +71,7 @@ pub struct LinkError(pub String); pub enum InstantiationError { /// Insufficient resources available for execution. #[error("Insufficient resources: {0}")] - Resource(String), + Resource(anyhow::Error), /// A wasm link error occured. #[error("Failed to link module")] @@ -91,7 +91,7 @@ pub enum InstantiationError { pub enum FiberStackError { /// Insufficient resources available for the request. #[error("Insufficient resources: {0}")] - Resource(String), + Resource(anyhow::Error), /// An error for when the allocator doesn't support custom fiber stacks. #[error("Custom fiber stacks are not supported by the allocator")] NotSupported, @@ -569,10 +569,8 @@ impl OnDemandInstanceAllocator { let mut memories: PrimaryMap = PrimaryMap::with_capacity(module.memory_plans.len() - num_imports); for plan in &module.memory_plans.values().as_slice()[num_imports..] { - memories.push( - Memory::new_dynamic(plan, creator) - .map_err(|e| InstantiationError::Resource(e.to_string()))?, - ); + memories + .push(Memory::new_dynamic(plan, creator).map_err(InstantiationError::Resource)?); } Ok(memories) } diff --git a/crates/runtime/src/instance/allocator/pooling.rs b/crates/runtime/src/instance/allocator/pooling.rs index 6b63d6de526f..2aa8c4b7ff16 100644 --- a/crates/runtime/src/instance/allocator/pooling.rs +++ b/crates/runtime/src/instance/allocator/pooling.rs @@ -485,7 +485,7 @@ impl InstancePool { max_pages, commit_memory_pages, ) - .map_err(|e| InstantiationError::Resource(e.to_string()))?, + .map_err(InstantiationError::Resource)?, ); } @@ -509,7 +509,7 @@ impl InstancePool { let base = tables.next().unwrap(); commit_table_pages(base, max_elements as usize * mem::size_of::<*mut u8>()) - .map_err(|e| InstantiationError::Resource(e.to_string()))?; + .map_err(InstantiationError::Resource)?; instance .tables @@ -785,7 +785,7 @@ impl StackPool { .add((index * self.stack_size) + self.page_size); commit_stack_pages(bottom_of_stack, size_without_guard) - .map_err(|e| FiberStackError::Resource(e.to_string()))?; + .map_err(FiberStackError::Resource)?; // The top of the stack should be returned Ok(bottom_of_stack.add(size_without_guard))