diff --git a/build.rs b/build.rs index 3cbcac678e1d..3b395c3edbd2 100644 --- a/build.rs +++ b/build.rs @@ -211,7 +211,8 @@ fn ignore(testsuite: &str, testname: &str, strategy: &str) -> bool { ("simd", "simd_const") => return true, ("reference_types", "table_copy_on_imported_tables") - | ("reference_types", "externref_id_function") => { + | ("reference_types", "externref_id_function") + | ("reference_types", "table_size") => { // TODO(#1886): Ignore if this isn't x64, because Cranelift only // supports reference types on x64. return env::var("CARGO_CFG_TARGET_ARCH").unwrap() != "x86_64"; diff --git a/crates/environ/src/func_environ.rs b/crates/environ/src/func_environ.rs index 0f9b8f755d0e..384ffe63a544 100644 --- a/crates/environ/src/func_environ.rs +++ b/crates/environ/src/func_environ.rs @@ -7,11 +7,11 @@ use cranelift_codegen::ir::condcodes::*; use cranelift_codegen::ir::immediates::{Offset32, Uimm64}; use cranelift_codegen::ir::types::*; use cranelift_codegen::ir::{AbiParam, ArgumentPurpose, Function, InstBuilder, Signature}; -use cranelift_codegen::isa::TargetFrontendConfig; +use cranelift_codegen::isa::{self, TargetFrontendConfig}; use cranelift_entity::EntityRef; use cranelift_wasm::{ - self, FuncIndex, GlobalIndex, GlobalVariable, MemoryIndex, SignatureIndex, TableIndex, - TargetEnvironment, WasmError, WasmResult, + self, FuncIndex, GlobalIndex, GlobalVariable, MemoryIndex, SignatureIndex, TableElementType, + TableIndex, TargetEnvironment, WasmError, WasmResult, }; #[cfg(feature = "lightbeam")] use cranelift_wasm::{DefinedFuncIndex, DefinedGlobalIndex, DefinedMemoryIndex, DefinedTableIndex}; @@ -26,65 +26,146 @@ pub fn get_func_name(func_index: FuncIndex) -> ir::ExternalName { #[derive(Copy, Clone, Debug)] pub struct BuiltinFunctionIndex(u32); -impl BuiltinFunctionIndex { - /// Returns an index for wasm's `memory.grow` builtin function. - pub const fn get_memory32_grow_index() -> Self { - Self(0) +macro_rules! declare_builtin_functions { + ( + $( + $( #[$attr:meta] )* + $name:ident( $( $param:ident ),* ) -> ( $( $result:ident ),* ); + )* + ) => { + /// A struct with an `Option` member for every builtin + /// function, to de-duplicate constructing/getting its signature. + struct BuiltinFunctionSignatures { + pointer_type: ir::Type, + reference_type: ir::Type, + call_conv: isa::CallConv, + $( + $name: Option, + )* + } + + impl BuiltinFunctionSignatures { + fn new( + pointer_type: ir::Type, + reference_type: ir::Type, + call_conv: isa::CallConv, + ) -> Self { + Self { + pointer_type, + reference_type, + call_conv, + $( + $name: None, + )* + } + } + + fn vmctx(&self) -> AbiParam { + AbiParam::special(self.pointer_type, ArgumentPurpose::VMContext) + } + + fn reference(&self) -> AbiParam { + AbiParam::new(self.reference_type) + } + + fn i32(&self) -> AbiParam { + AbiParam::new(I32) + } + + $( + fn $name(&mut self, func: &mut Function) -> ir::SigRef { + let sig = self.$name.unwrap_or_else(|| { + func.import_signature(Signature { + params: vec![ $( self.$param() ),* ], + returns: vec![ $( self.$result() ),* ], + call_conv: self.call_conv, + }) + }); + self.$name = Some(sig); + sig + } + )* + } + + impl BuiltinFunctionIndex { + declare_builtin_functions!( + @indices; + 0; + $( $( #[$attr] )* $name; )* + ); + } + }; + + // Base case: no more indices to declare, so define the total number of + // function indices. + ( + @indices; + $len:expr; + ) => { + /// Returns the total number of builtin functions. + pub const fn builtin_functions_total_number() -> u32 { + $len + } + }; + + // Recursive case: declare the next index, and then keep declaring the rest of + // the indices. + ( + @indices; + $index:expr; + $( #[$this_attr:meta] )* + $this_name:ident; + $( + $( #[$rest_attr:meta] )* + $rest_name:ident; + )* + ) => { + $( #[$this_attr] )* + pub const fn $this_name() -> Self { + Self($index) + } + + declare_builtin_functions!( + @indices; + ($index + 1); + $( $( #[$rest_attr] )* $rest_name; )* + ); } +} + +declare_builtin_functions! { + /// Returns an index for wasm's `memory.grow` builtin function. + memory32_grow(vmctx, i32, i32) -> (i32); /// Returns an index for wasm's imported `memory.grow` builtin function. - pub const fn get_imported_memory32_grow_index() -> Self { - Self(1) - } + imported_memory32_grow(vmctx, i32, i32) -> (i32); /// Returns an index for wasm's `memory.size` builtin function. - pub const fn get_memory32_size_index() -> Self { - Self(2) - } + memory32_size(vmctx, i32) -> (i32); /// Returns an index for wasm's imported `memory.size` builtin function. - pub const fn get_imported_memory32_size_index() -> Self { - Self(3) - } + imported_memory32_size(vmctx, i32) -> (i32); /// Returns an index for wasm's `table.copy` when both tables are locally /// defined. - pub const fn get_table_copy_index() -> Self { - Self(4) - } + table_copy(vmctx, i32, i32, i32, i32, i32) -> (); /// Returns an index for wasm's `table.init`. - pub const fn get_table_init_index() -> Self { - Self(5) - } + table_init(vmctx, i32, i32, i32, i32, i32) -> (); /// Returns an index for wasm's `elem.drop`. - pub const fn get_elem_drop_index() -> Self { - Self(6) - } + elem_drop(vmctx, i32) -> (); /// Returns an index for wasm's `memory.copy` for locally defined memories. - pub const fn get_defined_memory_copy_index() -> Self { - Self(7) - } + defined_memory_copy(vmctx, i32, i32, i32, i32) -> (); /// Returns an index for wasm's `memory.copy` for imported memories. - pub const fn get_imported_memory_copy_index() -> Self { - Self(8) - } + imported_memory_copy(vmctx, i32, i32, i32, i32) -> (); /// Returns an index for wasm's `memory.fill` for locally defined memories. - pub const fn get_memory_fill_index() -> Self { - Self(9) - } + memory_fill(vmctx, i32, i32, i32, i32) -> (); /// Returns an index for wasm's `memory.fill` for imported memories. - pub const fn get_imported_memory_fill_index() -> Self { - Self(10) - } + imported_memory_fill(vmctx, i32, i32, i32, i32) -> (); /// Returns an index for wasm's `memory.init` instruction. - pub const fn get_memory_init_index() -> Self { - Self(11) - } + memory_init(vmctx, i32, i32, i32, i32, i32) -> (); /// Returns an index for wasm's `data.drop` instruction. - pub const fn get_data_drop_index() -> Self { - Self(12) - } - /// Returns the total number of builtin functions. - pub const fn builtin_functions_total_number() -> u32 { - 13 - } + data_drop(vmctx, i32) -> (); + /// Returns an index for Wasm's `table.grow` instruction for `externref`s. + table_grow_extern_ref(vmctx, i32, i32, reference) -> (i32); +} +impl BuiltinFunctionIndex { /// Create a new `BuiltinFunctionIndex` from its index pub const fn from_u32(i: u32) -> Self { Self(i) @@ -107,37 +188,8 @@ pub struct FuncEnvironment<'module_environment> { /// The Cranelift global holding the vmctx address. vmctx: Option, - /// The external function signature for implementing wasm's `memory.size` - /// for locally-defined 32-bit memories. - memory32_size_sig: Option, - - /// The external function signature for implementing wasm's `memory.grow` - /// for locally-defined memories. - memory_grow_sig: Option, - - /// The external function signature for implementing wasm's `table.copy` - /// (it's the same for both local and imported tables). - table_copy_sig: Option, - - /// The external function signature for implementing wasm's `table.init`. - table_init_sig: Option, - - /// The external function signature for implementing wasm's `elem.drop`. - elem_drop_sig: Option, - - /// The external function signature for implementing wasm's `memory.copy` - /// (it's the same for both local and imported memories). - memory_copy_sig: Option, - - /// The external function signature for implementing wasm's `memory.fill` - /// (it's the same for both local and imported memories). - memory_fill_sig: Option, - - /// The external function signature for implementing wasm's `memory.init`. - memory_init_sig: Option, - - /// The external function signature for implementing wasm's `data.drop`. - data_drop_sig: Option, + /// Caches of signatures for builtin functions. + builtin_function_signatures: BuiltinFunctionSignatures, /// Offsets to struct fields accessed by JIT code. pub(crate) offsets: VMOffsets, @@ -151,19 +203,20 @@ impl<'module_environment> FuncEnvironment<'module_environment> { module: &'module_environment ModuleLocal, tunables: &'module_environment Tunables, ) -> Self { + let builtin_function_signatures = BuiltinFunctionSignatures::new( + target_config.pointer_type(), + match target_config.pointer_type() { + ir::types::I32 => ir::types::R32, + ir::types::I64 => ir::types::R64, + _ => panic!(), + }, + target_config.default_call_conv, + ); Self { target_config, module, vmctx: None, - memory32_size_sig: None, - memory_grow_sig: None, - table_copy_sig: None, - table_init_sig: None, - elem_drop_sig: None, - memory_copy_sig: None, - memory_fill_sig: None, - memory_init_sig: None, - data_drop_sig: None, + builtin_function_signatures, offsets: VMOffsets::new(target_config.pointer_bytes(), module), tunables, } @@ -181,22 +234,6 @@ impl<'module_environment> FuncEnvironment<'module_environment> { }) } - fn get_memory_grow_sig(&mut self, func: &mut Function) -> ir::SigRef { - let sig = self.memory_grow_sig.unwrap_or_else(|| { - func.import_signature(Signature { - params: vec![ - AbiParam::special(self.pointer_type(), ArgumentPurpose::VMContext), - AbiParam::new(I32), - AbiParam::new(I32), - ], - returns: vec![AbiParam::new(I32)], - call_conv: self.target_config.default_call_conv, - }) - }); - self.memory_grow_sig = Some(sig); - sig - } - /// Return the memory.grow function signature to call for the given index, along with the /// translated index value to pass to it and its index in `VMBuiltinFunctionsArray`. fn get_memory_grow_func( @@ -206,34 +243,20 @@ impl<'module_environment> FuncEnvironment<'module_environment> { ) -> (ir::SigRef, usize, BuiltinFunctionIndex) { if self.module.is_imported_memory(index) { ( - self.get_memory_grow_sig(func), + self.builtin_function_signatures + .imported_memory32_grow(func), index.index(), - BuiltinFunctionIndex::get_imported_memory32_grow_index(), + BuiltinFunctionIndex::imported_memory32_grow(), ) } else { ( - self.get_memory_grow_sig(func), + self.builtin_function_signatures.memory32_grow(func), self.module.defined_memory_index(index).unwrap().index(), - BuiltinFunctionIndex::get_memory32_grow_index(), + BuiltinFunctionIndex::memory32_grow(), ) } } - fn get_memory32_size_sig(&mut self, func: &mut Function) -> ir::SigRef { - let sig = self.memory32_size_sig.unwrap_or_else(|| { - func.import_signature(Signature { - params: vec![ - AbiParam::special(self.pointer_type(), ArgumentPurpose::VMContext), - AbiParam::new(I32), - ], - returns: vec![AbiParam::new(I32)], - call_conv: self.target_config.default_call_conv, - }) - }); - self.memory32_size_sig = Some(sig); - sig - } - /// Return the memory.size function signature to call for the given index, along with the /// translated index value to pass to it and its index in `VMBuiltinFunctionsArray`. fn get_memory_size_func( @@ -243,41 +266,38 @@ impl<'module_environment> FuncEnvironment<'module_environment> { ) -> (ir::SigRef, usize, BuiltinFunctionIndex) { if self.module.is_imported_memory(index) { ( - self.get_memory32_size_sig(func), + self.builtin_function_signatures + .imported_memory32_size(func), index.index(), - BuiltinFunctionIndex::get_imported_memory32_size_index(), + BuiltinFunctionIndex::imported_memory32_size(), ) } else { ( - self.get_memory32_size_sig(func), + self.builtin_function_signatures.memory32_size(func), self.module.defined_memory_index(index).unwrap().index(), - BuiltinFunctionIndex::get_memory32_size_index(), + BuiltinFunctionIndex::memory32_size(), ) } } - fn get_table_copy_sig(&mut self, func: &mut Function) -> ir::SigRef { - let sig = self.table_copy_sig.unwrap_or_else(|| { - func.import_signature(Signature { - params: vec![ - AbiParam::special(self.pointer_type(), ArgumentPurpose::VMContext), - // Destination table index. - AbiParam::new(I32), - // Source table index. - AbiParam::new(I32), - // Index within destination table. - AbiParam::new(I32), - // Index within source table. - AbiParam::new(I32), - // Number of elements to copy. - AbiParam::new(I32), - ], - returns: vec![], - call_conv: self.target_config.default_call_conv, - }) - }); - self.table_copy_sig = Some(sig); - sig + fn get_table_grow_func( + &mut self, + func: &mut Function, + table_index: TableIndex, + ) -> WasmResult<(ir::SigRef, BuiltinFunctionIndex, usize)> { + match self.module.table_plans[table_index].table.ty { + TableElementType::Func => Err(WasmError::Unsupported( + "the `table.grow` instruction is not supported with `funcref` yet".into(), + )), + TableElementType::Val(ty) => { + assert_eq!(ty, self.reference_type()); + Ok(( + self.builtin_function_signatures.table_grow_extern_ref(func), + BuiltinFunctionIndex::table_grow_extern_ref(), + table_index.as_u32() as usize, + )) + } + } } fn get_table_copy_func( @@ -286,94 +306,28 @@ impl<'module_environment> FuncEnvironment<'module_environment> { dst_table_index: TableIndex, src_table_index: TableIndex, ) -> (ir::SigRef, usize, usize, BuiltinFunctionIndex) { - let sig = self.get_table_copy_sig(func); + let sig = self.builtin_function_signatures.table_copy(func); ( sig, dst_table_index.as_u32() as usize, src_table_index.as_u32() as usize, - BuiltinFunctionIndex::get_table_copy_index(), + BuiltinFunctionIndex::table_copy(), ) } - fn get_table_init_sig(&mut self, func: &mut Function) -> ir::SigRef { - let sig = self.table_init_sig.unwrap_or_else(|| { - func.import_signature(Signature { - params: vec![ - AbiParam::special(self.pointer_type(), ArgumentPurpose::VMContext), - // Table index. - AbiParam::new(I32), - // Segment index. - AbiParam::new(I32), - // Destination index within table. - AbiParam::new(I32), - // Source index within segment. - AbiParam::new(I32), - // Number of elements to initialize. - AbiParam::new(I32), - ], - returns: vec![], - call_conv: self.target_config.default_call_conv, - }) - }); - self.table_init_sig = Some(sig); - sig - } - fn get_table_init_func( &mut self, func: &mut Function, table_index: TableIndex, ) -> (ir::SigRef, usize, BuiltinFunctionIndex) { - let sig = self.get_table_init_sig(func); + let sig = self.builtin_function_signatures.table_init(func); let table_index = table_index.as_u32() as usize; - ( - sig, - table_index, - BuiltinFunctionIndex::get_table_init_index(), - ) - } - - fn get_elem_drop_sig(&mut self, func: &mut Function) -> ir::SigRef { - let sig = self.elem_drop_sig.unwrap_or_else(|| { - func.import_signature(Signature { - params: vec![ - AbiParam::special(self.pointer_type(), ArgumentPurpose::VMContext), - // Element index. - AbiParam::new(I32), - ], - returns: vec![], - call_conv: self.target_config.default_call_conv, - }) - }); - self.elem_drop_sig = Some(sig); - sig + (sig, table_index, BuiltinFunctionIndex::table_init()) } fn get_elem_drop_func(&mut self, func: &mut Function) -> (ir::SigRef, BuiltinFunctionIndex) { - let sig = self.get_elem_drop_sig(func); - (sig, BuiltinFunctionIndex::get_elem_drop_index()) - } - - fn get_memory_copy_sig(&mut self, func: &mut Function) -> ir::SigRef { - let sig = self.memory_copy_sig.unwrap_or_else(|| { - func.import_signature(Signature { - params: vec![ - AbiParam::special(self.pointer_type(), ArgumentPurpose::VMContext), - // Memory index. - AbiParam::new(I32), - // Destination address. - AbiParam::new(I32), - // Source address. - AbiParam::new(I32), - // Length. - AbiParam::new(I32), - ], - returns: vec![], - call_conv: self.target_config.default_call_conv, - }) - }); - self.memory_copy_sig = Some(sig); - sig + let sig = self.builtin_function_signatures.elem_drop(func); + (sig, BuiltinFunctionIndex::elem_drop()) } fn get_memory_copy_func( @@ -381,113 +335,53 @@ impl<'module_environment> FuncEnvironment<'module_environment> { func: &mut Function, memory_index: MemoryIndex, ) -> (ir::SigRef, usize, BuiltinFunctionIndex) { - let sig = self.get_memory_copy_sig(func); if let Some(defined_memory_index) = self.module.defined_memory_index(memory_index) { ( - sig, + self.builtin_function_signatures.defined_memory_copy(func), defined_memory_index.index(), - BuiltinFunctionIndex::get_defined_memory_copy_index(), + BuiltinFunctionIndex::defined_memory_copy(), ) } else { ( - sig, + self.builtin_function_signatures.imported_memory_copy(func), memory_index.index(), - BuiltinFunctionIndex::get_imported_memory_copy_index(), + BuiltinFunctionIndex::imported_memory_copy(), ) } } - fn get_memory_fill_sig(&mut self, func: &mut Function) -> ir::SigRef { - let sig = self.memory_fill_sig.unwrap_or_else(|| { - func.import_signature(Signature { - params: vec![ - AbiParam::special(self.pointer_type(), ArgumentPurpose::VMContext), - // Memory index. - AbiParam::new(I32), - // Destination address. - AbiParam::new(I32), - // Value. - AbiParam::new(I32), - // Length. - AbiParam::new(I32), - ], - returns: vec![], - call_conv: self.target_config.default_call_conv, - }) - }); - self.memory_fill_sig = Some(sig); - sig - } - fn get_memory_fill_func( &mut self, func: &mut Function, memory_index: MemoryIndex, ) -> (ir::SigRef, usize, BuiltinFunctionIndex) { - let sig = self.get_memory_fill_sig(func); if let Some(defined_memory_index) = self.module.defined_memory_index(memory_index) { ( - sig, + self.builtin_function_signatures.memory_fill(func), defined_memory_index.index(), - BuiltinFunctionIndex::get_memory_fill_index(), + BuiltinFunctionIndex::memory_fill(), ) } else { ( - sig, + self.builtin_function_signatures.imported_memory_fill(func), memory_index.index(), - BuiltinFunctionIndex::get_imported_memory_fill_index(), + BuiltinFunctionIndex::imported_memory_fill(), ) } } - fn get_memory_init_sig(&mut self, func: &mut Function) -> ir::SigRef { - let sig = self.memory_init_sig.unwrap_or_else(|| { - func.import_signature(Signature { - params: vec![ - AbiParam::special(self.pointer_type(), ArgumentPurpose::VMContext), - // Memory index. - AbiParam::new(I32), - // Data index. - AbiParam::new(I32), - // Destination address. - AbiParam::new(I32), - // Source index within the data segment. - AbiParam::new(I32), - // Length. - AbiParam::new(I32), - ], - returns: vec![], - call_conv: self.target_config.default_call_conv, - }) - }); - self.memory_init_sig = Some(sig); - sig - } - fn get_memory_init_func(&mut self, func: &mut Function) -> (ir::SigRef, BuiltinFunctionIndex) { - let sig = self.get_memory_init_sig(func); - (sig, BuiltinFunctionIndex::get_memory_init_index()) - } - - fn get_data_drop_sig(&mut self, func: &mut Function) -> ir::SigRef { - let sig = self.data_drop_sig.unwrap_or_else(|| { - func.import_signature(Signature { - params: vec![ - AbiParam::special(self.pointer_type(), ArgumentPurpose::VMContext), - // Data index. - AbiParam::new(I32), - ], - returns: vec![], - call_conv: self.target_config.default_call_conv, - }) - }); - self.data_drop_sig = Some(sig); - sig + ( + self.builtin_function_signatures.memory_init(func), + BuiltinFunctionIndex::memory_init(), + ) } fn get_data_drop_func(&mut self, func: &mut Function) -> (ir::SigRef, BuiltinFunctionIndex) { - let sig = self.get_data_drop_sig(func); - (sig, BuiltinFunctionIndex::get_data_drop_index()) + ( + self.builtin_function_signatures.data_drop(func), + BuiltinFunctionIndex::data_drop(), + ) } /// Translates load of builtin function and returns a pair of values `vmctx` @@ -726,14 +620,29 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m fn translate_table_grow( &mut self, - _: cranelift_codegen::cursor::FuncCursor<'_>, - _: TableIndex, - _: ir::Value, - _: ir::Value, + mut pos: cranelift_codegen::cursor::FuncCursor<'_>, + table_index: TableIndex, + delta: ir::Value, + init_value: ir::Value, ) -> WasmResult { - Err(WasmError::Unsupported( - "the `table.grow` instruction is not supported yet".into(), - )) + let (func_sig, func_idx, table_index_arg) = + self.get_table_grow_func(&mut pos.func, table_index)?; + + let table_index_arg = pos.ins().iconst(I32, table_index_arg as i64); + + let (vmctx, func_addr) = self.translate_load_builtin_function_address(&mut pos, func_idx); + let call_inst = pos.ins().call_indirect( + func_sig, + func_addr, + &[vmctx, table_index_arg, delta, init_value], + ); + Ok(pos + .func + .dfg + .inst_results(call_inst) + .first() + .copied() + .unwrap()) } fn translate_table_get( @@ -1180,13 +1089,12 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m fn translate_table_size( &mut self, - _pos: FuncCursor, - _index: TableIndex, - _table: ir::Table, + mut pos: FuncCursor, + _table_index: TableIndex, + table: ir::Table, ) -> WasmResult { - Err(WasmError::Unsupported( - "reference types: `table.size`".to_string(), - )) + let size_gv = pos.func.tables[table].bound_gv; + Ok(pos.ins().global_value(ir::types::I32, size_gv)) } fn translate_table_copy( diff --git a/crates/runtime/src/instance.rs b/crates/runtime/src/instance.rs index b77bd8f02c40..aea72a0b289e 100644 --- a/crates/runtime/src/instance.rs +++ b/crates/runtime/src/instance.rs @@ -448,21 +448,41 @@ impl Instance { foreign_instance.memory_size(foreign_index) } - /// Grow table by the specified amount of elements. + /// Grow table by the specified amount of elements, filling them with + /// `init_value`. /// - /// Returns `None` if table can't be grown by the specified amount - /// of elements. - pub(crate) fn table_grow(&self, table_index: DefinedTableIndex, delta: u32) -> Option { - let result = self - .tables - .get(table_index) - .unwrap_or_else(|| panic!("no table for index {}", table_index.index())) - .grow(delta); + /// Returns `None` if table can't be grown by the specified amount of + /// elements, or if `init_value` is the wrong type of table element. + pub(crate) fn table_grow( + &self, + table_index: TableIndex, + delta: u32, + init_value: TableElement, + ) -> Option { + let (defined_table_index, instance) = + self.get_defined_table_index_and_instance(table_index); + instance.defined_table_grow(defined_table_index, delta, init_value) + } - // Keep current the VMContext pointers used by compiled wasm code. - self.set_table(table_index, self.tables[table_index].vmtable()); + fn defined_table_grow( + &self, + table_index: DefinedTableIndex, + delta: u32, + init_value: TableElement, + ) -> Option { + unsafe { + let orig_size = self + .tables + .get(table_index) + .unwrap_or_else(|| panic!("no table for index {}", table_index.index())) + .grow(delta, init_value)?; - result + // Keep the `VMContext` pointers used by compiled Wasm code up to + // date. + self.set_table(table_index, self.tables[table_index].vmtable()); + + Some(orig_size) + } } // Get table element by index. @@ -757,6 +777,21 @@ impl Instance { let foreign_index = foreign_instance.table_index(foreign_table); &foreign_instance.tables[foreign_index] } + + pub(crate) fn get_defined_table_index_and_instance( + &self, + index: TableIndex, + ) -> (DefinedTableIndex, &Instance) { + if let Some(defined_table_index) = self.module.local.defined_table_index(index) { + (defined_table_index, self) + } else { + let import = self.imported_table(index); + let foreign_instance = unsafe { (&mut *(import).vmctx).instance() }; + let foreign_table_def = unsafe { &mut *(import).from }; + let foreign_table_index = foreign_instance.table_index(foreign_table_def); + (foreign_table_index, foreign_instance) + } + } } /// A handle holding an `Instance` of a WebAssembly module. @@ -1000,12 +1035,37 @@ impl InstanceHandle { self.instance().table_index(table) } - /// Grow table in this instance by the specified amount of pages. + /// Grow table in this instance by the specified amount of elements. /// - /// Returns `None` if memory can't be grown by the specified amount - /// of pages. - pub fn table_grow(&self, table_index: DefinedTableIndex, delta: u32) -> Option { - self.instance().table_grow(table_index, delta) + /// When the table is successfully grown, returns the original size of the + /// table. + /// + /// Returns `None` if memory can't be grown by the specified amount of pages + /// or if the `init_value` is the incorrect table element type. + pub fn table_grow( + &self, + table_index: TableIndex, + delta: u32, + init_value: TableElement, + ) -> Option { + self.instance().table_grow(table_index, delta, init_value) + } + + /// Grow table in this instance by the specified amount of elements. + /// + /// When the table is successfully grown, returns the original size of the + /// table. + /// + /// Returns `None` if memory can't be grown by the specified amount of pages + /// or if the `init_value` is the incorrect table element type. + pub fn defined_table_grow( + &self, + table_index: DefinedTableIndex, + delta: u32, + init_value: TableElement, + ) -> Option { + self.instance() + .defined_table_grow(table_index, delta, init_value) } /// Get table element reference. diff --git a/crates/runtime/src/libcalls.rs b/crates/runtime/src/libcalls.rs index be5e5b9e16e7..2b730e84787c 100644 --- a/crates/runtime/src/libcalls.rs +++ b/crates/runtime/src/libcalls.rs @@ -31,8 +31,33 @@ //! } //! } //! ``` +//! +//! * When receiving a raw `*mut u8` that is actually a `VMExternRef` reference, +//! convert it into a proper `VMExternRef` with `VMExternRef::clone_from_raw` +//! as soon as apossible. Any GC before raw pointer is converted into a +//! reference can potentially collect the referenced object, which could lead +//! to use after free. Avoid this by eagerly converting into a proper +//! `VMExternRef`! +//! +//! ```ignore +//! pub unsafe extern "C" my_lib_takes_ref(raw_extern_ref: *mut u8) { +//! // Before `clone_from_raw`, `raw_extern_ref` is potentially unrooted, +//! // and doing GC here could lead to use after free! +//! +//! let my_extern_ref = if raw_extern_ref.is_null() { +//! None +//! } else { +//! Some(VMExternRef::clone_from_raw(raw_extern_ref)) +//! }; +//! +//! // Now that we did `clone_from_raw`, it is safe to do a GC (or do +//! // anything else that might transitively GC, like call back into +//! // Wasm!) +//! } +//! ``` -use crate::table::Table; +use crate::externref::VMExternRef; +use crate::table::{Table, TableElement}; use crate::traphandlers::raise_lib_trap; use crate::vmcontext::VMContext; use wasmtime_environ::wasm::{DataIndex, DefinedMemoryIndex, ElemIndex, MemoryIndex, TableIndex}; @@ -201,6 +226,26 @@ pub unsafe extern "C" fn wasmtime_imported_memory32_size( instance.imported_memory_size(memory_index) } +/// Implementation of `table.grow` for `externref`s. +pub unsafe extern "C" fn wasmtime_table_grow_extern_ref( + vmctx: *mut VMContext, + table_index: u32, + delta: u32, + init_value: *mut u8, +) -> u32 { + let init_value = if init_value.is_null() { + None + } else { + Some(VMExternRef::clone_from_raw(init_value)) + }; + + let instance = (&mut *vmctx).instance(); + let table_index = TableIndex::from_u32(table_index); + instance + .table_grow(table_index, delta, TableElement::ExternRef(init_value)) + .unwrap_or(-1_i32 as u32) +} + /// Implementation of `table.copy`. pub unsafe extern "C" fn wasmtime_table_copy( vmctx: *mut VMContext, diff --git a/crates/runtime/src/table.rs b/crates/runtime/src/table.rs index aa2f793f8d6f..4023e250738e 100644 --- a/crates/runtime/src/table.rs +++ b/crates/runtime/src/table.rs @@ -67,29 +67,42 @@ impl Table { /// Grow table by the specified amount of elements. /// - /// Returns `None` if table can't be grown by the specified amount - /// of elements. Returns the previous size of the table if growth is - /// successful. - pub fn grow(&self, delta: u32) -> Option { + /// Returns the previous size of the table if growth is successful. + /// + /// Returns `None` if table can't be grown by the specified amount of + /// elements, or if the `init_value` is the wrong kind of table element. + /// + /// # Unsafety + /// + /// Resizing the table can reallocate its internal elements buffer. This + /// table's instance's `VMContext` has raw pointers to the elements buffer + /// that are used by Wasm, and they need to be fixed up before we call into + /// Wasm again. Failure to do so will result in use-after-free inside Wasm. + /// + /// Generally, prefer using `InstanceHandle::table_grow`, which encapsulates + /// this unsafety. + pub unsafe fn grow(&self, delta: u32, init_value: TableElement) -> Option { let size = self.size(); - let new_len = match size.checked_add(delta) { - Some(len) => { - if let Some(max) = self.maximum { - if len > max { - return None; - } - } - len - } - None => { + + let new_len = size.checked_add(delta)?; + if let Some(max) = self.maximum { + if new_len > max { return None; } - }; + } let new_len = usize::try_from(new_len).unwrap(); + match &mut *self.elements.borrow_mut() { - TableElements::FuncRefs(x) => x.resize(new_len, VMCallerCheckedAnyfunc::default()), - TableElements::ExternRefs(x) => x.resize(new_len, None), + TableElements::FuncRefs(x) => { + let init_value = init_value.try_into().ok()?; + x.resize(new_len, init_value) + } + TableElements::ExternRefs(x) => { + let init_value = init_value.try_into().ok()?; + x.resize(new_len, init_value) + } } + Some(size) } @@ -207,3 +220,21 @@ impl TryFrom for Option { } } } + +impl From for TableElement { + fn from(f: VMCallerCheckedAnyfunc) -> TableElement { + TableElement::FuncRef(f) + } +} + +impl From> for TableElement { + fn from(x: Option) -> TableElement { + TableElement::ExternRef(x) + } +} + +impl From for TableElement { + fn from(x: VMExternRef) -> TableElement { + TableElement::ExternRef(Some(x)) + } +} diff --git a/crates/runtime/src/vmcontext.rs b/crates/runtime/src/vmcontext.rs index 94dc4080d995..e9f681926cb6 100644 --- a/crates/runtime/src/vmcontext.rs +++ b/crates/runtime/src/vmcontext.rs @@ -540,39 +540,34 @@ impl VMBuiltinFunctionsArray { let mut ptrs = [0; Self::len()]; - ptrs[BuiltinFunctionIndex::get_memory32_grow_index().index() as usize] = + ptrs[BuiltinFunctionIndex::memory32_grow().index() as usize] = wasmtime_memory32_grow as usize; - ptrs[BuiltinFunctionIndex::get_imported_memory32_grow_index().index() as usize] = + ptrs[BuiltinFunctionIndex::imported_memory32_grow().index() as usize] = wasmtime_imported_memory32_grow as usize; - - ptrs[BuiltinFunctionIndex::get_memory32_size_index().index() as usize] = + ptrs[BuiltinFunctionIndex::memory32_size().index() as usize] = wasmtime_memory32_size as usize; - ptrs[BuiltinFunctionIndex::get_imported_memory32_size_index().index() as usize] = + ptrs[BuiltinFunctionIndex::imported_memory32_size().index() as usize] = wasmtime_imported_memory32_size as usize; - - ptrs[BuiltinFunctionIndex::get_table_copy_index().index() as usize] = - wasmtime_table_copy as usize; - - ptrs[BuiltinFunctionIndex::get_table_init_index().index() as usize] = - wasmtime_table_init as usize; - ptrs[BuiltinFunctionIndex::get_elem_drop_index().index() as usize] = - wasmtime_elem_drop as usize; - - ptrs[BuiltinFunctionIndex::get_defined_memory_copy_index().index() as usize] = + ptrs[BuiltinFunctionIndex::table_copy().index() as usize] = wasmtime_table_copy as usize; + ptrs[BuiltinFunctionIndex::table_grow_extern_ref().index() as usize] = + wasmtime_table_grow_extern_ref as usize; + ptrs[BuiltinFunctionIndex::table_init().index() as usize] = wasmtime_table_init as usize; + ptrs[BuiltinFunctionIndex::elem_drop().index() as usize] = wasmtime_elem_drop as usize; + ptrs[BuiltinFunctionIndex::defined_memory_copy().index() as usize] = wasmtime_defined_memory_copy as usize; - ptrs[BuiltinFunctionIndex::get_imported_memory_copy_index().index() as usize] = + ptrs[BuiltinFunctionIndex::imported_memory_copy().index() as usize] = wasmtime_imported_memory_copy as usize; - ptrs[BuiltinFunctionIndex::get_memory_fill_index().index() as usize] = - wasmtime_memory_fill as usize; - ptrs[BuiltinFunctionIndex::get_imported_memory_fill_index().index() as usize] = + ptrs[BuiltinFunctionIndex::memory_fill().index() as usize] = wasmtime_memory_fill as usize; + ptrs[BuiltinFunctionIndex::imported_memory_fill().index() as usize] = wasmtime_imported_memory_fill as usize; - ptrs[BuiltinFunctionIndex::get_memory_init_index().index() as usize] = - wasmtime_memory_init as usize; - ptrs[BuiltinFunctionIndex::get_data_drop_index().index() as usize] = - wasmtime_data_drop as usize; - - debug_assert!(ptrs.iter().cloned().all(|p| p != 0)); + ptrs[BuiltinFunctionIndex::memory_init().index() as usize] = wasmtime_memory_init as usize; + ptrs[BuiltinFunctionIndex::data_drop().index() as usize] = wasmtime_data_drop as usize; + if cfg!(debug_assertions) { + for i in 0..ptrs.len() { + debug_assert!(ptrs[i] != 0, "index {} is not initialized", i); + } + } Self { ptrs } } } diff --git a/crates/wasmtime/src/externals.rs b/crates/wasmtime/src/externals.rs index 77e75f64f465..29faa28dd2a2 100644 --- a/crates/wasmtime/src/externals.rs +++ b/crates/wasmtime/src/externals.rs @@ -395,12 +395,28 @@ impl Table { /// error if `init` is not of the right type. pub fn grow(&self, delta: u32, init: Val) -> Result { let index = self.wasmtime_table_index(); - let item = into_checked_anyfunc(init, &self.instance.store)?; - if let Some(len) = self.instance.table_grow(index, delta) { - for i in 0..delta { - set_table_item(&self.instance, index, len + i, item.clone())?; + let orig_size = match self.ty().element() { + ValType::FuncRef => { + let init = into_checked_anyfunc(init, &self.instance.store)?; + self.instance + .defined_table_grow(index, delta, runtime::TableElement::FuncRef(init)) } - Ok(len) + ValType::ExternRef => { + let init = match init { + Val::ExternRef(Some(x)) => Some(x.inner), + Val::ExternRef(None) => None, + _ => bail!("incorrect init value for growing table"), + }; + self.instance.defined_table_grow( + index, + delta, + runtime::TableElement::ExternRef(init), + ) + } + _ => unreachable!("only `funcref` and `externref` tables are supported"), + }; + if let Some(size) = orig_size { + Ok(size) } else { bail!("failed to grow table by `{}`", delta) } diff --git a/crates/wasmtime/src/types.rs b/crates/wasmtime/src/types.rs index 96f5ce9f3f39..ad1905963a6b 100644 --- a/crates/wasmtime/src/types.rs +++ b/crates/wasmtime/src/types.rs @@ -394,12 +394,14 @@ impl TableType { } pub(crate) fn from_wasmtime_table(table: &wasm::Table) -> TableType { - assert!(if let wasm::TableElementType::Func = table.ty { - true - } else { - false - }); - let ty = ValType::FuncRef; + let ty = match table.ty { + wasm::TableElementType::Func => ValType::FuncRef, + #[cfg(target_pointer_width = "64")] + wasm::TableElementType::Val(ir::types::R64) => ValType::ExternRef, + #[cfg(target_pointer_width = "32")] + wasm::TableElementType::Val(ir::types::R32) => ValType::ExternRef, + _ => panic!("only `funcref` and `externref` tables supported"), + }; let limits = Limits::new(table.minimum, table.maximum); TableType::new(ty, limits) }