make use of symbolic vtables in interpreter

compiler/rustc_const_eval/src/interpret/cast.rs

@@ -299,29 +299,35 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
             }
             (&ty::Dynamic(ref data_a, ..), &ty::Dynamic(ref data_b, ..)) => {
                 let val = self.read_immediate(src)?;
+                let (old_data, old_vptr) = val.to_scalar_pair()?;
+                let old_vptr = self.scalar_to_ptr(old_vptr)?;
                 if data_a.principal_def_id() == data_b.principal_def_id() {
                     return self.write_immediate(*val, dest);
                 }
                 // trait upcasting coercion
-                let vptr_entry_idx = self.tcx.vtable_trait_upcasting_coercion_new_vptr_slot((
+                let Some(vptr_entry_idx) = self.tcx.vtable_trait_upcasting_coercion_new_vptr_slot((
                     src_pointee_ty,
                     dest_pointee_ty,
-                ));
-
-                if let Some(entry_idx) = vptr_entry_idx {
-                    let entry_idx = u64::try_from(entry_idx).unwrap();
-                    let (old_data, old_vptr) = val.to_scalar_pair()?;
-                    let old_vptr = self.scalar_to_ptr(old_vptr)?;
-                    let new_vptr = self
-                        .read_new_vtable_after_trait_upcasting_from_vtable(old_vptr, entry_idx)?;
-                    self.write_immediate(Immediate::new_dyn_trait(old_data, new_vptr, self), dest)
-                } else {
-                    self.write_immediate(*val, dest)
-                }
+                )) else {
+                    return self.write_immediate(*val, dest);
+                };
+
+                let (ty, _) = self.get_ptr_vtable(old_vptr)?;
+                let Some(ty::VtblEntry::TraitVPtr(new_trait)) = self.get_vtable_entries(old_vptr)?.get(vptr_entry_idx) else {
+                    throw_ub_format!(
+                        "upcasting to index {vptr_entry_idx} of vtable {old_vptr} but \
+                        that vtable is too small or does not have an upcast-vtable at that index"
+                    )
+                };
+                let new_trait = new_trait.map_bound(|trait_ref| {
+                    ty::ExistentialTraitRef::erase_self_ty(*self.tcx, trait_ref)
+                });
+                let new_vptr = self.get_vtable_ptr(ty, Some(new_trait))?;
+                self.write_immediate(Immediate::new_dyn_trait(old_data, new_vptr, self), dest)
             }
             (_, &ty::Dynamic(ref data, _)) => {
                 // Initial cast from sized to dyn trait
-                let vtable = self.get_vtable(src_pointee_ty, data.principal())?;
+                let vtable = self.get_vtable_ptr(src_pointee_ty, data.principal())?;
                 let ptr = self.read_immediate(src)?.to_scalar()?;
                 let val = Immediate::new_dyn_trait(ptr, vtable, &*self.tcx);
                 self.write_immediate(val, dest)

compiler/rustc_const_eval/src/interpret/eval_context.rs

@@ -635,7 +635,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
             ty::Dynamic(..) => {
                 let vtable = self.scalar_to_ptr(metadata.unwrap_meta())?;
                 // Read size and align from vtable (already checks size).
-                Ok(Some(self.read_size_and_align_from_vtable(vtable)?))
+                Ok(Some(self.get_vtable_size_and_align(vtable)?))
             }
 
             ty::Slice(_) | ty::Str => {

compiler/rustc_const_eval/src/interpret/memory.rs

@@ -16,7 +16,7 @@ use std::ptr;
 use rustc_ast::Mutability;
 use rustc_data_structures::fx::{FxHashMap, FxHashSet};
 use rustc_middle::mir::display_allocation;
-use rustc_middle::ty::{Instance, ParamEnv, TyCtxt};
+use rustc_middle::ty::{self, Instance, ParamEnv, Ty, TyCtxt};
 use rustc_target::abi::{Align, HasDataLayout, Size};
 
 use super::{
@@ -496,6 +496,8 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
                 // contains a reference to memory that was created during its evaluation (i.e., not
                 // to another static), those inner references only exist in "resolved" form.
                 if self.tcx.is_foreign_item(def_id) {
+                    // This is unreachable in Miri, but can happen in CTFE where we actually *do* support
+                    // referencing arbitrary (declared) extern statics.
                     throw_unsup!(ReadExternStatic(def_id));
                 }
 
@@ -666,11 +668,13 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
         // Can't do this in the match argument, we may get cycle errors since the lock would
         // be held throughout the match.
         match self.tcx.try_get_global_alloc(id) {
-            Some(GlobalAlloc::Static(did)) => {
-                assert!(!self.tcx.is_thread_local_static(did));
+            Some(GlobalAlloc::Static(def_id)) => {
+                assert!(self.tcx.is_static(def_id));
+                assert!(!self.tcx.is_thread_local_static(def_id));
                 // Use size and align of the type.
-                let ty = self.tcx.type_of(did);
+                let ty = self.tcx.type_of(def_id);
                 let layout = self.tcx.layout_of(ParamEnv::empty().and(ty)).unwrap();
+                assert!(!layout.is_unsized());
                 (layout.size, layout.align.abi, AllocKind::LiveData)
             }
             Some(GlobalAlloc::Memory(alloc)) => {
@@ -681,8 +685,8 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
             }
             Some(GlobalAlloc::Function(_)) => bug!("We already checked function pointers above"),
             Some(GlobalAlloc::Vtable(..)) => {
-                // No data to be accessed here.
-                return (Size::ZERO, Align::ONE, AllocKind::Vtable);
+                // No data to be accessed here. But vtables are pointer-aligned.
+                return (Size::ZERO, self.tcx.data_layout.pointer_align.abi, AllocKind::Vtable);
             }
             // The rest must be dead.
             None => {
@@ -722,7 +726,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
         &self,
         ptr: Pointer<Option<M::PointerTag>>,
     ) -> InterpResult<'tcx, FnVal<'tcx, M::ExtraFnVal>> {
-        trace!("get_fn({:?})", ptr);
+        trace!("get_ptr_fn({:?})", ptr);
         let (alloc_id, offset, _tag) = self.ptr_get_alloc_id(ptr)?;
         if offset.bytes() != 0 {
             throw_ub!(InvalidFunctionPointer(Pointer::new(alloc_id, offset)))
@@ -731,6 +735,21 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
             .ok_or_else(|| err_ub!(InvalidFunctionPointer(Pointer::new(alloc_id, offset))).into())
     }
 
+    pub fn get_ptr_vtable(
+        &self,
+        ptr: Pointer<Option<M::PointerTag>>,
+    ) -> InterpResult<'tcx, (Ty<'tcx>, Option<ty::PolyExistentialTraitRef<'tcx>>)> {
+        trace!("get_ptr_vtable({:?})", ptr);
+        let (alloc_id, offset, _tag) = self.ptr_get_alloc_id(ptr)?;
+        if offset.bytes() != 0 {
+            throw_ub!(InvalidVtablePointer(Pointer::new(alloc_id, offset)))
+        }
+        match self.tcx.try_get_global_alloc(alloc_id) {
+            Some(GlobalAlloc::Vtable(ty, trait_ref)) => Ok((ty, trait_ref)),
+            _ => throw_ub!(InvalidVtablePointer(Pointer::new(alloc_id, offset))),
+        }
+    }
+
     pub fn alloc_mark_immutable(&mut self, id: AllocId) -> InterpResult<'tcx> {
         self.get_alloc_raw_mut(id)?.0.mutability = Mutability::Not;
         Ok(())

compiler/rustc_const_eval/src/interpret/place.rs

@@ -886,28 +886,19 @@ where
     }
 
     /// Turn a place with a `dyn Trait` type into a place with the actual dynamic type.
-    /// Also return some more information so drop doesn't have to run the same code twice.
     pub(super) fn unpack_dyn_trait(
         &self,
         mplace: &MPlaceTy<'tcx, M::PointerTag>,
-    ) -> InterpResult<'tcx, (ty::Instance<'tcx>, MPlaceTy<'tcx, M::PointerTag>)> {
+    ) -> InterpResult<'tcx, MPlaceTy<'tcx, M::PointerTag>> {
         let vtable = self.scalar_to_ptr(mplace.vtable())?; // also sanity checks the type
-        let (instance, ty) = self.read_drop_type_from_vtable(vtable)?;
+        let (ty, _) = self.get_ptr_vtable(vtable)?;
         let layout = self.layout_of(ty)?;
 
-        // More sanity checks
-        if cfg!(debug_assertions) {
-            let (size, align) = self.read_size_and_align_from_vtable(vtable)?;
-            assert_eq!(size, layout.size);
-            // only ABI alignment is preserved
-            assert_eq!(align, layout.align.abi);
-        }
-
         let mplace = MPlaceTy {
             mplace: MemPlace { meta: MemPlaceMeta::None, ..**mplace },
             layout,
             align: layout.align.abi,
         };
-        Ok((instance, mplace))
+        Ok(mplace)
     }
 }

compiler/rustc_const_eval/src/interpret/terminator.rs

@@ -1,5 +1,4 @@
 use std::borrow::Cow;
-use std::convert::TryFrom;
 
 use rustc_middle::ty::layout::{FnAbiOf, LayoutOf};
 use rustc_middle::ty::Instance;
@@ -563,7 +562,12 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
                     ty::Dynamic(..)
                 ));
                 let vtable = self.scalar_to_ptr(receiver_place.meta.unwrap_meta())?;
-                let fn_val = self.get_vtable_slot(vtable, u64::try_from(idx).unwrap())?;
+                let Some(ty::VtblEntry::Method(fn_inst)) = self.get_vtable_entries(vtable)?.get(idx).copied() else {
+                    throw_ub_format!(
+                        "calling index {idx} of vtable {vtable} but \
+                        that vtable is too small or does not have a method at that index"
+                    )
+                };
 
                 // `*mut receiver_place.layout.ty` is almost the layout that we
                 // want for args[0]: We have to project to field 0 because we want
@@ -579,7 +583,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
                 trace!("Patched receiver operand to {:#?}", args[0]);
                 // recurse with concrete function
                 self.eval_fn_call(
-                    fn_val,
+                    FnVal::Instance(fn_inst),
                     (caller_abi, caller_fn_abi),
                     &args,
                     with_caller_location,
@@ -606,8 +610,10 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
 
         let (instance, place) = match place.layout.ty.kind() {
             ty::Dynamic(..) => {
-                // Dropping a trait object.
-                self.unpack_dyn_trait(&place)?
+                // Dropping a trait object. Need to find actual drop fn.
+                let place = self.unpack_dyn_trait(&place)?;
+                let instance = ty::Instance::resolve_drop_in_place(*self.tcx, place.layout.ty);
+                (instance, place)
             }
             _ => (instance, place),
         };

compiler/rustc_const_eval/src/interpret/traits.rs

@@ -1,14 +1,10 @@
-use std::convert::TryFrom;
-
-use rustc_middle::mir::interpret::{alloc_range, InterpResult, Pointer, PointerArithmetic};
-use rustc_middle::ty::{
-    self, Ty, TyCtxt, COMMON_VTABLE_ENTRIES_ALIGN, COMMON_VTABLE_ENTRIES_DROPINPLACE,
-    COMMON_VTABLE_ENTRIES_SIZE,
-};
+use rustc_middle::mir::interpret::{InterpResult, Pointer};
+use rustc_middle::ty::layout::LayoutOf;
+use rustc_middle::ty::{self, Ty, TyCtxt};
 use rustc_target::abi::{Align, Size};
 
 use super::util::ensure_monomorphic_enough;
-use super::{FnVal, InterpCx, Machine};
+use super::{InterpCx, Machine};
 
 impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
     /// Creates a dynamic vtable for the given type and vtable origin. This is used only for
@@ -17,8 +13,8 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
     /// The `trait_ref` encodes the erased self type. Hence, if we are
     /// making an object `Foo<Trait>` from a value of type `Foo<T>`, then
     /// `trait_ref` would map `T: Trait`.
-    pub fn get_vtable(
-        &mut self,
+    pub fn get_vtable_ptr(
+        &self,
         ty: Ty<'tcx>,
         poly_trait_ref: Option<ty::PolyExistentialTraitRef<'tcx>>,
     ) -> InterpResult<'tcx, Pointer<Option<M::PointerTag>>> {
@@ -30,114 +26,33 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
         ensure_monomorphic_enough(*self.tcx, ty)?;
         ensure_monomorphic_enough(*self.tcx, poly_trait_ref)?;
 
-        let vtable_allocation = self.tcx.vtable_allocation((ty, poly_trait_ref));
-
-        let vtable_ptr = self.global_base_pointer(Pointer::from(vtable_allocation))?;
-
+        let vtable_symbolic_allocation = self.tcx.create_vtable_alloc(ty, poly_trait_ref);
+        let vtable_ptr = self.global_base_pointer(Pointer::from(vtable_symbolic_allocation))?;
         Ok(vtable_ptr.into())
     }
 
-    /// Resolves the function at the specified slot in the provided
-    /// vtable. Currently an index of '3' (`TyCtxt::COMMON_VTABLE_ENTRIES.len()`)
-    /// corresponds to the first method declared in the trait of the provided vtable.
-    pub fn get_vtable_slot(
+    /// Returns a high-level representation of the entires of the given vtable.
+    pub fn get_vtable_entries(
         &self,
         vtable: Pointer<Option<M::PointerTag>>,
-        idx: u64,
-    ) -> InterpResult<'tcx, FnVal<'tcx, M::ExtraFnVal>> {
-        let ptr_size = self.pointer_size();
-        let vtable_slot = vtable.offset(ptr_size * idx, self)?;
-        let vtable_slot = self
-            .get_ptr_alloc(vtable_slot, ptr_size, self.tcx.data_layout.pointer_align.abi)?
-            .expect("cannot be a ZST");
-        let fn_ptr = self.scalar_to_ptr(vtable_slot.read_pointer(Size::ZERO)?.check_init()?)?;
-        self.get_ptr_fn(fn_ptr)
+    ) -> InterpResult<'tcx, &'tcx [ty::VtblEntry<'tcx>]> {
+        let (ty, poly_trait_ref) = self.get_ptr_vtable(vtable)?;
+        Ok(if let Some(poly_trait_ref) = poly_trait_ref {
+            let trait_ref = poly_trait_ref.with_self_ty(*self.tcx, ty);
+            let trait_ref = self.tcx.erase_regions(trait_ref);
+            self.tcx.vtable_entries(trait_ref)
+        } else {
+            TyCtxt::COMMON_VTABLE_ENTRIES
+        })
     }
 
-    /// Returns the drop fn instance as well as the actual dynamic type.
-    pub fn read_drop_type_from_vtable(
-        &self,
-        vtable: Pointer<Option<M::PointerTag>>,
-    ) -> InterpResult<'tcx, (ty::Instance<'tcx>, Ty<'tcx>)> {
-        let pointer_size = self.pointer_size();
-        // We don't care about the pointee type; we just want a pointer.
-        let vtable = self
-            .get_ptr_alloc(
-                vtable,
-                pointer_size * u64::try_from(TyCtxt::COMMON_VTABLE_ENTRIES.len()).unwrap(),
-                self.tcx.data_layout.pointer_align.abi,
-            )?
-            .expect("cannot be a ZST");
-        let drop_fn = vtable
-            .read_pointer(pointer_size * u64::try_from(COMMON_VTABLE_ENTRIES_DROPINPLACE).unwrap())?
-            .check_init()?;
-        // We *need* an instance here, no other kind of function value, to be able
-        // to determine the type.
-        let drop_instance = self.get_ptr_fn(self.scalar_to_ptr(drop_fn)?)?.as_instance()?;
-        trace!("Found drop fn: {:?}", drop_instance);
-        let fn_sig = drop_instance.ty(*self.tcx, self.param_env).fn_sig(*self.tcx);
-        let fn_sig = self.tcx.normalize_erasing_late_bound_regions(self.param_env, fn_sig);
-        // The drop function takes `*mut T` where `T` is the type being dropped, so get that.
-        let args = fn_sig.inputs();
-        if args.len() != 1 {
-            throw_ub!(InvalidVtableDropFn(fn_sig));
-        }
-        let ty =
-            args[0].builtin_deref(true).ok_or_else(|| err_ub!(InvalidVtableDropFn(fn_sig)))?.ty;
-        Ok((drop_instance, ty))
-    }
-
-    pub fn read_size_and_align_from_vtable(
+    pub fn get_vtable_size_and_align(
         &self,
         vtable: Pointer<Option<M::PointerTag>>,
     ) -> InterpResult<'tcx, (Size, Align)> {
-        let pointer_size = self.pointer_size();
-        // We check for `size = 3 * ptr_size`, which covers the drop fn (unused here),
-        // the size, and the align (which we read below).
-        let vtable = self
-            .get_ptr_alloc(
-                vtable,
-                pointer_size * u64::try_from(TyCtxt::COMMON_VTABLE_ENTRIES.len()).unwrap(),
-                self.tcx.data_layout.pointer_align.abi,
-            )?
-            .expect("cannot be a ZST");
-        let size = vtable
-            .read_integer(alloc_range(
-                pointer_size * u64::try_from(COMMON_VTABLE_ENTRIES_SIZE).unwrap(),
-                pointer_size,
-            ))?
-            .check_init()?;
-        let size = size.to_machine_usize(self)?;
-        let size = Size::from_bytes(size);
-        let align = vtable
-            .read_integer(alloc_range(
-                pointer_size * u64::try_from(COMMON_VTABLE_ENTRIES_ALIGN).unwrap(),
-                pointer_size,
-            ))?
-            .check_init()?;
-        let align = align.to_machine_usize(self)?;
-        let align = Align::from_bytes(align).map_err(|e| err_ub!(InvalidVtableAlignment(e)))?;
-
-        if size > self.max_size_of_val() {
-            throw_ub!(InvalidVtableSize);
-        }
-        Ok((size, align))
-    }
-
-    pub fn read_new_vtable_after_trait_upcasting_from_vtable(
-        &self,
-        vtable: Pointer<Option<M::PointerTag>>,
-        idx: u64,
-    ) -> InterpResult<'tcx, Pointer<Option<M::PointerTag>>> {
-        let pointer_size = self.pointer_size();
-
-        let vtable_slot = vtable.offset(pointer_size * idx, self)?;
-        let new_vtable = self
-            .get_ptr_alloc(vtable_slot, pointer_size, self.tcx.data_layout.pointer_align.abi)?
-            .expect("cannot be a ZST");
-
-        let new_vtable = self.scalar_to_ptr(new_vtable.read_pointer(Size::ZERO)?.check_init()?)?;
-
-        Ok(new_vtable)
+        let (ty, _trait_ref) = self.get_ptr_vtable(vtable)?;
+        let layout = self.layout_of(ty)?;
+        assert!(!layout.is_unsized(), "there are no vtables for unsized types");
+        Ok((layout.size, layout.align.abi))
     }
 }