From be3452a02f7006f50acca2b89e04a9bb710ad652 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?John=20K=C3=A5re=20Alsaker?= Date: Fri, 3 Feb 2023 08:05:59 +0100 Subject: [PATCH 1/8] Test that TLS access works outside of the dylib it's defined in --- tests/ui/auxiliary/tls-export.rs | 13 +++++++++++++ tests/ui/tls-dylib-access.rs | 13 +++++++++++++ 2 files changed, 26 insertions(+) create mode 100644 tests/ui/auxiliary/tls-export.rs create mode 100644 tests/ui/tls-dylib-access.rs diff --git a/tests/ui/auxiliary/tls-export.rs b/tests/ui/auxiliary/tls-export.rs new file mode 100644 index 0000000000000..f02faaa4c26d7 --- /dev/null +++ b/tests/ui/auxiliary/tls-export.rs @@ -0,0 +1,13 @@ +#![crate_type = "dylib"] +#![feature(thread_local)] +#![feature(cfg_target_thread_local)] + +#[cfg(target_thread_local)] +#[thread_local] +pub static FOO: bool = true; + +#[cfg(target_thread_local)] +#[inline(never)] +pub fn foo_addr() -> usize { + &FOO as *const bool as usize +} diff --git a/tests/ui/tls-dylib-access.rs b/tests/ui/tls-dylib-access.rs new file mode 100644 index 0000000000000..1ee8cf2f7247d --- /dev/null +++ b/tests/ui/tls-dylib-access.rs @@ -0,0 +1,13 @@ +// aux-build: tls-export.rs +// run-pass + +#![feature(cfg_target_thread_local)] + +#[cfg(target_thread_local)] +extern crate tls_export; + +fn main() { + // Check that we get the real address of the TLS in the dylib + #[cfg(target_thread_local)] + assert_eq!(&tls_export::FOO as *const bool as usize, tls_export::foo_addr()); +} From 34c87080edee005f80775d895b629082b5c187cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?John=20K=C3=A5re=20Alsaker?= Date: Fri, 3 Feb 2023 09:04:12 +0100 Subject: [PATCH 2/8] Support TLS access into dylibs on Windows --- .../src/back/symbol_export.rs | 40 +++++- compiler/rustc_codegen_ssa/src/mir/block.rs | 71 ++++++++--- compiler/rustc_codegen_ssa/src/mir/mod.rs | 120 +++++++++++++++++- .../src/interpret/terminator.rs | 1 + .../src/rmeta/decoder/cstore_impl.rs | 12 ++ .../src/middle/exported_symbols.rs | 5 + compiler/rustc_middle/src/mir/mono.rs | 1 + compiler/rustc_middle/src/mir/visit.rs | 1 + compiler/rustc_middle/src/query/mod.rs | 4 + compiler/rustc_middle/src/ty/instance.rs | 16 ++- compiler/rustc_middle/src/ty/mod.rs | 1 + compiler/rustc_mir_transform/src/inline.rs | 1 + .../rustc_mir_transform/src/inline/cycle.rs | 1 + compiler/rustc_mir_transform/src/shim.rs | 31 +++++ compiler/rustc_monomorphize/src/collector.rs | 24 +++- .../src/partitioning/default.rs | 10 ++ compiler/rustc_symbol_mangling/src/legacy.rs | 4 + compiler/rustc_symbol_mangling/src/v0.rs | 1 + compiler/rustc_target/src/spec/mod.rs | 5 + compiler/rustc_target/src/spec/msvc_base.rs | 1 + .../rustc_target/src/spec/windows_gnu_base.rs | 1 + .../src/spec/windows_gnullvm_base.rs | 1 + compiler/rustc_ty_utils/src/abi.rs | 16 ++- 23 files changed, 333 insertions(+), 35 deletions(-) diff --git a/compiler/rustc_codegen_ssa/src/back/symbol_export.rs b/compiler/rustc_codegen_ssa/src/back/symbol_export.rs index 11bd47a8f0c79..c465fd43f2962 100644 --- a/compiler/rustc_codegen_ssa/src/back/symbol_export.rs +++ b/compiler/rustc_codegen_ssa/src/back/symbol_export.rs @@ -178,14 +178,29 @@ fn exported_symbols_provider_local( // FIXME: Sorting this is unnecessary since we are sorting later anyway. // Can we skip the later sorting? - let mut symbols: Vec<_> = tcx.with_stable_hashing_context(|hcx| { - tcx.reachable_non_generics(LOCAL_CRATE) - .to_sorted(&hcx, true) - .into_iter() - .map(|(&def_id, &info)| (ExportedSymbol::NonGeneric(def_id), info)) - .collect() + let sorted = tcx.with_stable_hashing_context(|hcx| { + tcx.reachable_non_generics(LOCAL_CRATE).to_sorted(&hcx, true) }); + let mut symbols: Vec<_> = + sorted.iter().map(|(&def_id, &info)| (ExportedSymbol::NonGeneric(def_id), info)).collect(); + + // Export TLS shims + if !tcx.sess.target.dll_tls_export { + symbols.extend(sorted.iter().filter_map(|(&def_id, &info)| { + tcx.is_thread_local_static(def_id).then(|| { + ( + ExportedSymbol::ThreadLocalShim(def_id), + SymbolExportInfo { + level: info.level, + kind: SymbolExportKind::Text, + used: info.used, + }, + ) + }) + })) + } + if tcx.entry_fn(()).is_some() { let exported_symbol = ExportedSymbol::NoDefId(SymbolName::new(tcx, tcx.sess.target.entry_name.as_ref())); @@ -380,7 +395,9 @@ fn upstream_monomorphizations_provider( continue; } } - ExportedSymbol::NonGeneric(..) | ExportedSymbol::NoDefId(..) => { + ExportedSymbol::NonGeneric(..) + | ExportedSymbol::ThreadLocalShim(..) + | ExportedSymbol::NoDefId(..) => { // These are no monomorphizations continue; } @@ -500,6 +517,13 @@ pub fn symbol_name_for_instance_in_crate<'tcx>( instantiating_crate, ) } + ExportedSymbol::ThreadLocalShim(def_id) => { + rustc_symbol_mangling::symbol_name_for_instance_in_crate( + tcx, + Instance::new(def_id, ty::InternalSubsts::empty()), + instantiating_crate, + ) + } ExportedSymbol::DropGlue(ty) => rustc_symbol_mangling::symbol_name_for_instance_in_crate( tcx, Instance::resolve_drop_in_place(tcx, ty), @@ -548,6 +572,8 @@ pub fn linking_symbol_name_for_instance_in_crate<'tcx>( ExportedSymbol::DropGlue(..) => None, // NoDefId always follow the target's default symbol decoration scheme. ExportedSymbol::NoDefId(..) => None, + // ThreadLocalShim always follow the target's default symbol decoration scheme. + ExportedSymbol::ThreadLocalShim(..) => None, }; let (conv, args) = instance diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs index cdc3e1dc237b2..e301acee3bbd3 100644 --- a/compiler/rustc_codegen_ssa/src/mir/block.rs +++ b/compiler/rustc_codegen_ssa/src/mir/block.rs @@ -12,7 +12,8 @@ use crate::MemFlags; use rustc_ast as ast; use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece}; use rustc_hir::lang_items::LangItem; -use rustc_index::vec::Idx; +use rustc_hir::Unsafety; +use rustc_index::vec::{Idx, IndexVec}; use rustc_middle::mir::{self, AssertKind, SwitchTargets}; use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf}; use rustc_middle::ty::print::{with_no_trimmed_paths, with_no_visible_paths}; @@ -743,25 +744,58 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { let source_info = terminator.source_info; let span = source_info.span; - // Create the callee. This is a fn ptr or zero-sized and hence a kind of scalar. - let callee = self.codegen_operand(bx, func); + let thread_local_shim_call = if !self.call_thread_local_shims.is_empty() { + self.call_thread_local_shims.iter().find(|e| &e.0 == func).map(|e| e.1) + } else { + None + }; - let (instance, mut llfn) = match *callee.layout.ty.kind() { - ty::FnDef(def_id, substs) => ( - Some( - ty::Instance::expect_resolve( - bx.tcx(), - ty::ParamEnv::reveal_all(), - def_id, - substs, - ) - .polymorphize(bx.tcx()), - ), - None, - ), - ty::FnPtr(_) => (None, Some(callee.immediate())), - _ => bug!("{} is not callable", callee.layout.ty), + let (sig, instance, mut llfn) = match thread_local_shim_call { + Some(thread_local) => { + // Replace thread local dummy calls with calls to the real shim + let instance = ty::Instance { + def: ty::InstanceDef::ThreadLocalShim(thread_local), + substs: ty::InternalSubsts::empty(), + }; + let ty = mir::Rvalue::ThreadLocalRef(thread_local).ty(&IndexVec::new(), bx.tcx()); + ( + ty::Binder::dummy(bx.tcx().mk_fn_sig( + [].iter(), + &ty, + false, + Unsafety::Normal, + Abi::Unadjusted, + )), + Some(instance), + None, + ) + } + None => { + // Create the callee. This is a fn ptr or zero-sized and hence a kind of scalar. + let callee = self.codegen_operand(bx, func); + + let sig = callee.layout.ty.fn_sig(bx.tcx()); + + match *callee.layout.ty.kind() { + ty::FnDef(def_id, substs) => ( + sig, + Some( + ty::Instance::expect_resolve( + bx.tcx(), + ty::ParamEnv::reveal_all(), + def_id, + substs, + ) + .polymorphize(bx.tcx()), + ), + None, + ), + ty::FnPtr(_) => (sig, None, Some(callee.immediate())), + _ => bug!("{} is not callable", callee.layout.ty), + } + } }; + let def = instance.map(|i| i.def); if let Some(ty::InstanceDef::DropGlue(_, None)) = def { @@ -773,7 +807,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { // FIXME(eddyb) avoid computing this if possible, when `instance` is // available - right now `sig` is only needed for getting the `abi` // and figuring out how many extra args were passed to a C-variadic `fn`. - let sig = callee.layout.ty.fn_sig(bx.tcx()); let abi = sig.abi(); // Handle intrinsics old codegen wants Expr's for, ourselves. diff --git a/compiler/rustc_codegen_ssa/src/mir/mod.rs b/compiler/rustc_codegen_ssa/src/mir/mod.rs index 2ec9fdbf44f11..78bf37ecff7c0 100644 --- a/compiler/rustc_codegen_ssa/src/mir/mod.rs +++ b/compiler/rustc_codegen_ssa/src/mir/mod.rs @@ -1,9 +1,15 @@ use crate::base; use crate::traits::*; +use rustc_hir::def_id::DefId; +use rustc_index::vec::Idx; use rustc_middle::mir; use rustc_middle::mir::interpret::ErrorHandled; +use rustc_middle::mir::visit::MutVisitor; +use rustc_middle::mir::visit::Visitor; use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt, TyAndLayout}; +use rustc_middle::ty::TyCtxt; use rustc_middle::ty::{self, Instance, Ty, TyCtxt, TypeFoldable, TypeVisitableExt}; +use rustc_span::DUMMY_SP; use rustc_target::abi::call::{FnAbi, PassMode}; use std::iter; @@ -43,6 +49,9 @@ pub struct FunctionCx<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> { fn_abi: &'tcx FnAbi<'tcx, Ty<'tcx>>, + // Used to replace call terminators with a call to a thread local shim. + call_thread_local_shims: Vec<(mir::Operand<'tcx>, DefId)>, + /// When unwinding is initiated, we have to store this personality /// value somewhere so that we can load it and re-use it in the /// resume instruction. The personality is (afaik) some kind of @@ -142,6 +151,112 @@ impl<'a, 'tcx, V: CodegenObject> LocalRef<'tcx, V> { } } +struct FindThreadLocal(bool); + +impl<'tcx> Visitor<'tcx> for FindThreadLocal { + fn visit_rvalue(&mut self, rvalue: &mir::Rvalue<'tcx>, location: mir::Location) { + if let mir::Rvalue::ThreadLocalRef(..) = rvalue { + self.0 = true; + } + self.super_rvalue(rvalue, location); + } +} + +struct ReplaceThreadLocal<'tcx> { + tcx: TyCtxt<'tcx>, + local_start: usize, + list: Vec, +} + +impl<'tcx> MutVisitor<'tcx> for ReplaceThreadLocal<'tcx> { + fn tcx(&self) -> TyCtxt<'tcx> { + self.tcx + } + + fn visit_rvalue(&mut self, rvalue: &mut mir::Rvalue<'tcx>, location: mir::Location) { + if let mir::Rvalue::ThreadLocalRef(def_id) = *rvalue { + if self.tcx.is_in_upstream_dylib(def_id.krate) { + *rvalue = mir::Rvalue::Use(mir::Operand::Copy(mir::Place { + local: mir::Local::new(self.local_start + self.list.len()), + projection: self.tcx.intern_place_elems(&[]), + })); + self.list.push(def_id); + } + } + self.super_rvalue(rvalue, location); + } +} + +// Convert thread local references to thread local function shims if necessary +fn convert_tls_rvalues<'tcx>( + tcx: TyCtxt<'tcx>, + mir: &mut &'tcx mir::Body<'tcx>, +) -> Vec<(mir::Operand<'tcx>, DefId)> { + if tcx.sess.target.dll_tls_export { + // The target supports DLL TLS exports. We don't need to do anything + return Vec::new(); + } + + // Fast path to look for any thread locals + let mut visitor = FindThreadLocal(false); + visitor.visit_body(&mir); + if !visitor.0 { + return Vec::new(); + } + + // Don't modify shims + if let ty::InstanceDef::ThreadLocalShim(..) = mir.source.instance { + return Vec::new(); + } + + let mut result = Vec::new(); + let mut body = mir.clone(); + + let mut visitor = + ReplaceThreadLocal { tcx, local_start: mir.local_decls.len(), list: Vec::new() }; + visitor.visit_body(&mut body); + + for (i, &def_id) in visitor.list.iter().enumerate() { + let ty = mir::Rvalue::ThreadLocalRef(def_id).ty(&IndexVec::new(), tcx); + body.local_decls.push(mir::LocalDecl::new(ty, DUMMY_SP)); + let local = mir::Local::new(visitor.local_start + i); + let place = mir::Place { local, projection: tcx.intern_place_elems(&[]) }; + let func = mir::Operand::Copy(place); + + result.push((func.clone(), def_id)); + + let blocks = body.basic_blocks.as_mut(); + + let new_entry = mir::BasicBlock::new(blocks.len()); + + let entry = std::mem::replace( + &mut blocks[mir::BasicBlock::new(0)], + mir::BasicBlockData { + statements: Vec::new(), + terminator: Some(mir::Terminator { + source_info: mir::SourceInfo::outermost(DUMMY_SP), + kind: mir::TerminatorKind::Call { + func, + args: Vec::new(), + destination: place, + target: Some(new_entry), + cleanup: None, + from_hir_call: false, + fn_span: DUMMY_SP, + }, + }), + is_cleanup: false, + }, + ); + + blocks.push(entry); + } + + *mir = tcx.arena.alloc(body); + + result +} + /////////////////////////////////////////////////////////////////////////// #[instrument(level = "debug", skip(cx))] @@ -153,7 +268,9 @@ pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( let llfn = cx.get_fn(instance); - let mir = cx.tcx().instance_mir(instance.def); + let mut mir = cx.tcx().instance_mir(instance.def); + + let call_thread_local_shims = convert_tls_rvalues(cx.tcx(), &mut mir); let fn_abi = cx.fn_abi_of_instance(instance, ty::List::empty()); debug!("fn_abi: {:?}", fn_abi); @@ -183,6 +300,7 @@ pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( llfn, fn_abi, cx, + call_thread_local_shims, personality_slot: None, cached_llbbs, unreachable_block: None, diff --git a/compiler/rustc_const_eval/src/interpret/terminator.rs b/compiler/rustc_const_eval/src/interpret/terminator.rs index d934cfbbb84ea..6a53775ab8e10 100644 --- a/compiler/rustc_const_eval/src/interpret/terminator.rs +++ b/compiler/rustc_const_eval/src/interpret/terminator.rs @@ -389,6 +389,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { | ty::InstanceDef::FnPtrShim(..) | ty::InstanceDef::DropGlue(..) | ty::InstanceDef::CloneShim(..) + | ty::InstanceDef::ThreadLocalShim(..) | ty::InstanceDef::Item(_) => { // We need MIR for this fn let Some((body, instance)) = diff --git a/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs b/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs index 83a0e833edc1d..15de38cc14f28 100644 --- a/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs +++ b/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs @@ -11,6 +11,7 @@ use rustc_hir::def_id::{CrateNum, DefId, DefIdMap, LOCAL_CRATE}; use rustc_hir::definitions::{DefKey, DefPath, DefPathHash}; use rustc_middle::arena::ArenaAllocatable; use rustc_middle::metadata::ModChild; +use rustc_middle::middle::dependency_format::Linkage; use rustc_middle::middle::exported_symbols::ExportedSymbol; use rustc_middle::middle::stability::DeprecationEntry; use rustc_middle::ty::fast_reject::SimplifiedType; @@ -501,6 +502,17 @@ pub(in crate::rmeta) fn provide(providers: &mut Providers) { tcx.arena .alloc_slice(&CStore::from_tcx(tcx).crate_dependencies_in_postorder(LOCAL_CRATE)) }, + is_in_upstream_dylib: |tcx, cnum| { + if cnum == LOCAL_CRATE { + return false; + } + tcx.dependency_formats(()).iter().any(|(_, linkage)| { + match linkage[cnum.as_usize() - 1] { + Linkage::NotLinked | Linkage::Static => false, + Linkage::IncludedFromDylib | Linkage::Dynamic => true, + } + }) + }, crates: |tcx, ()| tcx.arena.alloc_from_iter(CStore::from_tcx(tcx).crates_untracked()), ..*providers }; diff --git a/compiler/rustc_middle/src/middle/exported_symbols.rs b/compiler/rustc_middle/src/middle/exported_symbols.rs index 631fd09ec4cf6..c0c0fd07b6e06 100644 --- a/compiler/rustc_middle/src/middle/exported_symbols.rs +++ b/compiler/rustc_middle/src/middle/exported_symbols.rs @@ -43,6 +43,7 @@ pub enum ExportedSymbol<'tcx> { NonGeneric(DefId), Generic(DefId, SubstsRef<'tcx>), DropGlue(Ty<'tcx>), + ThreadLocalShim(DefId), NoDefId(ty::SymbolName<'tcx>), } @@ -58,6 +59,10 @@ impl<'tcx> ExportedSymbol<'tcx> { ExportedSymbol::DropGlue(ty) => { tcx.symbol_name(ty::Instance::resolve_drop_in_place(tcx, ty)) } + ExportedSymbol::ThreadLocalShim(def_id) => tcx.symbol_name(ty::Instance { + def: ty::InstanceDef::ThreadLocalShim(def_id), + substs: ty::InternalSubsts::empty(), + }), ExportedSymbol::NoDefId(symbol_name) => symbol_name, } } diff --git a/compiler/rustc_middle/src/mir/mono.rs b/compiler/rustc_middle/src/mir/mono.rs index 7a05ee2ff37fd..c66d9fb51b917 100644 --- a/compiler/rustc_middle/src/mir/mono.rs +++ b/compiler/rustc_middle/src/mir/mono.rs @@ -381,6 +381,7 @@ impl<'tcx> CodegenUnit<'tcx> { | InstanceDef::Virtual(..) | InstanceDef::ClosureOnceShim { .. } | InstanceDef::DropGlue(..) + | InstanceDef::ThreadLocalShim(..) | InstanceDef::CloneShim(..) => None, } } diff --git a/compiler/rustc_middle/src/mir/visit.rs b/compiler/rustc_middle/src/mir/visit.rs index 7f0935fb149fa..ef9b6911318de 100644 --- a/compiler/rustc_middle/src/mir/visit.rs +++ b/compiler/rustc_middle/src/mir/visit.rs @@ -335,6 +335,7 @@ macro_rules! make_mir_visitor { ty::InstanceDef::VTableShim(_def_id) | ty::InstanceDef::ReifyShim(_def_id) | ty::InstanceDef::Virtual(_def_id, _) | + ty::InstanceDef::ThreadLocalShim(_def_id) | ty::InstanceDef::ClosureOnceShim { call_once: _def_id, track_caller: _ } | ty::InstanceDef::DropGlue(_def_id, None) => {} diff --git a/compiler/rustc_middle/src/query/mod.rs b/compiler/rustc_middle/src/query/mod.rs index 6a34e5ede1938..558496a8da537 100644 --- a/compiler/rustc_middle/src/query/mod.rs +++ b/compiler/rustc_middle/src/query/mod.rs @@ -1808,6 +1808,10 @@ rustc_queries! { eval_always desc { "generating a postorder list of CrateNums" } } + query is_in_upstream_dylib(_: CrateNum) -> bool { + eval_always + desc { "checking if a crate is placed in an upstream dylib" } + } /// Returns whether or not the crate with CrateNum 'cnum' /// is marked as a private dependency query is_private_dep(c: CrateNum) -> bool { diff --git a/compiler/rustc_middle/src/ty/instance.rs b/compiler/rustc_middle/src/ty/instance.rs index d07fa0e546f34..d356cbfb29641 100644 --- a/compiler/rustc_middle/src/ty/instance.rs +++ b/compiler/rustc_middle/src/ty/instance.rs @@ -82,6 +82,10 @@ pub enum InstanceDef<'tcx> { /// The `DefId` is the ID of the `call_once` method in `FnOnce`. ClosureOnceShim { call_once: DefId, track_caller: bool }, + /// Compiler-generated accessor for thread locals. This is used to export thread locals + /// from dylibs on platforms lacking native support. + ThreadLocalShim(DefId), + /// `core::ptr::drop_in_place::`. /// /// The `DefId` is for `core::ptr::drop_in_place`. @@ -149,6 +153,7 @@ impl<'tcx> InstanceDef<'tcx> { | InstanceDef::FnPtrShim(def_id, _) | InstanceDef::Virtual(def_id, _) | InstanceDef::Intrinsic(def_id) + | InstanceDef::ThreadLocalShim(def_id) | InstanceDef::ClosureOnceShim { call_once: def_id, track_caller: _ } | InstanceDef::DropGlue(def_id, _) | InstanceDef::CloneShim(def_id, _) => def_id, @@ -159,7 +164,9 @@ impl<'tcx> InstanceDef<'tcx> { pub fn def_id_if_not_guaranteed_local_codegen(self) -> Option { match self { ty::InstanceDef::Item(def) => Some(def.did), - ty::InstanceDef::DropGlue(def_id, Some(_)) => Some(def_id), + ty::InstanceDef::DropGlue(def_id, Some(_)) | InstanceDef::ThreadLocalShim(def_id) => { + Some(def_id) + } InstanceDef::VTableShim(..) | InstanceDef::ReifyShim(..) | InstanceDef::FnPtrShim(..) @@ -182,6 +189,7 @@ impl<'tcx> InstanceDef<'tcx> { | InstanceDef::Intrinsic(def_id) | InstanceDef::ClosureOnceShim { call_once: def_id, track_caller: _ } | InstanceDef::DropGlue(def_id, _) + | InstanceDef::ThreadLocalShim(def_id) | InstanceDef::CloneShim(def_id, _) => ty::WithOptConstParam::unknown(def_id), } } @@ -201,6 +209,7 @@ impl<'tcx> InstanceDef<'tcx> { let def_id = match *self { ty::InstanceDef::Item(def) => def.did, ty::InstanceDef::DropGlue(_, Some(_)) => return false, + ty::InstanceDef::ThreadLocalShim(_) => return false, _ => return true, }; matches!( @@ -241,6 +250,9 @@ impl<'tcx> InstanceDef<'tcx> { ) }); } + if let ty::InstanceDef::ThreadLocalShim(..) = *self { + return false; + } tcx.codegen_fn_attrs(self.def_id()).requests_inline() } @@ -264,6 +276,7 @@ impl<'tcx> InstanceDef<'tcx> { pub fn has_polymorphic_mir_body(&self) -> bool { match *self { InstanceDef::CloneShim(..) + | InstanceDef::ThreadLocalShim(..) | InstanceDef::FnPtrShim(..) | InstanceDef::DropGlue(_, Some(_)) => false, InstanceDef::ClosureOnceShim { .. } @@ -295,6 +308,7 @@ fn fmt_instance( InstanceDef::Item(_) => Ok(()), InstanceDef::VTableShim(_) => write!(f, " - shim(vtable)"), InstanceDef::ReifyShim(_) => write!(f, " - shim(reify)"), + InstanceDef::ThreadLocalShim(_) => write!(f, " - shim(tls)"), InstanceDef::Intrinsic(_) => write!(f, " - intrinsic"), InstanceDef::Virtual(_, num) => write!(f, " - virtual#{}", num), InstanceDef::FnPtrShim(_, ty) => write!(f, " - shim({})", ty), diff --git a/compiler/rustc_middle/src/ty/mod.rs b/compiler/rustc_middle/src/ty/mod.rs index 7dcc3ff4e7b33..46ca1778e8a0a 100644 --- a/compiler/rustc_middle/src/ty/mod.rs +++ b/compiler/rustc_middle/src/ty/mod.rs @@ -2346,6 +2346,7 @@ impl<'tcx> TyCtxt<'tcx> { | ty::InstanceDef::Virtual(..) | ty::InstanceDef::ClosureOnceShim { .. } | ty::InstanceDef::DropGlue(..) + | ty::InstanceDef::ThreadLocalShim(..) | ty::InstanceDef::CloneShim(..) => self.mir_shims(instance), } } diff --git a/compiler/rustc_mir_transform/src/inline.rs b/compiler/rustc_mir_transform/src/inline.rs index 8c6b0463a739a..66c72f00b6a14 100644 --- a/compiler/rustc_mir_transform/src/inline.rs +++ b/compiler/rustc_mir_transform/src/inline.rs @@ -270,6 +270,7 @@ impl<'tcx> Inliner<'tcx> { | InstanceDef::FnPtrShim(..) | InstanceDef::ClosureOnceShim { .. } | InstanceDef::DropGlue(..) + | InstanceDef::ThreadLocalShim(..) | InstanceDef::CloneShim(..) => return Ok(()), } diff --git a/compiler/rustc_mir_transform/src/inline/cycle.rs b/compiler/rustc_mir_transform/src/inline/cycle.rs index 792457c80b0b7..0a3b498edf30a 100644 --- a/compiler/rustc_mir_transform/src/inline/cycle.rs +++ b/compiler/rustc_mir_transform/src/inline/cycle.rs @@ -83,6 +83,7 @@ pub(crate) fn mir_callgraph_reachable<'tcx>( | InstanceDef::ReifyShim(_) | InstanceDef::FnPtrShim(..) | InstanceDef::ClosureOnceShim { .. } + | InstanceDef::ThreadLocalShim { .. } | InstanceDef::CloneShim(..) => {} InstanceDef::DropGlue(..) => { // FIXME: A not fully substituted drop shim can cause ICEs if one attempts to diff --git a/compiler/rustc_mir_transform/src/shim.rs b/compiler/rustc_mir_transform/src/shim.rs index 682ad081f5cf3..c312b8512caf8 100644 --- a/compiler/rustc_mir_transform/src/shim.rs +++ b/compiler/rustc_mir_transform/src/shim.rs @@ -76,6 +76,7 @@ fn make_shim<'tcx>(tcx: TyCtxt<'tcx>, instance: ty::InstanceDef<'tcx>) -> Body<' build_drop_shim(tcx, def_id, ty) } + ty::InstanceDef::ThreadLocalShim(..) => build_thread_local_shim(tcx, instance), ty::InstanceDef::CloneShim(def_id, ty) => build_clone_shim(tcx, def_id, ty), ty::InstanceDef::Virtual(..) => { bug!("InstanceDef::Virtual ({:?}) is for direct calls only", instance) @@ -321,6 +322,36 @@ impl<'a, 'tcx> DropElaborator<'a, 'tcx> for DropShimElaborator<'a, 'tcx> { } } +fn build_thread_local_shim<'tcx>(tcx: TyCtxt<'tcx>, instance: ty::InstanceDef<'tcx>) -> Body<'tcx> { + let def_id = instance.def_id(); + + let span = tcx.def_span(def_id); + let source_info = SourceInfo::outermost(span); + + let mut blocks = IndexVec::with_capacity(1); + blocks.push(BasicBlockData { + statements: vec![Statement { + source_info, + kind: StatementKind::Assign(Box::new(( + Place::return_place(), + Rvalue::ThreadLocalRef(def_id), + ))), + }], + terminator: Some(Terminator { source_info, kind: TerminatorKind::Return }), + is_cleanup: false, + }); + + let ret_ty = Rvalue::ThreadLocalRef(def_id).ty(&IndexVec::new(), tcx); + + new_body( + MirSource::from_instance(instance), + blocks, + iter::once(LocalDecl::new(ret_ty, span)).collect(), + 0, + span, + ) +} + /// Builds a `Clone::clone` shim for `self_ty`. Here, `def_id` is `Clone::clone`. fn build_clone_shim<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId, self_ty: Ty<'tcx>) -> Body<'tcx> { debug!("build_clone_shim(def_id={:?})", def_id); diff --git a/compiler/rustc_monomorphize/src/collector.rs b/compiler/rustc_monomorphize/src/collector.rs index ff409a8071611..be9e06420fe7d 100644 --- a/compiler/rustc_monomorphize/src/collector.rs +++ b/compiler/rustc_monomorphize/src/collector.rs @@ -190,7 +190,8 @@ use rustc_middle::ty::print::with_no_trimmed_paths; use rustc_middle::ty::query::TyCtxtAt; use rustc_middle::ty::subst::{GenericArgKind, InternalSubsts}; use rustc_middle::ty::{ - self, GenericParamDefKind, Instance, Ty, TyCtxt, TypeFoldable, TypeVisitableExt, VtblEntry, + self, GenericParamDefKind, Instance, InstanceDef, Ty, TyCtxt, TypeFoldable, TypeVisitableExt, + VtblEntry, }; use rustc_middle::{middle::codegen_fn_attrs::CodegenFnAttrFlags, mir::visit::TyContext}; use rustc_session::config::EntryFnType; @@ -462,6 +463,16 @@ fn collect_items_rec<'tcx>( collect_miri(tcx, id, &mut neighbors); } } + + if !tcx.sess.target.dll_tls_export && tcx.is_thread_local_static(def_id) { + neighbors.push(respan( + starting_point.span, + MonoItem::Fn(Instance { + def: InstanceDef::ThreadLocalShim(def_id), + substs: InternalSubsts::empty(), + }), + )); + } } MonoItem::Fn(instance) => { // Sanity check whether this ended up being collected accidentally @@ -963,6 +974,9 @@ fn visit_instance_use<'tcx>( bug!("{:?} being reified", instance); } } + ty::InstanceDef::ThreadLocalShim(..) => { + bug!("{:?} being reified", instance); + } ty::InstanceDef::DropGlue(_, None) => { // Don't need to emit noop drop glue if we are calling directly. if !is_direct_call { @@ -1210,11 +1224,9 @@ impl<'v> RootCollector<'_, 'v> { self.output.push(dummy_spanned(MonoItem::GlobalAsm(id))); } DefKind::Static(..) => { - debug!( - "RootCollector: ItemKind::Static({})", - self.tcx.def_path_str(id.owner_id.to_def_id()) - ); - self.output.push(dummy_spanned(MonoItem::Static(id.owner_id.to_def_id()))); + let def_id = id.owner_id.to_def_id(); + debug!("RootCollector: ItemKind::Static({})", self.tcx.def_path_str(def_id)); + self.output.push(dummy_spanned(MonoItem::Static(def_id))); } DefKind::Const => { // const items only generate mono items if they are diff --git a/compiler/rustc_monomorphize/src/partitioning/default.rs b/compiler/rustc_monomorphize/src/partitioning/default.rs index 2c56edd89bc31..d51b8a7552b82 100644 --- a/compiler/rustc_monomorphize/src/partitioning/default.rs +++ b/compiler/rustc_monomorphize/src/partitioning/default.rs @@ -278,6 +278,7 @@ fn characteristic_def_id_of_mono_item<'tcx>( | ty::InstanceDef::Intrinsic(..) | ty::InstanceDef::DropGlue(..) | ty::InstanceDef::Virtual(..) + | ty::InstanceDef::ThreadLocalShim(..) | ty::InstanceDef::CloneShim(..) => return None, }; @@ -424,6 +425,15 @@ fn mono_item_visibility<'tcx>( InstanceDef::Item(def) => def.did, InstanceDef::DropGlue(def_id, Some(_)) => def_id, + InstanceDef::ThreadLocalShim(def_id) => { + return if tcx.is_reachable_non_generic(def_id) { + *can_be_internalized = false; + default_visibility(tcx, def_id, false) + } else { + Visibility::Hidden + }; + } + // These are all compiler glue and such, never exported, always hidden. InstanceDef::VTableShim(..) | InstanceDef::ReifyShim(..) diff --git a/compiler/rustc_symbol_mangling/src/legacy.rs b/compiler/rustc_symbol_mangling/src/legacy.rs index 2368468c89123..5cbca81926b9a 100644 --- a/compiler/rustc_symbol_mangling/src/legacy.rs +++ b/compiler/rustc_symbol_mangling/src/legacy.rs @@ -65,6 +65,10 @@ pub(super) fn mangle<'tcx>( ) .unwrap(); + if let ty::InstanceDef::ThreadLocalShim(..) = instance.def { + let _ = printer.write_str("{{tls-shim}}"); + } + if let ty::InstanceDef::VTableShim(..) = instance.def { let _ = printer.write_str("{{vtable-shim}}"); } diff --git a/compiler/rustc_symbol_mangling/src/v0.rs b/compiler/rustc_symbol_mangling/src/v0.rs index 2f20d42139c8d..cac7ff72267db 100644 --- a/compiler/rustc_symbol_mangling/src/v0.rs +++ b/compiler/rustc_symbol_mangling/src/v0.rs @@ -42,6 +42,7 @@ pub(super) fn mangle<'tcx>( // Append `::{shim:...#0}` to shims that can coexist with a non-shim instance. let shim_kind = match instance.def { + ty::InstanceDef::ThreadLocalShim(_) => Some("tls"), ty::InstanceDef::VTableShim(_) => Some("vtable"), ty::InstanceDef::ReifyShim(_) => Some("reify"), diff --git a/compiler/rustc_target/src/spec/mod.rs b/compiler/rustc_target/src/spec/mod.rs index 0d86a3032a659..4538f03da3ebf 100644 --- a/compiler/rustc_target/src/spec/mod.rs +++ b/compiler/rustc_target/src/spec/mod.rs @@ -1466,6 +1466,8 @@ pub struct TargetOptions { pub features: StaticCow, /// Whether dynamic linking is available on this target. Defaults to false. pub dynamic_linking: bool, + /// Whether dynamic linking can export TLS globals. Defaults to true. + pub dll_tls_export: bool, /// If dynamic linking is available, whether only cdylibs are supported. pub only_cdylib: bool, /// Whether executables are available on this target. Defaults to true. @@ -1857,6 +1859,7 @@ impl Default for TargetOptions { cpu: "generic".into(), features: "".into(), dynamic_linking: false, + dll_tls_export: true, only_cdylib: false, executables: true, relocation_model: RelocModel::Pic, @@ -2528,6 +2531,7 @@ impl Target { key!(cpu); key!(features); key!(dynamic_linking, bool); + key!(dll_tls_export, bool); key!(only_cdylib, bool); key!(executables, bool); key!(relocation_model, RelocModel)?; @@ -2781,6 +2785,7 @@ impl ToJson for Target { target_option_val!(cpu); target_option_val!(features); target_option_val!(dynamic_linking); + target_option_val!(dll_tls_export); target_option_val!(only_cdylib); target_option_val!(executables); target_option_val!(relocation_model); diff --git a/compiler/rustc_target/src/spec/msvc_base.rs b/compiler/rustc_target/src/spec/msvc_base.rs index 1dad9133ea3d2..efe949a4e9074 100644 --- a/compiler/rustc_target/src/spec/msvc_base.rs +++ b/compiler/rustc_target/src/spec/msvc_base.rs @@ -8,6 +8,7 @@ pub fn opts() -> TargetOptions { TargetOptions { linker_flavor: LinkerFlavor::Msvc(Lld::No), + dll_tls_export: false, is_like_windows: true, is_like_msvc: true, pre_link_args, diff --git a/compiler/rustc_target/src/spec/windows_gnu_base.rs b/compiler/rustc_target/src/spec/windows_gnu_base.rs index a32ca469b2f58..2231983f07126 100644 --- a/compiler/rustc_target/src/spec/windows_gnu_base.rs +++ b/compiler/rustc_target/src/spec/windows_gnu_base.rs @@ -78,6 +78,7 @@ pub fn opts() -> TargetOptions { function_sections: false, linker: Some("gcc".into()), dynamic_linking: true, + dll_tls_export: false, dll_prefix: "".into(), dll_suffix: ".dll".into(), exe_suffix: ".exe".into(), diff --git a/compiler/rustc_target/src/spec/windows_gnullvm_base.rs b/compiler/rustc_target/src/spec/windows_gnullvm_base.rs index cada28652f98a..b1d8e2be5a61f 100644 --- a/compiler/rustc_target/src/spec/windows_gnullvm_base.rs +++ b/compiler/rustc_target/src/spec/windows_gnullvm_base.rs @@ -23,6 +23,7 @@ pub fn opts() -> TargetOptions { abi: "llvm".into(), linker: Some("clang".into()), dynamic_linking: true, + dll_tls_export: false, dll_prefix: "".into(), dll_suffix: ".dll".into(), exe_suffix: ".exe".into(), diff --git a/compiler/rustc_ty_utils/src/abi.rs b/compiler/rustc_ty_utils/src/abi.rs index 41924dc2a6d93..62bd36a09609c 100644 --- a/compiler/rustc_ty_utils/src/abi.rs +++ b/compiler/rustc_ty_utils/src/abi.rs @@ -1,9 +1,11 @@ use rustc_hir as hir; use rustc_hir::lang_items::LangItem; +use rustc_index::vec::IndexVec; +use rustc_middle::mir::Rvalue; use rustc_middle::ty::layout::{ fn_can_unwind, FnAbiError, HasParamEnv, HasTyCtxt, LayoutCx, LayoutOf, TyAndLayout, }; -use rustc_middle::ty::{self, Ty, TyCtxt}; +use rustc_middle::ty::{self, InstanceDef, Ty, TyCtxt}; use rustc_session::config::OptLevel; use rustc_span::def_id::DefId; use rustc_target::abi::call::{ @@ -29,6 +31,18 @@ fn fn_sig_for_fn_abi<'tcx>( instance: ty::Instance<'tcx>, param_env: ty::ParamEnv<'tcx>, ) -> ty::PolyFnSig<'tcx> { + if let InstanceDef::ThreadLocalShim(..) = instance.def { + let ret_ty = Rvalue::ThreadLocalRef(instance.def_id()).ty(&IndexVec::new(), tcx); + + return ty::Binder::dummy(tcx.mk_fn_sig( + [].iter(), + &ret_ty, + false, + hir::Unsafety::Normal, + rustc_target::spec::abi::Abi::Unadjusted, + )); + } + let ty = instance.ty(tcx, param_env); match *ty.kind() { ty::FnDef(..) => { From 98d2669d0def4d49f7f4b7f5b2ace9265952900f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?John=20K=C3=A5re=20Alsaker?= Date: Wed, 15 Feb 2023 16:50:28 +0100 Subject: [PATCH 3/8] Use #[inline] on Windows for thread local access --- library/std/src/thread/local.rs | 26 ++------------------------ 1 file changed, 2 insertions(+), 24 deletions(-) diff --git a/library/std/src/thread/local.rs b/library/std/src/thread/local.rs index cf7c2e05a2e9d..8f573b5ce5db9 100644 --- a/library/std/src/thread/local.rs +++ b/library/std/src/thread/local.rs @@ -181,7 +181,7 @@ macro_rules! thread_local { macro_rules! __thread_local_inner { // used to generate the `LocalKey` value for const-initialized thread locals (@key $t:ty, const $init:expr) => {{ - #[cfg_attr(not(windows), inline)] // see comments below + #[cfg_attr(not(bootstrap), inline)] #[deny(unsafe_op_in_unsafe_fn)] unsafe fn __getit( _init: $crate::option::Option<&mut $crate::option::Option<$t>>, @@ -293,29 +293,7 @@ macro_rules! __thread_local_inner { #[inline] fn __init() -> $t { $init } - // When reading this function you might ask "why is this inlined - // everywhere other than Windows?", and that's a very reasonable - // question to ask. The short story is that it segfaults rustc if - // this function is inlined. The longer story is that Windows looks - // to not support `extern` references to thread locals across DLL - // boundaries. This appears to at least not be supported in the ABI - // that LLVM implements. - // - // Because of this we never inline on Windows, but we do inline on - // other platforms (where external references to thread locals - // across DLLs are supported). A better fix for this would be to - // inline this function on Windows, but only for "statically linked" - // components. For example if two separately compiled rlibs end up - // getting linked into a DLL then it's fine to inline this function - // across that boundary. It's only not fine to inline this function - // across a DLL boundary. Unfortunately rustc doesn't currently - // have this sort of logic available in an attribute, and it's not - // clear that rustc is even equipped to answer this (it's more of a - // Cargo question kinda). This means that, unfortunately, Windows - // gets the pessimistic path for now where it's never inlined. - // - // The issue of "should enable on Windows sometimes" is #84933 - #[cfg_attr(not(windows), inline)] + #[cfg_attr(not(bootstrap), inline)] unsafe fn __getit( init: $crate::option::Option<&mut $crate::option::Option<$t>>, ) -> $crate::option::Option<&'static $t> { From c044df2d5fd19ef0b74e34de16c1a3afe9efdcdc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?John=20K=C3=A5re=20Alsaker?= Date: Thu, 16 Feb 2023 20:35:55 +0100 Subject: [PATCH 4/8] Only avoid shims for the local crate --- compiler/rustc_codegen_ssa/src/mir/mod.rs | 2 +- .../rustc_metadata/src/rmeta/decoder/cstore_impl.rs | 12 ------------ compiler/rustc_middle/src/query/mod.rs | 4 ---- 3 files changed, 1 insertion(+), 17 deletions(-) diff --git a/compiler/rustc_codegen_ssa/src/mir/mod.rs b/compiler/rustc_codegen_ssa/src/mir/mod.rs index 78bf37ecff7c0..c84181ff32a27 100644 --- a/compiler/rustc_codegen_ssa/src/mir/mod.rs +++ b/compiler/rustc_codegen_ssa/src/mir/mod.rs @@ -175,7 +175,7 @@ impl<'tcx> MutVisitor<'tcx> for ReplaceThreadLocal<'tcx> { fn visit_rvalue(&mut self, rvalue: &mut mir::Rvalue<'tcx>, location: mir::Location) { if let mir::Rvalue::ThreadLocalRef(def_id) = *rvalue { - if self.tcx.is_in_upstream_dylib(def_id.krate) { + if !def_id.is_local() { *rvalue = mir::Rvalue::Use(mir::Operand::Copy(mir::Place { local: mir::Local::new(self.local_start + self.list.len()), projection: self.tcx.intern_place_elems(&[]), diff --git a/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs b/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs index 15de38cc14f28..83a0e833edc1d 100644 --- a/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs +++ b/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs @@ -11,7 +11,6 @@ use rustc_hir::def_id::{CrateNum, DefId, DefIdMap, LOCAL_CRATE}; use rustc_hir::definitions::{DefKey, DefPath, DefPathHash}; use rustc_middle::arena::ArenaAllocatable; use rustc_middle::metadata::ModChild; -use rustc_middle::middle::dependency_format::Linkage; use rustc_middle::middle::exported_symbols::ExportedSymbol; use rustc_middle::middle::stability::DeprecationEntry; use rustc_middle::ty::fast_reject::SimplifiedType; @@ -502,17 +501,6 @@ pub(in crate::rmeta) fn provide(providers: &mut Providers) { tcx.arena .alloc_slice(&CStore::from_tcx(tcx).crate_dependencies_in_postorder(LOCAL_CRATE)) }, - is_in_upstream_dylib: |tcx, cnum| { - if cnum == LOCAL_CRATE { - return false; - } - tcx.dependency_formats(()).iter().any(|(_, linkage)| { - match linkage[cnum.as_usize() - 1] { - Linkage::NotLinked | Linkage::Static => false, - Linkage::IncludedFromDylib | Linkage::Dynamic => true, - } - }) - }, crates: |tcx, ()| tcx.arena.alloc_from_iter(CStore::from_tcx(tcx).crates_untracked()), ..*providers }; diff --git a/compiler/rustc_middle/src/query/mod.rs b/compiler/rustc_middle/src/query/mod.rs index 558496a8da537..6a34e5ede1938 100644 --- a/compiler/rustc_middle/src/query/mod.rs +++ b/compiler/rustc_middle/src/query/mod.rs @@ -1808,10 +1808,6 @@ rustc_queries! { eval_always desc { "generating a postorder list of CrateNums" } } - query is_in_upstream_dylib(_: CrateNum) -> bool { - eval_always - desc { "checking if a crate is placed in an upstream dylib" } - } /// Returns whether or not the crate with CrateNum 'cnum' /// is marked as a private dependency query is_private_dep(c: CrateNum) -> bool { From 524d14042b5dfbe8fa24a4c1fd9704959969c0c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?John=20K=C3=A5re=20Alsaker?= Date: Thu, 16 Feb 2023 20:59:41 +0100 Subject: [PATCH 5/8] Lower TLS to calls for rvalues directly --- compiler/rustc_codegen_ssa/src/mir/block.rs | 71 +++-------- compiler/rustc_codegen_ssa/src/mir/mod.rs | 120 +------------------ compiler/rustc_codegen_ssa/src/mir/rvalue.rs | 13 +- 3 files changed, 32 insertions(+), 172 deletions(-) diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs index e301acee3bbd3..cdc3e1dc237b2 100644 --- a/compiler/rustc_codegen_ssa/src/mir/block.rs +++ b/compiler/rustc_codegen_ssa/src/mir/block.rs @@ -12,8 +12,7 @@ use crate::MemFlags; use rustc_ast as ast; use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece}; use rustc_hir::lang_items::LangItem; -use rustc_hir::Unsafety; -use rustc_index::vec::{Idx, IndexVec}; +use rustc_index::vec::Idx; use rustc_middle::mir::{self, AssertKind, SwitchTargets}; use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf}; use rustc_middle::ty::print::{with_no_trimmed_paths, with_no_visible_paths}; @@ -744,58 +743,25 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { let source_info = terminator.source_info; let span = source_info.span; - let thread_local_shim_call = if !self.call_thread_local_shims.is_empty() { - self.call_thread_local_shims.iter().find(|e| &e.0 == func).map(|e| e.1) - } else { - None - }; + // Create the callee. This is a fn ptr or zero-sized and hence a kind of scalar. + let callee = self.codegen_operand(bx, func); - let (sig, instance, mut llfn) = match thread_local_shim_call { - Some(thread_local) => { - // Replace thread local dummy calls with calls to the real shim - let instance = ty::Instance { - def: ty::InstanceDef::ThreadLocalShim(thread_local), - substs: ty::InternalSubsts::empty(), - }; - let ty = mir::Rvalue::ThreadLocalRef(thread_local).ty(&IndexVec::new(), bx.tcx()); - ( - ty::Binder::dummy(bx.tcx().mk_fn_sig( - [].iter(), - &ty, - false, - Unsafety::Normal, - Abi::Unadjusted, - )), - Some(instance), - None, - ) - } - None => { - // Create the callee. This is a fn ptr or zero-sized and hence a kind of scalar. - let callee = self.codegen_operand(bx, func); - - let sig = callee.layout.ty.fn_sig(bx.tcx()); - - match *callee.layout.ty.kind() { - ty::FnDef(def_id, substs) => ( - sig, - Some( - ty::Instance::expect_resolve( - bx.tcx(), - ty::ParamEnv::reveal_all(), - def_id, - substs, - ) - .polymorphize(bx.tcx()), - ), - None, - ), - ty::FnPtr(_) => (sig, None, Some(callee.immediate())), - _ => bug!("{} is not callable", callee.layout.ty), - } - } + let (instance, mut llfn) = match *callee.layout.ty.kind() { + ty::FnDef(def_id, substs) => ( + Some( + ty::Instance::expect_resolve( + bx.tcx(), + ty::ParamEnv::reveal_all(), + def_id, + substs, + ) + .polymorphize(bx.tcx()), + ), + None, + ), + ty::FnPtr(_) => (None, Some(callee.immediate())), + _ => bug!("{} is not callable", callee.layout.ty), }; - let def = instance.map(|i| i.def); if let Some(ty::InstanceDef::DropGlue(_, None)) = def { @@ -807,6 +773,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { // FIXME(eddyb) avoid computing this if possible, when `instance` is // available - right now `sig` is only needed for getting the `abi` // and figuring out how many extra args were passed to a C-variadic `fn`. + let sig = callee.layout.ty.fn_sig(bx.tcx()); let abi = sig.abi(); // Handle intrinsics old codegen wants Expr's for, ourselves. diff --git a/compiler/rustc_codegen_ssa/src/mir/mod.rs b/compiler/rustc_codegen_ssa/src/mir/mod.rs index c84181ff32a27..2ec9fdbf44f11 100644 --- a/compiler/rustc_codegen_ssa/src/mir/mod.rs +++ b/compiler/rustc_codegen_ssa/src/mir/mod.rs @@ -1,15 +1,9 @@ use crate::base; use crate::traits::*; -use rustc_hir::def_id::DefId; -use rustc_index::vec::Idx; use rustc_middle::mir; use rustc_middle::mir::interpret::ErrorHandled; -use rustc_middle::mir::visit::MutVisitor; -use rustc_middle::mir::visit::Visitor; use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt, TyAndLayout}; -use rustc_middle::ty::TyCtxt; use rustc_middle::ty::{self, Instance, Ty, TyCtxt, TypeFoldable, TypeVisitableExt}; -use rustc_span::DUMMY_SP; use rustc_target::abi::call::{FnAbi, PassMode}; use std::iter; @@ -49,9 +43,6 @@ pub struct FunctionCx<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> { fn_abi: &'tcx FnAbi<'tcx, Ty<'tcx>>, - // Used to replace call terminators with a call to a thread local shim. - call_thread_local_shims: Vec<(mir::Operand<'tcx>, DefId)>, - /// When unwinding is initiated, we have to store this personality /// value somewhere so that we can load it and re-use it in the /// resume instruction. The personality is (afaik) some kind of @@ -151,112 +142,6 @@ impl<'a, 'tcx, V: CodegenObject> LocalRef<'tcx, V> { } } -struct FindThreadLocal(bool); - -impl<'tcx> Visitor<'tcx> for FindThreadLocal { - fn visit_rvalue(&mut self, rvalue: &mir::Rvalue<'tcx>, location: mir::Location) { - if let mir::Rvalue::ThreadLocalRef(..) = rvalue { - self.0 = true; - } - self.super_rvalue(rvalue, location); - } -} - -struct ReplaceThreadLocal<'tcx> { - tcx: TyCtxt<'tcx>, - local_start: usize, - list: Vec, -} - -impl<'tcx> MutVisitor<'tcx> for ReplaceThreadLocal<'tcx> { - fn tcx(&self) -> TyCtxt<'tcx> { - self.tcx - } - - fn visit_rvalue(&mut self, rvalue: &mut mir::Rvalue<'tcx>, location: mir::Location) { - if let mir::Rvalue::ThreadLocalRef(def_id) = *rvalue { - if !def_id.is_local() { - *rvalue = mir::Rvalue::Use(mir::Operand::Copy(mir::Place { - local: mir::Local::new(self.local_start + self.list.len()), - projection: self.tcx.intern_place_elems(&[]), - })); - self.list.push(def_id); - } - } - self.super_rvalue(rvalue, location); - } -} - -// Convert thread local references to thread local function shims if necessary -fn convert_tls_rvalues<'tcx>( - tcx: TyCtxt<'tcx>, - mir: &mut &'tcx mir::Body<'tcx>, -) -> Vec<(mir::Operand<'tcx>, DefId)> { - if tcx.sess.target.dll_tls_export { - // The target supports DLL TLS exports. We don't need to do anything - return Vec::new(); - } - - // Fast path to look for any thread locals - let mut visitor = FindThreadLocal(false); - visitor.visit_body(&mir); - if !visitor.0 { - return Vec::new(); - } - - // Don't modify shims - if let ty::InstanceDef::ThreadLocalShim(..) = mir.source.instance { - return Vec::new(); - } - - let mut result = Vec::new(); - let mut body = mir.clone(); - - let mut visitor = - ReplaceThreadLocal { tcx, local_start: mir.local_decls.len(), list: Vec::new() }; - visitor.visit_body(&mut body); - - for (i, &def_id) in visitor.list.iter().enumerate() { - let ty = mir::Rvalue::ThreadLocalRef(def_id).ty(&IndexVec::new(), tcx); - body.local_decls.push(mir::LocalDecl::new(ty, DUMMY_SP)); - let local = mir::Local::new(visitor.local_start + i); - let place = mir::Place { local, projection: tcx.intern_place_elems(&[]) }; - let func = mir::Operand::Copy(place); - - result.push((func.clone(), def_id)); - - let blocks = body.basic_blocks.as_mut(); - - let new_entry = mir::BasicBlock::new(blocks.len()); - - let entry = std::mem::replace( - &mut blocks[mir::BasicBlock::new(0)], - mir::BasicBlockData { - statements: Vec::new(), - terminator: Some(mir::Terminator { - source_info: mir::SourceInfo::outermost(DUMMY_SP), - kind: mir::TerminatorKind::Call { - func, - args: Vec::new(), - destination: place, - target: Some(new_entry), - cleanup: None, - from_hir_call: false, - fn_span: DUMMY_SP, - }, - }), - is_cleanup: false, - }, - ); - - blocks.push(entry); - } - - *mir = tcx.arena.alloc(body); - - result -} - /////////////////////////////////////////////////////////////////////////// #[instrument(level = "debug", skip(cx))] @@ -268,9 +153,7 @@ pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( let llfn = cx.get_fn(instance); - let mut mir = cx.tcx().instance_mir(instance.def); - - let call_thread_local_shims = convert_tls_rvalues(cx.tcx(), &mut mir); + let mir = cx.tcx().instance_mir(instance.def); let fn_abi = cx.fn_abi_of_instance(instance, ty::List::empty()); debug!("fn_abi: {:?}", fn_abi); @@ -300,7 +183,6 @@ pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( llfn, fn_abi, cx, - call_thread_local_shims, personality_slot: None, cached_llbbs, unreachable_block: None, diff --git a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs index 41cd1c09a4e70..2bb94001d1f66 100644 --- a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs +++ b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs @@ -462,8 +462,19 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { mir::Rvalue::ThreadLocalRef(def_id) => { assert!(bx.cx().tcx().is_static(def_id)); - let static_ = bx.get_static(def_id); let layout = bx.layout_of(bx.cx().tcx().static_ptr_ty(def_id)); + let static_ = if !def_id.is_local() && !bx.cx().tcx().sess.target.dll_tls_export { + let instance = ty::Instance { + def: ty::InstanceDef::ThreadLocalShim(def_id), + substs: ty::InternalSubsts::empty(), + }; + let fn_ptr = bx.get_fn_addr(instance); + let fn_abi = bx.fn_abi_of_instance(instance, ty::List::empty()); + let fn_ty = bx.fn_decl_backend_type(&fn_abi); + bx.call(fn_ty, Some(fn_abi), fn_ptr, &[], None) + } else { + bx.get_static(def_id) + }; OperandRef { val: OperandValue::Immediate(static_), layout } } mir::Rvalue::Use(ref operand) => self.codegen_operand(bx, operand), From cf3683f0629fd78ea3c732362e17404399e18c98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?John=20K=C3=A5re=20Alsaker?= Date: Sun, 19 Feb 2023 01:04:54 +0100 Subject: [PATCH 6/8] Address comments --- compiler/rustc_middle/src/mir/tcx.rs | 12 +----- compiler/rustc_middle/src/ty/instance.rs | 5 ++- compiler/rustc_middle/src/ty/util.rs | 13 +++++++ compiler/rustc_mir_transform/src/shim.rs | 4 +- .../src/partitioning/default.rs | 37 +++++++++---------- compiler/rustc_ty_utils/src/abi.rs | 8 +--- 6 files changed, 37 insertions(+), 42 deletions(-) diff --git a/compiler/rustc_middle/src/mir/tcx.rs b/compiler/rustc_middle/src/mir/tcx.rs index aa9f170477bcb..e1ef6aebb8a3a 100644 --- a/compiler/rustc_middle/src/mir/tcx.rs +++ b/compiler/rustc_middle/src/mir/tcx.rs @@ -164,17 +164,7 @@ impl<'tcx> Rvalue<'tcx> { Rvalue::Repeat(ref operand, count) => { tcx.mk_array_with_const_len(operand.ty(local_decls, tcx), count) } - Rvalue::ThreadLocalRef(did) => { - let static_ty = tcx.type_of(did).subst_identity(); - if tcx.is_mutable_static(did) { - tcx.mk_mut_ptr(static_ty) - } else if tcx.is_foreign_item(did) { - tcx.mk_imm_ptr(static_ty) - } else { - // FIXME: These things don't *really* have 'static lifetime. - tcx.mk_imm_ref(tcx.lifetimes.re_static, static_ty) - } - } + Rvalue::ThreadLocalRef(did) => tcx.thread_local_ptr_ty(did), Rvalue::Ref(reg, bk, ref place) => { let place_ty = place.ty(local_decls, tcx).ty; tcx.mk_ref(reg, ty::TypeAndMut { ty: place_ty, mutbl: bk.to_mutbl_lossy() }) diff --git a/compiler/rustc_middle/src/ty/instance.rs b/compiler/rustc_middle/src/ty/instance.rs index d356cbfb29641..4e304c35324b3 100644 --- a/compiler/rustc_middle/src/ty/instance.rs +++ b/compiler/rustc_middle/src/ty/instance.rs @@ -82,8 +82,9 @@ pub enum InstanceDef<'tcx> { /// The `DefId` is the ID of the `call_once` method in `FnOnce`. ClosureOnceShim { call_once: DefId, track_caller: bool }, - /// Compiler-generated accessor for thread locals. This is used to export thread locals - /// from dylibs on platforms lacking native support. + /// Compiler-generated accessor for thread locals which returns a reference to the thread local + /// the `DefId` defines. This is used to export thread locals from dylibs on platforms lacking + /// native support. ThreadLocalShim(DefId), /// `core::ptr::drop_in_place::`. diff --git a/compiler/rustc_middle/src/ty/util.rs b/compiler/rustc_middle/src/ty/util.rs index 2ac3adda80b94..5718ec5bca1b2 100644 --- a/compiler/rustc_middle/src/ty/util.rs +++ b/compiler/rustc_middle/src/ty/util.rs @@ -599,6 +599,19 @@ impl<'tcx> TyCtxt<'tcx> { self.static_mutability(def_id) == Some(hir::Mutability::Mut) } + /// Returns the type a reference to the thread local takes in MIR. + pub fn thread_local_ptr_ty(self, def_id: DefId) -> Ty<'tcx> { + let static_ty = self.type_of(def_id).subst_identity(); + if self.is_mutable_static(def_id) { + self.mk_mut_ptr(static_ty) + } else if self.is_foreign_item(def_id) { + self.mk_imm_ptr(static_ty) + } else { + // FIXME: These things don't *really* have 'static lifetime. + self.mk_imm_ref(self.lifetimes.re_static, static_ty) + } + } + /// Get the type of the pointer to the static that we use in MIR. pub fn static_ptr_ty(self, def_id: DefId) -> Ty<'tcx> { // Make sure that any constants in the static's type are evaluated. diff --git a/compiler/rustc_mir_transform/src/shim.rs b/compiler/rustc_mir_transform/src/shim.rs index c312b8512caf8..b976278b4750b 100644 --- a/compiler/rustc_mir_transform/src/shim.rs +++ b/compiler/rustc_mir_transform/src/shim.rs @@ -341,12 +341,10 @@ fn build_thread_local_shim<'tcx>(tcx: TyCtxt<'tcx>, instance: ty::InstanceDef<'t is_cleanup: false, }); - let ret_ty = Rvalue::ThreadLocalRef(def_id).ty(&IndexVec::new(), tcx); - new_body( MirSource::from_instance(instance), blocks, - iter::once(LocalDecl::new(ret_ty, span)).collect(), + IndexVec::from_raw(vec![LocalDecl::new(tcx.thread_local_ptr_ty(def_id), span)]), 0, span, ) diff --git a/compiler/rustc_monomorphize/src/partitioning/default.rs b/compiler/rustc_monomorphize/src/partitioning/default.rs index d51b8a7552b82..b8d47f527afa9 100644 --- a/compiler/rustc_monomorphize/src/partitioning/default.rs +++ b/compiler/rustc_monomorphize/src/partitioning/default.rs @@ -392,6 +392,19 @@ fn mono_item_linkage_and_visibility<'tcx>( type CguNameCache = FxHashMap<(DefId, bool), Symbol>; +fn static_visibility<'tcx>( + tcx: TyCtxt<'tcx>, + can_be_internalized: &mut bool, + def_id: DefId, +) -> Visibility { + if tcx.is_reachable_non_generic(def_id) { + *can_be_internalized = false; + default_visibility(tcx, def_id, false) + } else { + Visibility::Hidden + } +} + fn mono_item_visibility<'tcx>( tcx: TyCtxt<'tcx>, mono_item: &MonoItem<'tcx>, @@ -403,21 +416,9 @@ fn mono_item_visibility<'tcx>( MonoItem::Fn(instance) => instance, // Misc handling for generics and such, but otherwise: - MonoItem::Static(def_id) => { - return if tcx.is_reachable_non_generic(*def_id) { - *can_be_internalized = false; - default_visibility(tcx, *def_id, false) - } else { - Visibility::Hidden - }; - } + MonoItem::Static(def_id) => return static_visibility(tcx, can_be_internalized, *def_id), MonoItem::GlobalAsm(item_id) => { - return if tcx.is_reachable_non_generic(item_id.owner_id) { - *can_be_internalized = false; - default_visibility(tcx, item_id.owner_id.to_def_id(), false) - } else { - Visibility::Hidden - }; + return static_visibility(tcx, can_be_internalized, item_id.owner_id.to_def_id()); } }; @@ -425,13 +426,9 @@ fn mono_item_visibility<'tcx>( InstanceDef::Item(def) => def.did, InstanceDef::DropGlue(def_id, Some(_)) => def_id, + // We match the visiblity of statics here InstanceDef::ThreadLocalShim(def_id) => { - return if tcx.is_reachable_non_generic(def_id) { - *can_be_internalized = false; - default_visibility(tcx, def_id, false) - } else { - Visibility::Hidden - }; + return static_visibility(tcx, can_be_internalized, def_id); } // These are all compiler glue and such, never exported, always hidden. diff --git a/compiler/rustc_ty_utils/src/abi.rs b/compiler/rustc_ty_utils/src/abi.rs index 62bd36a09609c..9166f426d8518 100644 --- a/compiler/rustc_ty_utils/src/abi.rs +++ b/compiler/rustc_ty_utils/src/abi.rs @@ -1,7 +1,5 @@ use rustc_hir as hir; use rustc_hir::lang_items::LangItem; -use rustc_index::vec::IndexVec; -use rustc_middle::mir::Rvalue; use rustc_middle::ty::layout::{ fn_can_unwind, FnAbiError, HasParamEnv, HasTyCtxt, LayoutCx, LayoutOf, TyAndLayout, }; @@ -32,11 +30,9 @@ fn fn_sig_for_fn_abi<'tcx>( param_env: ty::ParamEnv<'tcx>, ) -> ty::PolyFnSig<'tcx> { if let InstanceDef::ThreadLocalShim(..) = instance.def { - let ret_ty = Rvalue::ThreadLocalRef(instance.def_id()).ty(&IndexVec::new(), tcx); - return ty::Binder::dummy(tcx.mk_fn_sig( - [].iter(), - &ret_ty, + [], + tcx.thread_local_ptr_ty(instance.def_id()), false, hir::Unsafety::Normal, rustc_target::spec::abi::Abi::Unadjusted, From e61ac42702b19e025a889454fe6c280696abcbdc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?John=20K=C3=A5re=20Alsaker?= Date: Sun, 19 Feb 2023 01:18:19 +0100 Subject: [PATCH 7/8] Don't use thread local shims for foreign thread locals --- compiler/rustc_codegen_ssa/src/back/symbol_export.rs | 2 +- compiler/rustc_codegen_ssa/src/mir/rvalue.rs | 3 ++- compiler/rustc_middle/src/ty/util.rs | 9 +++++++++ compiler/rustc_monomorphize/src/collector.rs | 2 +- 4 files changed, 13 insertions(+), 3 deletions(-) diff --git a/compiler/rustc_codegen_ssa/src/back/symbol_export.rs b/compiler/rustc_codegen_ssa/src/back/symbol_export.rs index c465fd43f2962..96c5af8d14020 100644 --- a/compiler/rustc_codegen_ssa/src/back/symbol_export.rs +++ b/compiler/rustc_codegen_ssa/src/back/symbol_export.rs @@ -188,7 +188,7 @@ fn exported_symbols_provider_local( // Export TLS shims if !tcx.sess.target.dll_tls_export { symbols.extend(sorted.iter().filter_map(|(&def_id, &info)| { - tcx.is_thread_local_static(def_id).then(|| { + tcx.needs_thread_local_shim(def_id).then(|| { ( ExportedSymbol::ThreadLocalShim(def_id), SymbolExportInfo { diff --git a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs index 2bb94001d1f66..ee57a5a66d0d0 100644 --- a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs +++ b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs @@ -463,7 +463,8 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { mir::Rvalue::ThreadLocalRef(def_id) => { assert!(bx.cx().tcx().is_static(def_id)); let layout = bx.layout_of(bx.cx().tcx().static_ptr_ty(def_id)); - let static_ = if !def_id.is_local() && !bx.cx().tcx().sess.target.dll_tls_export { + let static_ = if !def_id.is_local() && bx.cx().tcx().needs_thread_local_shim(def_id) + { let instance = ty::Instance { def: ty::InstanceDef::ThreadLocalShim(def_id), substs: ty::InternalSubsts::empty(), diff --git a/compiler/rustc_middle/src/ty/util.rs b/compiler/rustc_middle/src/ty/util.rs index 5718ec5bca1b2..6f61101c1bd09 100644 --- a/compiler/rustc_middle/src/ty/util.rs +++ b/compiler/rustc_middle/src/ty/util.rs @@ -599,6 +599,15 @@ impl<'tcx> TyCtxt<'tcx> { self.static_mutability(def_id) == Some(hir::Mutability::Mut) } + /// Returns `true` if the item pointed to by `def_id` is a thread local which needs a + /// thread local shim generated. + #[inline] + pub fn needs_thread_local_shim(self, def_id: DefId) -> bool { + !self.sess.target.dll_tls_export + && self.is_thread_local_static(def_id) + && !self.is_foreign_item(def_id) + } + /// Returns the type a reference to the thread local takes in MIR. pub fn thread_local_ptr_ty(self, def_id: DefId) -> Ty<'tcx> { let static_ty = self.type_of(def_id).subst_identity(); diff --git a/compiler/rustc_monomorphize/src/collector.rs b/compiler/rustc_monomorphize/src/collector.rs index be9e06420fe7d..2e31be4828920 100644 --- a/compiler/rustc_monomorphize/src/collector.rs +++ b/compiler/rustc_monomorphize/src/collector.rs @@ -464,7 +464,7 @@ fn collect_items_rec<'tcx>( } } - if !tcx.sess.target.dll_tls_export && tcx.is_thread_local_static(def_id) { + if tcx.needs_thread_local_shim(def_id) { neighbors.push(respan( starting_point.span, MonoItem::Fn(Instance { From 6c9d7505e6413bbcbf696396f72506b060f62f26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?John=20K=C3=A5re=20Alsaker?= Date: Sun, 19 Feb 2023 15:57:04 +0100 Subject: [PATCH 8/8] Add cranelift support --- .../rustc_codegen_cranelift/src/constant.rs | 22 ++++++++++++++----- 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/compiler/rustc_codegen_cranelift/src/constant.rs b/compiler/rustc_codegen_cranelift/src/constant.rs index 49c4f1aaaefc6..67ff008ff504c 100644 --- a/compiler/rustc_codegen_cranelift/src/constant.rs +++ b/compiler/rustc_codegen_cranelift/src/constant.rs @@ -54,12 +54,22 @@ pub(crate) fn codegen_tls_ref<'tcx>( def_id: DefId, layout: TyAndLayout<'tcx>, ) -> CValue<'tcx> { - let data_id = data_id_for_static(fx.tcx, fx.module, def_id, false); - let local_data_id = fx.module.declare_data_in_func(data_id, &mut fx.bcx.func); - if fx.clif_comments.enabled() { - fx.add_comment(local_data_id, format!("tls {:?}", def_id)); - } - let tls_ptr = fx.bcx.ins().tls_value(fx.pointer_type, local_data_id); + let tls_ptr = if !def_id.is_local() && fx.tcx.needs_thread_local_shim(def_id) { + let instance = ty::Instance { + def: ty::InstanceDef::ThreadLocalShim(def_id), + substs: ty::InternalSubsts::empty(), + }; + let func_ref = fx.get_function_ref(instance); + let call = fx.bcx.ins().call(func_ref, &[]); + fx.bcx.func.dfg.first_result(call) + } else { + let data_id = data_id_for_static(fx.tcx, fx.module, def_id, false); + let local_data_id = fx.module.declare_data_in_func(data_id, &mut fx.bcx.func); + if fx.clif_comments.enabled() { + fx.add_comment(local_data_id, format!("tls {:?}", def_id)); + } + fx.bcx.ins().tls_value(fx.pointer_type, local_data_id) + }; CValue::by_val(tls_ptr, layout) }