Skip to content

Commit

Permalink
Auto merge of rust-lang#116505 - saethlin:infer-inline, r=<try>
Browse files Browse the repository at this point in the history
Automatically enable cross-crate inlining for small functions

This is a work-in-progress. For example I have not thought at all about the cost model and I am sure that the threshold is too high.

But I'm curious to know how this looks in perf. It certainly has some unique effects on codegen.
  • Loading branch information
bors committed Oct 7, 2023
2 parents 91db3bd + aac4020 commit 26bbeca
Show file tree
Hide file tree
Showing 23 changed files with 381 additions and 157 deletions.
1 change: 1 addition & 0 deletions compiler/rustc_interface/src/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -769,6 +769,7 @@ fn test_unstable_options_tracking_hash() {
);
tracked!(codegen_backend, Some("abc".to_string()));
tracked!(crate_attr, vec!["abc".to_string()]);
tracked!(cross_crate_inline_threshold, Some(200));
tracked!(debug_info_for_profiling, true);
tracked!(debug_macros, true);
tracked!(dep_info_omit_d_target, true);
Expand Down
9 changes: 9 additions & 0 deletions compiler/rustc_metadata/src/rmeta/decoder.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1273,6 +1273,15 @@ impl<'a, 'tcx> CrateMetadataRef<'a> {
self.root.tables.optimized_mir.get(self, id).is_some()
}

fn cross_crate_inlinable(self, tcx: TyCtxt<'tcx>, id: DefIndex) -> bool {
self.root
.tables
.cross_crate_inlinable
.get(self, id)
.map(|v| v.decode((self, tcx.sess)))
.unwrap_or(false)
}

fn get_fn_has_self_parameter(self, id: DefIndex, sess: &'a Session) -> bool {
self.root
.tables
Expand Down
1 change: 1 addition & 0 deletions compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs
Original file line number Diff line number Diff line change
Expand Up @@ -287,6 +287,7 @@ provide! { tcx, def_id, other, cdata,
item_attrs => { tcx.arena.alloc_from_iter(cdata.get_item_attrs(def_id.index, tcx.sess)) }
is_mir_available => { cdata.is_item_mir_available(def_id.index) }
is_ctfe_mir_available => { cdata.is_ctfe_mir_available(def_id.index) }
cross_crate_inlinable => { cdata.cross_crate_inlinable(tcx, def_id.index) }

dylib_dependency_formats => { cdata.get_dylib_dependency_formats(tcx) }
is_private_dep => {
Expand Down
3 changes: 2 additions & 1 deletion compiler/rustc_metadata/src/rmeta/encoder.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1046,7 +1046,7 @@ fn should_encode_mir(
|| (tcx.sess.opts.output_types.should_codegen()
&& reachable_set.contains(&def_id)
&& (generics.requires_monomorphization(tcx)
|| tcx.codegen_fn_attrs(def_id).requests_inline()));
|| tcx.cross_crate_inlinable(def_id)));
// The function has a `const` modifier or is in a `#[const_trait]`.
let is_const_fn = tcx.is_const_fn_raw(def_id.to_def_id())
|| tcx.is_const_default_method(def_id.to_def_id());
Expand Down Expand Up @@ -1612,6 +1612,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
debug!("EntryBuilder::encode_mir({:?})", def_id);
if encode_opt {
record!(self.tables.optimized_mir[def_id.to_def_id()] <- tcx.optimized_mir(def_id));
record!(self.tables.cross_crate_inlinable[def_id.to_def_id()] <- self.tcx.cross_crate_inlinable(def_id));
record!(self.tables.closure_saved_names_of_captured_variables[def_id.to_def_id()]
<- tcx.closure_saved_names_of_captured_variables(def_id));

Expand Down
1 change: 1 addition & 0 deletions compiler/rustc_metadata/src/rmeta/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -427,6 +427,7 @@ define_tables! {
object_lifetime_default: Table<DefIndex, LazyValue<ObjectLifetimeDefault>>,
optimized_mir: Table<DefIndex, LazyValue<mir::Body<'static>>>,
mir_for_ctfe: Table<DefIndex, LazyValue<mir::Body<'static>>>,
cross_crate_inlinable: Table<DefIndex, LazyValue<bool>>,
closure_saved_names_of_captured_variables: Table<DefIndex, LazyValue<IndexVec<FieldIdx, Symbol>>>,
mir_generator_witnesses: Table<DefIndex, LazyValue<mir::GeneratorLayout<'static>>>,
promoted_mir: Table<DefIndex, LazyValue<IndexVec<mir::Promoted, mir::Body<'static>>>>,
Expand Down
23 changes: 23 additions & 0 deletions compiler/rustc_metadata/src/rmeta/table.rs
Original file line number Diff line number Diff line change
Expand Up @@ -299,6 +299,29 @@ impl FixedSizeEncoding for bool {
}
}

impl FixedSizeEncoding for Option<bool> {
type ByteArray = [u8; 1];

#[inline]
fn from_bytes(b: &[u8; 1]) -> Self {
match b[0] {
0 => Some(false),
1 => Some(true),
_ => None,
}
}

#[inline]
fn write_to_bytes(self, b: &mut [u8; 1]) {
debug_assert!(!self.is_default());
b[0] = match self {
Some(false) => 0,
Some(true) => 1,
None => 2,
};
}
}

impl FixedSizeEncoding for UnusedGenericParams {
type ByteArray = [u8; 4];

Expand Down
5 changes: 5 additions & 0 deletions compiler/rustc_middle/src/query/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2202,6 +2202,11 @@ rustc_queries! {
query generics_require_sized_self(def_id: DefId) -> bool {
desc { "check whether the item has a `where Self: Sized` bound" }
}

query cross_crate_inlinable(def_id: DefId) -> bool {
desc { "whether the item should be made inlinable across crates" }
separate_provide_extern
}
}

rustc_query_append! { define_callbacks! }
Expand Down
9 changes: 4 additions & 5 deletions compiler/rustc_middle/src/ty/instance.rs
Original file line number Diff line number Diff line change
Expand Up @@ -245,16 +245,15 @@ impl<'tcx> InstanceDef<'tcx> {
// drops of `Option::None` before LTO. We also respect the intent of
// `#[inline]` on `Drop::drop` implementations.
return ty.ty_adt_def().map_or(true, |adt_def| {
adt_def.destructor(tcx).map_or_else(
|| adt_def.is_enum(),
|dtor| tcx.codegen_fn_attrs(dtor.did).requests_inline(),
)
adt_def
.destructor(tcx)
.map_or_else(|| adt_def.is_enum(), |dtor| tcx.cross_crate_inlinable(dtor.did))
});
}
if let ty::InstanceDef::ThreadLocalShim(..) = *self {
return false;
}
tcx.codegen_fn_attrs(self.def_id()).requests_inline()
tcx.cross_crate_inlinable(self.def_id())
}

pub fn requires_caller_location(&self, tcx: TyCtxt<'_>) -> bool {
Expand Down
101 changes: 101 additions & 0 deletions compiler/rustc_mir_transform/src/cross_crate_inline.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,101 @@
use rustc_attr::InlineAttr;
use rustc_hir::def_id::LocalDefId;
use rustc_middle::query::Providers;
use rustc_middle::ty::TyCtxt;
use rustc_session::config::OptLevel;

pub fn provide(providers: &mut Providers) {
providers.cross_crate_inlinable = cross_crate_inlinable;
}

fn cross_crate_inlinable(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool {
if tcx.sess.opts.incremental.is_some() {
return false;
}

match tcx.codegen_fn_attrs(def_id).inline {
InlineAttr::Never => return false,
InlineAttr::Hint | InlineAttr::Always => return true,
_ => {}
}

if matches!(tcx.sess.opts.optimize, OptLevel::No | OptLevel::Default) {
return false;
}

match tcx.hir().body_const_context(def_id) {
Some(rustc_hir::ConstContext::ConstFn) | None => {}
_ => return false,
}

if tcx.lang_items().iter().any(|(_, lang_def_id)| lang_def_id == def_id.into()) {
return false;
}

let mir = tcx.optimized_mir(def_id);
let mut checker = CostChecker { tcx, cost: 0, callee_body: mir };
checker.visit_body(mir);
checker.cost <= tcx.sess.opts.unstable_opts.cross_crate_inline_threshold.unwrap_or(100)
}

use rustc_middle::mir::visit::Visitor;
use rustc_middle::mir::*;

const INSTR_COST: usize = 5;
const CALL_PENALTY: usize = 25;
const LANDINGPAD_PENALTY: usize = 50;
const RESUME_PENALTY: usize = 45;

struct CostChecker<'b, 'tcx> {
tcx: TyCtxt<'tcx>,
cost: usize,
callee_body: &'b Body<'tcx>,
}

impl<'tcx> Visitor<'tcx> for CostChecker<'_, 'tcx> {
fn visit_statement(&mut self, statement: &Statement<'tcx>, _: Location) {
// Don't count StorageLive/StorageDead in the inlining cost.
match statement.kind {
StatementKind::StorageLive(_)
| StatementKind::StorageDead(_)
| StatementKind::Deinit(_)
| StatementKind::Nop => {}
_ => self.cost += INSTR_COST,
}
}

fn visit_terminator(&mut self, terminator: &Terminator<'tcx>, _: Location) {
let tcx = self.tcx;
match terminator.kind {
TerminatorKind::Drop { ref place, unwind, .. } => {
let ty = place.ty(self.callee_body, tcx).ty;
if !ty.is_trivially_pure_clone_copy() {
self.cost += CALL_PENALTY;
if let UnwindAction::Cleanup(_) = unwind {
self.cost += LANDINGPAD_PENALTY;
}
}
}
TerminatorKind::Call { unwind, .. } => {
self.cost += CALL_PENALTY;
if let UnwindAction::Cleanup(_) = unwind {
self.cost += LANDINGPAD_PENALTY;
}
}
TerminatorKind::Assert { unwind, .. } => {
self.cost += CALL_PENALTY;
if let UnwindAction::Cleanup(_) = unwind {
self.cost += LANDINGPAD_PENALTY;
}
}
TerminatorKind::UnwindResume => self.cost += RESUME_PENALTY,
TerminatorKind::InlineAsm { unwind, .. } => {
self.cost += INSTR_COST;
if let UnwindAction::Cleanup(_) = unwind {
self.cost += LANDINGPAD_PENALTY;
}
}
_ => self.cost += INSTR_COST,
}
}
}
16 changes: 12 additions & 4 deletions compiler/rustc_mir_transform/src/inline.rs
Original file line number Diff line number Diff line change
Expand Up @@ -170,7 +170,13 @@ impl<'tcx> Inliner<'tcx> {
callsite: &CallSite<'tcx>,
) -> Result<std::ops::Range<BasicBlock>, &'static str> {
let callee_attrs = self.tcx.codegen_fn_attrs(callsite.callee.def_id());
self.check_codegen_attributes(callsite, callee_attrs)?;
let cross_crate_inlinable = if callsite.callee.def_id().is_local() {
// Avoid a query cycle, cross_crate_inlinable is based on optimized_mir
callee_attrs.requests_inline()
} else {
self.tcx.cross_crate_inlinable(callsite.callee.def_id())
};
self.check_codegen_attributes(callsite, callee_attrs, cross_crate_inlinable)?;

let terminator = caller_body[callsite.block].terminator.as_ref().unwrap();
let TerminatorKind::Call { args, destination, .. } = &terminator.kind else { bug!() };
Expand All @@ -185,7 +191,7 @@ impl<'tcx> Inliner<'tcx> {

self.check_mir_is_available(caller_body, &callsite.callee)?;
let callee_body = try_instance_mir(self.tcx, callsite.callee.def)?;
self.check_mir_body(callsite, callee_body, callee_attrs)?;
self.check_mir_body(callsite, callee_body, callee_attrs, cross_crate_inlinable)?;

if !self.tcx.consider_optimizing(|| {
format!("Inline {:?} into {:?}", callsite.callee, caller_body.source)
Expand Down Expand Up @@ -401,6 +407,7 @@ impl<'tcx> Inliner<'tcx> {
&self,
callsite: &CallSite<'tcx>,
callee_attrs: &CodegenFnAttrs,
cross_crate_inlinable: bool,
) -> Result<(), &'static str> {
if let InlineAttr::Never = callee_attrs.inline {
return Err("never inline hint");
Expand All @@ -414,7 +421,7 @@ impl<'tcx> Inliner<'tcx> {
.non_erasable_generics(self.tcx, callsite.callee.def_id())
.next()
.is_some();
if !is_generic && !callee_attrs.requests_inline() {
if !is_generic && !cross_crate_inlinable {
return Err("not exported");
}

Expand Down Expand Up @@ -456,10 +463,11 @@ impl<'tcx> Inliner<'tcx> {
callsite: &CallSite<'tcx>,
callee_body: &Body<'tcx>,
callee_attrs: &CodegenFnAttrs,
cross_crate_inlinable: bool,
) -> Result<(), &'static str> {
let tcx = self.tcx;

let mut threshold = if callee_attrs.requests_inline() {
let mut threshold = if cross_crate_inlinable {
self.tcx.sess.opts.unstable_opts.inline_mir_hint_threshold.unwrap_or(100)
} else {
self.tcx.sess.opts.unstable_opts.inline_mir_threshold.unwrap_or(50)
Expand Down
2 changes: 2 additions & 0 deletions compiler/rustc_mir_transform/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@ mod const_prop;
mod const_prop_lint;
mod copy_prop;
mod coverage;
mod cross_crate_inline;
mod ctfe_limit;
mod dataflow_const_prop;
mod dead_store_elimination;
Expand Down Expand Up @@ -123,6 +124,7 @@ pub fn provide(providers: &mut Providers) {
coverage::query::provide(providers);
ffi_unwind_calls::provide(providers);
shim::provide(providers);
cross_crate_inline::provide(providers);
*providers = Providers {
mir_keys,
mir_const,
Expand Down
14 changes: 7 additions & 7 deletions compiler/rustc_passes/src/reachable.rs
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,8 @@ use rustc_target::spec::abi::Abi;
// Returns true if the given item must be inlined because it may be
// monomorphized or it was marked with `#[inline]`. This will only return
// true for functions.
fn item_might_be_inlined(tcx: TyCtxt<'_>, item: &hir::Item<'_>, attrs: &CodegenFnAttrs) -> bool {
if attrs.requests_inline() {
fn item_might_be_inlined(tcx: TyCtxt<'_>, item: &hir::Item<'_>, inlinable: bool) -> bool {
if inlinable {
return true;
}

Expand All @@ -41,9 +41,9 @@ fn method_might_be_inlined(
impl_item: &hir::ImplItem<'_>,
impl_src: LocalDefId,
) -> bool {
let codegen_fn_attrs = tcx.codegen_fn_attrs(impl_item.hir_id().owner.to_def_id());
let inlinable = tcx.cross_crate_inlinable(impl_item.hir_id().owner.to_def_id());
let generics = tcx.generics_of(impl_item.owner_id);
if codegen_fn_attrs.requests_inline() || generics.requires_monomorphization(tcx) {
if inlinable || generics.requires_monomorphization(tcx) {
return true;
}
if let hir::ImplItemKind::Fn(method_sig, _) = &impl_item.kind {
Expand All @@ -52,7 +52,7 @@ fn method_might_be_inlined(
}
}
match tcx.hir().find_by_def_id(impl_src) {
Some(Node::Item(item)) => item_might_be_inlined(tcx, &item, codegen_fn_attrs),
Some(Node::Item(item)) => item_might_be_inlined(tcx, &item, inlinable),
Some(..) | None => span_bug!(impl_item.span, "impl did is not an item"),
}
}
Expand Down Expand Up @@ -149,7 +149,7 @@ impl<'tcx> ReachableContext<'tcx> {
match self.tcx.hir().find_by_def_id(def_id) {
Some(Node::Item(item)) => match item.kind {
hir::ItemKind::Fn(..) => {
item_might_be_inlined(self.tcx, &item, self.tcx.codegen_fn_attrs(def_id))
item_might_be_inlined(self.tcx, &item, self.tcx.cross_crate_inlinable(def_id))
}
_ => false,
},
Expand Down Expand Up @@ -227,7 +227,7 @@ impl<'tcx> ReachableContext<'tcx> {
if item_might_be_inlined(
self.tcx,
&item,
self.tcx.codegen_fn_attrs(item.owner_id),
self.tcx.cross_crate_inlinable(item.owner_id),
) {
self.visit_nested_body(body);
}
Expand Down
2 changes: 2 additions & 0 deletions compiler/rustc_session/src/options.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1441,6 +1441,8 @@ options! {
"combine CGUs into a single one"),
crate_attr: Vec<String> = (Vec::new(), parse_string_push, [TRACKED],
"inject the given attribute in the crate"),
cross_crate_inline_threshold: Option<usize> = (None, parse_opt_number, [TRACKED],
"threshold to allow cross crate inlining of functions"),
debug_info_for_profiling: bool = (false, parse_bool, [TRACKED],
"emit discriminators and other data necessary for AutoFDO"),
debug_macros: bool = (false, parse_bool, [TRACKED],
Expand Down
1 change: 1 addition & 0 deletions library/alloc/src/raw_vec.rs
Original file line number Diff line number Diff line change
Expand Up @@ -530,6 +530,7 @@ fn alloc_guard(alloc_size: usize) -> Result<(), TryReserveError> {
// ensure that the code generation related to these panics is minimal as there's
// only one location which panics rather than a bunch throughout the module.
#[cfg(not(no_global_oom_handling))]
#[inline(never)]
fn capacity_overflow() -> ! {
panic!("capacity overflow");
}
2 changes: 1 addition & 1 deletion tests/mir-opt/enum_opt.cand.EnumSizeOpt.32bit.diff
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@
}
+ }
+
+ alloc15 (size: 8, align: 4) {
+ alloc14 (size: 8, align: 4) {
+ 02 00 00 00 05 20 00 00 │ ..... ..
}

2 changes: 1 addition & 1 deletion tests/mir-opt/enum_opt.cand.EnumSizeOpt.64bit.diff
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@
}
+ }
+
+ alloc15 (size: 16, align: 8) {
+ alloc14 (size: 16, align: 8) {
+ 02 00 00 00 00 00 00 00 05 20 00 00 00 00 00 00 │ ......... ......
}

2 changes: 1 addition & 1 deletion tests/mir-opt/enum_opt.unin.EnumSizeOpt.32bit.diff
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@
}
+ }
+
+ alloc14 (size: 8, align: 4) {
+ alloc15 (size: 8, align: 4) {
+ 05 20 00 00 01 00 00 00 │ . ......
}

2 changes: 1 addition & 1 deletion tests/mir-opt/enum_opt.unin.EnumSizeOpt.64bit.diff
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@
}
+ }
+
+ alloc14 (size: 16, align: 8) {
+ alloc15 (size: 16, align: 8) {
+ 05 20 00 00 00 00 00 00 01 00 00 00 00 00 00 00 │ . ..............
}

Loading

0 comments on commit 26bbeca

Please sign in to comment.