From ae92cb411baa234e44fad0b03a18edc63999433a Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Tue, 3 Sep 2024 13:53:40 -0500 Subject: [PATCH] Refactor `CallInfo` amongst Cranelift's backends (#9190) * Refactor backends to use the same `CallInfo` This commit refactors the various backends of cranelift, except for s390x, to use a shared definition of `CallInfo`. They were all already quite similar and the main change here is to push platform-specific pieces into the instructions outside of `CallInfo`. This is intended to make additions to `CallInfo` easier and require less refactoring in the future. Additionally this enables passing a `CallInfo` structure around instead of passing around all of its components which helps reduce the amount of arguments to various functions. * s390x: Use the same `CallInfo` as other backends This commit refactors s390x the same way as the previous commit to use the shared `CallInfo` that all other backends are using. This required more refactoring on the s390x side of things to notably extract a dedicated pseudo-instruction for `ElfTlsGetOffset` rather than bundling it within the `Call` instruction. * Review comments and test fixes * Fold `ExternalName` into `CallInfo` As predicted instruction sizes got larger when outlining this on some platforms so apply the same fix across all platforms by changing to `CallInfo` where the `T` will change depending on whether it's an indirect or direct call. * Update test expectations --- cranelift/codegen/src/isa/aarch64/abi.rs | 92 ++++----- cranelift/codegen/src/isa/aarch64/inst.isle | 15 +- .../codegen/src/isa/aarch64/inst/emit.rs | 43 ++--- .../src/isa/aarch64/inst/emit_tests.rs | 25 +-- cranelift/codegen/src/isa/aarch64/inst/mod.rs | 90 +++------ .../codegen/src/isa/aarch64/lower/isle.rs | 13 +- .../codegen/src/isa/pulley_shared/abi.rs | 43 +---- .../codegen/src/isa/pulley_shared/inst.isle | 7 +- .../src/isa/pulley_shared/inst/emit.rs | 4 +- .../codegen/src/isa/pulley_shared/inst/mod.rs | 27 +-- .../src/isa/pulley_shared/lower/isle.rs | 7 +- cranelift/codegen/src/isa/riscv64/abi.rs | 108 ++++------- cranelift/codegen/src/isa/riscv64/inst.isle | 15 +- .../codegen/src/isa/riscv64/inst/emit.rs | 34 ++-- cranelift/codegen/src/isa/riscv64/inst/mod.rs | 60 ++---- .../codegen/src/isa/riscv64/lower/isle.rs | 9 +- cranelift/codegen/src/isa/s390x/abi.rs | 11 +- cranelift/codegen/src/isa/s390x/inst.isle | 26 +-- cranelift/codegen/src/isa/s390x/inst/emit.rs | 36 ++-- .../codegen/src/isa/s390x/inst/emit_tests.rs | 25 +-- cranelift/codegen/src/isa/s390x/inst/mod.rs | 111 +++++------ cranelift/codegen/src/isa/s390x/lower.isle | 7 +- cranelift/codegen/src/isa/s390x/lower/isle.rs | 174 +++++++----------- cranelift/codegen/src/isa/x64/abi.rs | 110 +++++------ cranelift/codegen/src/isa/x64/inst.isle | 14 +- cranelift/codegen/src/isa/x64/inst/emit.rs | 68 +++---- .../codegen/src/isa/x64/inst/emit_tests.rs | 17 +- cranelift/codegen/src/isa/x64/inst/mod.rs | 110 ++++------- cranelift/codegen/src/isa/x64/lower/isle.rs | 10 +- cranelift/codegen/src/isa/x64/mod.rs | 2 +- cranelift/codegen/src/isa/x64/pcc.rs | 10 +- cranelift/codegen/src/lib.rs | 2 +- cranelift/codegen/src/machinst/abi.rs | 76 ++++++-- .../filetests/isa/pulley32/call.clif | 12 +- .../filetests/isa/pulley64/call.clif | 12 +- .../filetests/isa/s390x/tls_elf.clif | 2 +- winch/codegen/src/isa/x64/asm.rs | 16 +- 37 files changed, 577 insertions(+), 866 deletions(-) diff --git a/cranelift/codegen/src/isa/aarch64/abi.rs b/cranelift/codegen/src/isa/aarch64/abi.rs index 59a493ea2b41..33cb9dd7b127 100644 --- a/cranelift/codegen/src/isa/aarch64/abi.rs +++ b/cranelift/codegen/src/isa/aarch64/abi.rs @@ -1014,58 +1014,26 @@ impl ABIMachineSpec for AArch64MachineDeps { insts } - fn gen_call( - dest: &CallDest, - uses: CallArgList, - defs: CallRetList, - clobbers: PRegSet, - tmp: Writable, - callee_conv: isa::CallConv, - caller_conv: isa::CallConv, - callee_pop_size: u32, - ) -> SmallVec<[Inst; 2]> { + fn gen_call(dest: &CallDest, tmp: Writable, info: CallInfo<()>) -> SmallVec<[Inst; 2]> { let mut insts = SmallVec::new(); match &dest { - &CallDest::ExtName(ref name, RelocDistance::Near) => insts.push(Inst::Call { - info: Box::new(CallInfo { - dest: name.clone(), - uses, - defs, - clobbers, - caller_callconv: caller_conv, - callee_callconv: callee_conv, - callee_pop_size, - }), - }), + &CallDest::ExtName(ref name, RelocDistance::Near) => { + let info = Box::new(info.map(|()| name.clone())); + insts.push(Inst::Call { info }); + } &CallDest::ExtName(ref name, RelocDistance::Far) => { insts.push(Inst::LoadExtName { rd: tmp, name: Box::new(name.clone()), offset: 0, }); - insts.push(Inst::CallInd { - info: Box::new(CallIndInfo { - rn: tmp.to_reg(), - uses, - defs, - clobbers, - caller_callconv: caller_conv, - callee_callconv: callee_conv, - callee_pop_size, - }), - }); + let info = Box::new(info.map(|()| tmp.to_reg())); + insts.push(Inst::CallInd { info }); + } + &CallDest::Reg(reg) => { + let info = Box::new(info.map(|()| *reg)); + insts.push(Inst::CallInd { info }); } - &CallDest::Reg(reg) => insts.push(Inst::CallInd { - info: Box::new(CallIndInfo { - rn: *reg, - uses, - defs, - clobbers, - caller_callconv: caller_conv, - callee_callconv: callee_conv, - callee_pop_size, - }), - }), } insts @@ -1103,8 +1071,8 @@ impl ABIMachineSpec for AArch64MachineDeps { ], defs: smallvec![], clobbers: Self::get_regs_clobbered_by_call(call_conv), - caller_callconv: call_conv, - callee_callconv: call_conv, + caller_conv: call_conv, + callee_conv: call_conv, callee_pop_size: 0, }), }); @@ -1295,16 +1263,17 @@ impl AArch64CallSite { let dest = self.dest().clone(); let uses = self.take_uses(); - let info = Box::new(ReturnCallInfo { - uses, - new_stack_arg_size, - key: select_api_key(isa_flags, isa::CallConv::Tail, true), - }); + let key = select_api_key(isa_flags, isa::CallConv::Tail, true); match dest { CallDest::ExtName(callee, RelocDistance::Near) => { - let callee = Box::new(callee); - ctx.emit(Inst::ReturnCall { callee, info }); + let info = Box::new(ReturnCallInfo { + dest: callee, + uses, + key, + new_stack_arg_size, + }); + ctx.emit(Inst::ReturnCall { info }); } CallDest::ExtName(name, RelocDistance::Far) => { let callee = ctx.alloc_tmp(types::I64).only_reg().unwrap(); @@ -1313,12 +1282,23 @@ impl AArch64CallSite { name: Box::new(name), offset: 0, }); - ctx.emit(Inst::ReturnCallInd { - callee: callee.to_reg(), - info, + let info = Box::new(ReturnCallInfo { + dest: callee.to_reg(), + uses, + key, + new_stack_arg_size, + }); + ctx.emit(Inst::ReturnCallInd { info }); + } + CallDest::Reg(callee) => { + let info = Box::new(ReturnCallInfo { + dest: callee, + uses, + key, + new_stack_arg_size, }); + ctx.emit(Inst::ReturnCallInd { info }); } - CallDest::Reg(callee) => ctx.emit(Inst::ReturnCallInd { callee, info }), } } } diff --git a/cranelift/codegen/src/isa/aarch64/inst.isle b/cranelift/codegen/src/isa/aarch64/inst.isle index e55586a95f78..c19710376a3f 100644 --- a/cranelift/codegen/src/isa/aarch64/inst.isle +++ b/cranelift/codegen/src/isa/aarch64/inst.isle @@ -806,22 +806,16 @@ ;; of type `Reloc::Arm64Call`); if the destination distance is not `RelocDistance::Near`, the ;; code should use a `LoadExtName` / `CallInd` sequence instead, allowing an arbitrary 64-bit ;; target. - (Call - (info BoxCallInfo)) + (Call (info BoxCallInfo)) ;; A machine indirect-call instruction. - (CallInd - (info BoxCallIndInfo)) + (CallInd (info BoxCallIndInfo)) ;; A return-call macro instruction. - (ReturnCall - (callee BoxExternalName) - (info BoxReturnCallInfo)) + (ReturnCall (info BoxReturnCallInfo)) ;; An indirect return-call macro instruction. - (ReturnCallInd - (callee Reg) - (info BoxReturnCallInfo)) + (ReturnCallInd (info BoxReturnCallIndInfo)) ;; A pseudo-instruction that captures register arguments in vregs. (Args @@ -1073,6 +1067,7 @@ (type BoxCallInfo (primitive BoxCallInfo)) (type BoxCallIndInfo (primitive BoxCallIndInfo)) (type BoxReturnCallInfo (primitive BoxReturnCallInfo)) +(type BoxReturnCallIndInfo (primitive BoxReturnCallIndInfo)) (type CondBrKind (primitive CondBrKind)) (type BranchTarget (primitive BranchTarget)) (type BoxJTSequenceInfo (primitive BoxJTSequenceInfo)) diff --git a/cranelift/codegen/src/isa/aarch64/inst/emit.rs b/cranelift/codegen/src/isa/aarch64/inst/emit.rs index 8f76dd56a9f4..f96f22100bb0 100644 --- a/cranelift/codegen/src/isa/aarch64/inst/emit.rs +++ b/cranelift/codegen/src/isa/aarch64/inst/emit.rs @@ -2943,8 +2943,9 @@ impl MachInstEmit for Inst { } &Inst::CallInd { ref info } => { let user_stack_map = state.take_stack_map(); - let rn = info.rn; - sink.put4(0b1101011_0001_11111_000000_00000_00000 | (machreg_to_gpr(rn) << 5)); + sink.put4( + 0b1101011_0001_11111_000000_00000_00000 | (machreg_to_gpr(info.dest) << 5), + ); if let Some(s) = user_stack_map { let offset = sink.cur_offset(); sink.push_user_stack_map(state, offset, s); @@ -2959,16 +2960,13 @@ impl MachInstEmit for Inst { } } } - &Inst::ReturnCall { - ref callee, - ref info, - } => { + &Inst::ReturnCall { ref info } => { emit_return_call_common_sequence(sink, emit_info, state, info); // Note: this is not `Inst::Jump { .. }.emit(..)` because we // have different metadata in this case: we don't have a label // for the target, but rather a function relocation. - sink.add_reloc(Reloc::Arm64Call, &**callee, 0); + sink.add_reloc(Reloc::Arm64Call, &info.dest, 0); sink.put4(enc_jump26(0b000101, 0)); sink.add_call_site(); @@ -2977,11 +2975,11 @@ impl MachInstEmit for Inst { // in this case. start_off = sink.cur_offset(); } - &Inst::ReturnCallInd { callee, ref info } => { + &Inst::ReturnCallInd { ref info } => { emit_return_call_common_sequence(sink, emit_info, state, info); Inst::IndirectBr { - rn: callee, + rn: info.dest, targets: vec![], } .emit(sink, emit_info, state); @@ -3372,15 +3370,7 @@ impl MachInstEmit for Inst { // blr tmp sink.add_reloc(Reloc::Aarch64TlsDescCall, &**symbol, 0); Inst::CallInd { - info: crate::isa::Box::new(CallIndInfo { - rn: tmp.to_reg(), - uses: smallvec![], - defs: smallvec![], - clobbers: PRegSet::empty(), - caller_callconv: CallConv::SystemV, - callee_callconv: CallConv::SystemV, - callee_pop_size: 0, - }), + info: crate::isa::Box::new(CallInfo::empty(tmp.to_reg(), CallConv::SystemV)), } .emit(sink, emit_info, state); @@ -3432,15 +3422,10 @@ impl MachInstEmit for Inst { // call function pointer in temp register Inst::CallInd { - info: crate::isa::Box::new(CallIndInfo { - rn: rtmp.to_reg(), - uses: smallvec![], - defs: smallvec![], - clobbers: PRegSet::empty(), - caller_callconv: CallConv::AppleAarch64, - callee_callconv: CallConv::AppleAarch64, - callee_pop_size: 0, - }), + info: crate::isa::Box::new(CallInfo::empty( + rtmp.to_reg(), + CallConv::AppleAarch64, + )), } .emit(sink, emit_info, state); } @@ -3533,11 +3518,11 @@ impl MachInstEmit for Inst { } } -fn emit_return_call_common_sequence( +fn emit_return_call_common_sequence( sink: &mut MachBuffer, emit_info: &EmitInfo, state: &mut EmitState, - info: &ReturnCallInfo, + info: &ReturnCallInfo, ) { for inst in AArch64MachineDeps::gen_clobber_restore(CallConv::Tail, &emit_info.0, state.frame_layout()) diff --git a/cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs b/cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs index 0b50795ec39a..5cc26eab0c3c 100644 --- a/cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs +++ b/cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs @@ -1,5 +1,5 @@ use crate::ir::types::*; -use crate::ir::TrapCode; +use crate::ir::{ExternalName, TrapCode}; use crate::isa::aarch64::inst::*; use alloc::boxed::Box; @@ -6056,15 +6056,10 @@ fn test_aarch64_binemit() { insns.push(( Inst::Call { - info: Box::new(CallInfo { - dest: ExternalName::testcase("test0"), - uses: smallvec![], - defs: smallvec![], - clobbers: PRegSet::empty(), - caller_callconv: CallConv::SystemV, - callee_callconv: CallConv::SystemV, - callee_pop_size: 0, - }), + info: Box::new(CallInfo::empty( + ExternalName::testcase("test0"), + CallConv::SystemV, + )), }, "00000094", "bl 0", @@ -6072,15 +6067,7 @@ fn test_aarch64_binemit() { insns.push(( Inst::CallInd { - info: Box::new(CallIndInfo { - rn: xreg(10), - uses: smallvec![], - defs: smallvec![], - clobbers: PRegSet::empty(), - caller_callconv: CallConv::SystemV, - callee_callconv: CallConv::SystemV, - callee_pop_size: 0, - }), + info: Box::new(CallInfo::empty(xreg(10), CallConv::SystemV)), }, "40013FD6", "blr x10", diff --git a/cranelift/codegen/src/isa/aarch64/inst/mod.rs b/cranelift/codegen/src/isa/aarch64/inst/mod.rs index eff651e57694..8287eb6b0c35 100644 --- a/cranelift/codegen/src/isa/aarch64/inst/mod.rs +++ b/cranelift/codegen/src/isa/aarch64/inst/mod.rs @@ -2,7 +2,7 @@ use crate::binemit::{Addend, CodeOffset, Reloc}; use crate::ir::types::{F128, F16, F32, F64, I128, I16, I32, I64, I8, I8X16}; -use crate::ir::{types, ExternalName, MemFlags, Type}; +use crate::ir::{types, MemFlags, Type}; use crate::isa::{CallConv, FunctionAlignment}; use crate::machinst::*; use crate::{settings, CodegenError, CodegenResult}; @@ -10,7 +10,6 @@ use crate::{settings, CodegenError, CodegenResult}; use crate::machinst::{PrettyPrint, Reg, RegClass, Writable}; use alloc::vec::Vec; -use regalloc2::PRegSet; use smallvec::{smallvec, SmallVec}; use std::fmt::Write; use std::string::{String, ToString}; @@ -74,54 +73,12 @@ impl BitOp { } } -/// Additional information for (direct) Call instructions, left out of line to lower the size of -/// the Inst enum. -#[derive(Clone, Debug)] -pub struct CallInfo { - /// Call destination. - pub dest: ExternalName, - /// Arguments to the call instruction. - pub uses: CallArgList, - /// Return values from the call instruction. - pub defs: CallRetList, - /// Clobbers register set. - pub clobbers: PRegSet, - /// Caller calling convention. - pub caller_callconv: CallConv, - /// Callee calling convention. - pub callee_callconv: CallConv, - /// The number of bytes that the callee will pop from the stack for the - /// caller, if any. (Used for popping stack arguments with the `tail` - /// calling convention.) - pub callee_pop_size: u32, -} - -/// Additional information for CallInd instructions, left out of line to lower the size of the Inst -/// enum. -#[derive(Clone, Debug)] -pub struct CallIndInfo { - /// Function pointer for indirect call. - pub rn: Reg, - /// Arguments to the call instruction. - pub uses: SmallVec<[CallArgPair; 8]>, - /// Return values from the call instruction. - pub defs: SmallVec<[CallRetPair; 8]>, - /// Clobbers register set. - pub clobbers: PRegSet, - /// Caller calling convention. - pub caller_callconv: CallConv, - /// Callee calling convention. - pub callee_callconv: CallConv, - /// The number of bytes that the callee will pop from the stack for the - /// caller, if any. (Used for popping stack arguments with the `tail` - /// calling convention.) - pub callee_pop_size: u32, -} - /// Additional information for `return_call[_ind]` instructions, left out of /// line to lower the size of the `Inst` enum. #[derive(Clone, Debug)] -pub struct ReturnCallInfo { +pub struct ReturnCallInfo { + /// Where this call is going to + pub dest: T, /// Arguments to the call instruction. pub uses: CallArgList, /// The size of the new stack frame's stack arguments. This is necessary @@ -895,8 +852,10 @@ fn aarch64_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) { collector.reg_clobbers(info.clobbers); } Inst::CallInd { info, .. } => { - let CallIndInfo { rn, uses, defs, .. } = &mut **info; - collector.reg_use(rn); + let CallInfo { + dest, uses, defs, .. + } = &mut **info; + collector.reg_use(dest); for CallArgPair { vreg, preg } in uses { collector.reg_fixed_use(vreg, *preg); } @@ -905,17 +864,17 @@ fn aarch64_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) { } collector.reg_clobbers(info.clobbers); } - Inst::ReturnCall { info, callee: _ } => { + Inst::ReturnCall { info } => { for CallArgPair { vreg, preg } in &mut info.uses { collector.reg_fixed_use(vreg, *preg); } } - Inst::ReturnCallInd { info, callee } => { + Inst::ReturnCallInd { info } => { // TODO(https://github.com/bytecodealliance/regalloc2/issues/145): // This shouldn't be a fixed register constraint, but it's not clear how to pick a // register that won't be clobbered by the callee-save restore code emitted with a // return_call_indirect. - collector.reg_fixed_use(callee, xreg(1)); + collector.reg_fixed_use(&mut info.dest, xreg(1)); for CallArgPair { vreg, preg } in &mut info.uses { collector.reg_fixed_use(vreg, *preg); } @@ -1019,10 +978,10 @@ impl MachInst for Inst { } fn is_included_in_clobbers(&self) -> bool { - let (caller_callconv, callee_callconv) = match self { + let (caller, callee) = match self { Inst::Args { .. } => return false, - Inst::Call { info } => (info.caller_callconv, info.callee_callconv), - Inst::CallInd { info } => (info.caller_callconv, info.callee_callconv), + Inst::Call { info } => (info.caller_conv, info.callee_conv), + Inst::CallInd { info } => (info.caller_conv, info.callee_conv), _ => return true, }; @@ -1037,8 +996,8 @@ impl MachInst for Inst { // // See the note in [crate::isa::aarch64::abi::is_caller_save_reg] for // more information on this ABI-implementation hack. - let caller_clobbers = AArch64MachineDeps::get_regs_clobbered_by_call(caller_callconv); - let callee_clobbers = AArch64MachineDeps::get_regs_clobbered_by_call(callee_callconv); + let caller_clobbers = AArch64MachineDeps::get_regs_clobbered_by_call(caller); + let callee_clobbers = AArch64MachineDeps::get_regs_clobbered_by_call(callee); let mut all_clobbers = caller_clobbers; all_clobbers.union_from(callee_clobbers); @@ -2608,17 +2567,14 @@ impl Inst { } } &Inst::Call { .. } => format!("bl 0"), - &Inst::CallInd { ref info, .. } => { - let rn = pretty_print_reg(info.rn); + &Inst::CallInd { ref info } => { + let rn = pretty_print_reg(info.dest); format!("blr {rn}") } - &Inst::ReturnCall { - ref callee, - ref info, - } => { + &Inst::ReturnCall { ref info } => { let mut s = format!( - "return_call {callee:?} new_stack_arg_size:{}", - info.new_stack_arg_size + "return_call {:?} new_stack_arg_size:{}", + info.dest, info.new_stack_arg_size ); for ret in &info.uses { let preg = pretty_print_reg(ret.preg); @@ -2627,8 +2583,8 @@ impl Inst { } s } - &Inst::ReturnCallInd { callee, ref info } => { - let callee = pretty_print_reg(callee); + &Inst::ReturnCallInd { ref info } => { + let callee = pretty_print_reg(info.dest); let mut s = format!( "return_call_ind {callee} new_stack_arg_size:{}", info.new_stack_arg_size diff --git a/cranelift/codegen/src/isa/aarch64/lower/isle.rs b/cranelift/codegen/src/isa/aarch64/lower/isle.rs index 29a3f1c68a4b..44e3b4d0d594 100644 --- a/cranelift/codegen/src/isa/aarch64/lower/isle.rs +++ b/cranelift/codegen/src/isa/aarch64/lower/isle.rs @@ -7,9 +7,9 @@ use generated_code::Context; // Types that the generated ISLE code uses via `use super::*`. use super::{ fp_reg, lower_condcode, lower_fp_condcode, stack_reg, writable_link_reg, writable_zero_reg, - zero_reg, ASIMDFPModImm, ASIMDMovModImm, BranchTarget, CallIndInfo, CallInfo, Cond, CondBrKind, - ExtendOp, FPUOpRI, FPUOpRIMod, FloatCC, Imm12, ImmLogic, ImmShift, Inst as MInst, IntCC, - MachLabel, MemLabel, MoveWideConst, MoveWideOp, Opcode, OperandSize, Reg, SImm9, ScalarSize, + zero_reg, ASIMDFPModImm, ASIMDMovModImm, BranchTarget, CallInfo, Cond, CondBrKind, ExtendOp, + FPUOpRI, FPUOpRIMod, FloatCC, Imm12, ImmLogic, ImmShift, Inst as MInst, IntCC, MachLabel, + MemLabel, MoveWideConst, MoveWideOp, Opcode, OperandSize, Reg, SImm9, ScalarSize, ShiftOpAndAmt, UImm12Scaled, UImm5, VecMisc2, VectorSize, NZCV, }; use crate::ir::{condcodes, ArgumentExtension}; @@ -34,9 +34,10 @@ use regalloc2::PReg; use std::boxed::Box; use std::vec::Vec; -type BoxCallInfo = Box; -type BoxCallIndInfo = Box; -type BoxReturnCallInfo = Box; +type BoxCallInfo = Box>; +type BoxCallIndInfo = Box>; +type BoxReturnCallInfo = Box>; +type BoxReturnCallIndInfo = Box>; type VecMachLabel = Vec; type BoxExternalName = Box; type VecArgPair = Vec; diff --git a/cranelift/codegen/src/isa/pulley_shared/abi.rs b/cranelift/codegen/src/isa/pulley_shared/abi.rs index a88fe2ac81c5..7f08a80a0748 100644 --- a/cranelift/codegen/src/isa/pulley_shared/abi.rs +++ b/cranelift/codegen/src/isa/pulley_shared/abi.rs @@ -522,26 +522,11 @@ where insts } - fn gen_call( - dest: &CallDest, - uses: CallArgList, - defs: CallRetList, - clobbers: PRegSet, - tmp: Writable, - callee_conv: isa::CallConv, - caller_conv: isa::CallConv, - callee_pop_size: u32, - ) -> SmallVec<[Self::I; 2]> { - if callee_conv == isa::CallConv::Tail || callee_conv == isa::CallConv::Fast { + fn gen_call(dest: &CallDest, tmp: Writable, info: CallInfo<()>) -> SmallVec<[Self::I; 2]> { + if info.callee_conv == isa::CallConv::Tail || info.callee_conv == isa::CallConv::Fast { match &dest { &CallDest::ExtName(ref name, RelocDistance::Near) => smallvec![Inst::Call { - callee: Box::new(name.clone()), - info: Box::new(CallInfo { - uses, - defs, - clobbers, - callee_pop_size, - }), + info: Box::new(info.map(|()| name.clone())) } .into()], &CallDest::ExtName(ref name, RelocDistance::Far) => smallvec![ @@ -552,29 +537,21 @@ where } .into(), Inst::IndirectCall { - callee: XReg::new(tmp.to_reg()).unwrap(), - info: Box::new(CallInfo { - uses, - defs, - clobbers, - callee_pop_size, - }), + info: Box::new(info.map(|()| XReg::new(tmp.to_reg()).unwrap())) } .into(), ], &CallDest::Reg(reg) => smallvec![Inst::IndirectCall { - callee: XReg::new(*reg).unwrap(), - info: Box::new(CallInfo { - uses, - defs, - clobbers, - callee_pop_size, - }), + info: Box::new(info.map(|()| XReg::new(*reg).unwrap())) } .into()], } } else { - todo!("host calls? callee_conv = {callee_conv:?}; caller_conv = {caller_conv:?}") + todo!( + "host calls? callee_conv = {:?}; caller_conv = {:?}", + info.callee_conv, + info.caller_conv, + ) } } diff --git a/cranelift/codegen/src/isa/pulley_shared/inst.isle b/cranelift/codegen/src/isa/pulley_shared/inst.isle index 6301353a70e0..e5f3d7499396 100644 --- a/cranelift/codegen/src/isa/pulley_shared/inst.isle +++ b/cranelift/codegen/src/isa/pulley_shared/inst.isle @@ -35,12 +35,10 @@ (offset i64)) ;; A direct call to a known callee. - (Call (callee BoxExternalName) - (info BoxCallInfo)) + (Call (info BoxCallInfo)) ;; An indirect call to an unknown callee. - (IndirectCall (callee XReg) - (info BoxCallInfo)) + (IndirectCall (info BoxCallIndInfo)) ;; Unconditional jumps. (Jump (label MachLabel)) @@ -103,6 +101,7 @@ ) (type BoxCallInfo (primitive BoxCallInfo)) +(type BoxCallIndInfo (primitive BoxCallIndInfo)) ;;;; Address Modes ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; diff --git a/cranelift/codegen/src/isa/pulley_shared/inst/emit.rs b/cranelift/codegen/src/isa/pulley_shared/inst/emit.rs index 7470bcad67e3..66a458ff7d7f 100644 --- a/cranelift/codegen/src/isa/pulley_shared/inst/emit.rs +++ b/cranelift/codegen/src/isa/pulley_shared/inst/emit.rs @@ -147,12 +147,12 @@ fn pulley_emit

( Inst::LoadExtName { .. } => todo!(), - Inst::Call { callee, info } => { + Inst::Call { info } => { sink.put1(pulley_interpreter::Opcode::Call as u8); sink.add_reloc( // TODO: is it actually okay to reuse this reloc here? Reloc::X86CallPCRel4, - &**callee, + &info.dest, // This addend adjusts for the difference between the start of // the instruction and the beginning of the immediate field. -1, diff --git a/cranelift/codegen/src/isa/pulley_shared/inst/mod.rs b/cranelift/codegen/src/isa/pulley_shared/inst/mod.rs index 8051ae9f3fc0..5665a409d91f 100644 --- a/cranelift/codegen/src/isa/pulley_shared/inst/mod.rs +++ b/cranelift/codegen/src/isa/pulley_shared/inst/mod.rs @@ -10,7 +10,7 @@ use crate::isa::FunctionAlignment; use crate::{machinst::*, trace}; use crate::{settings, CodegenError, CodegenResult}; use alloc::string::{String, ToString}; -use regalloc2::{PRegSet, RegClass}; +use regalloc2::RegClass; use smallvec::SmallVec; pub mod regs; @@ -27,17 +27,6 @@ pub use crate::isa::pulley_shared::lower::isle::generated_code::MInst as Inst; use super::PulleyTargetKind; -/// Additional information for direct and indirect call instructions. -/// -/// Left out of line to lower the size of the `Inst` enum. -#[derive(Clone, Debug)] -pub struct CallInfo { - pub uses: CallArgList, - pub defs: CallRetList, - pub clobbers: PRegSet, - pub callee_pop_size: u32, -} - impl Inst { /// Generic constructor for a load (zero-extending where appropriate). pub fn gen_load(dst: Writable, mem: Amode, ty: Type, flags: MemFlags) -> Inst { @@ -91,7 +80,7 @@ fn pulley_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) { collector.reg_def(dst); } - Inst::Call { callee: _, info } => { + Inst::Call { info } => { let CallInfo { uses, defs, .. } = &mut **info; for CallArgPair { vreg, preg } in uses { collector.reg_fixed_use(vreg, *preg); @@ -101,8 +90,8 @@ fn pulley_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) { } collector.reg_clobbers(info.clobbers); } - Inst::IndirectCall { callee, info } => { - collector.reg_use(callee); + Inst::IndirectCall { info } => { + collector.reg_use(&mut info.dest); let CallInfo { uses, defs, .. } = &mut **info; for CallArgPair { vreg, preg } in uses { collector.reg_fixed_use(vreg, *preg); @@ -548,12 +537,12 @@ impl Inst { format!("{dst} = load_ext_name {name:?}, {offset}") } - Inst::Call { callee, info } => { - format!("call {callee:?}, {info:?}") + Inst::Call { info } => { + format!("call {info:?}") } - Inst::IndirectCall { callee, info } => { - let callee = format_reg(**callee); + Inst::IndirectCall { info } => { + let callee = format_reg(*info.dest); format!("indirect_call {callee}, {info:?}") } diff --git a/cranelift/codegen/src/isa/pulley_shared/lower/isle.rs b/cranelift/codegen/src/isa/pulley_shared/lower/isle.rs index 51e17d29ce67..20281fd81a1a 100644 --- a/cranelift/codegen/src/isa/pulley_shared/lower/isle.rs +++ b/cranelift/codegen/src/isa/pulley_shared/lower/isle.rs @@ -9,20 +9,21 @@ use inst::InstAndKind; use crate::ir::{condcodes::*, immediates::*, types::*, *}; use crate::isa::pulley_shared::{ abi::*, - inst::{CallInfo, FReg, VReg, WritableFReg, WritableVReg, WritableXReg, XReg}, + inst::{FReg, VReg, WritableFReg, WritableVReg, WritableXReg, XReg}, *, }; use crate::machinst::{ abi::{ArgPair, RetPair, StackAMode}, isle::*, - IsTailCall, MachInst, Reg, VCodeConstant, VCodeConstantData, + CallInfo, IsTailCall, MachInst, Reg, VCodeConstant, VCodeConstantData, }; use alloc::boxed::Box; use regalloc2::PReg; type Unit = (); type VecArgPair = Vec; type VecRetPair = Vec; -type BoxCallInfo = Box; +type BoxCallInfo = Box>; +type BoxCallIndInfo = Box>; type BoxExternalName = Box; pub(crate) struct PulleyIsleContext<'a, 'b, I, B> diff --git a/cranelift/codegen/src/isa/riscv64/abi.rs b/cranelift/codegen/src/isa/riscv64/abi.rs index 2a6b23f3af38..b90fa643c3ca 100644 --- a/cranelift/codegen/src/isa/riscv64/abi.rs +++ b/cranelift/codegen/src/isa/riscv64/abi.rs @@ -424,19 +424,16 @@ impl ABIMachineSpec for Riscv64MachineDeps { fn gen_probestack(insts: &mut SmallInstVec, frame_size: u32) { insts.extend(Inst::load_constant_u32(writable_a0(), frame_size as u64)); + let mut info = CallInfo::empty( + ExternalName::LibCall(LibCall::Probestack), + CallConv::SystemV, + ); + info.uses.push(CallArgPair { + vreg: a0(), + preg: a0(), + }); insts.push(Inst::Call { - info: Box::new(CallInfo { - dest: ExternalName::LibCall(LibCall::Probestack), - uses: smallvec![CallArgPair { - vreg: a0(), - preg: a0(), - }], - defs: smallvec![], - clobbers: PRegSet::empty(), - callee_callconv: CallConv::SystemV, - caller_callconv: CallConv::SystemV, - callee_pop_size: 0, - }), + info: Box::new(info), }); } @@ -567,58 +564,26 @@ impl ABIMachineSpec for Riscv64MachineDeps { insts } - fn gen_call( - dest: &CallDest, - uses: CallArgList, - defs: CallRetList, - clobbers: PRegSet, - tmp: Writable, - callee_conv: isa::CallConv, - caller_conv: isa::CallConv, - callee_pop_size: u32, - ) -> SmallVec<[Self::I; 2]> { + fn gen_call(dest: &CallDest, tmp: Writable, info: CallInfo<()>) -> SmallVec<[Self::I; 2]> { let mut insts = SmallVec::new(); match &dest { - &CallDest::ExtName(ref name, RelocDistance::Near) => insts.push(Inst::Call { - info: Box::new(CallInfo { - dest: name.clone(), - uses, - defs, - clobbers, - caller_callconv: caller_conv, - callee_callconv: callee_conv, - callee_pop_size, - }), - }), + &CallDest::ExtName(ref name, RelocDistance::Near) => { + let info = Box::new(info.map(|()| name.clone())); + insts.push(Inst::Call { info }) + } &CallDest::ExtName(ref name, RelocDistance::Far) => { insts.push(Inst::LoadExtName { rd: tmp, name: Box::new(name.clone()), offset: 0, }); - insts.push(Inst::CallInd { - info: Box::new(CallIndInfo { - rn: tmp.to_reg(), - uses, - defs, - clobbers, - caller_callconv: caller_conv, - callee_callconv: callee_conv, - callee_pop_size, - }), - }); + let info = Box::new(info.map(|()| tmp.to_reg())); + insts.push(Inst::CallInd { info }); + } + &CallDest::Reg(reg) => { + let info = Box::new(info.map(|()| *reg)); + insts.push(Inst::CallInd { info }); } - &CallDest::Reg(reg) => insts.push(Inst::CallInd { - info: Box::new(CallIndInfo { - rn: *reg, - uses, - defs, - clobbers, - caller_callconv: caller_conv, - callee_callconv: callee_conv, - callee_pop_size, - }), - }), } insts } @@ -655,8 +620,8 @@ impl ABIMachineSpec for Riscv64MachineDeps { ], defs: smallvec![], clobbers: Self::get_regs_clobbered_by_call(call_conv), - caller_callconv: call_conv, - callee_callconv: call_conv, + caller_conv: call_conv, + callee_conv: call_conv, callee_pop_size: 0, }), }); @@ -773,17 +738,15 @@ impl Riscv64ABICallSite { let dest = self.dest().clone(); let uses = self.take_uses(); - let info = Box::new(ReturnCallInfo { - uses, - new_stack_arg_size, - }); match dest { CallDest::ExtName(name, RelocDistance::Near) => { - ctx.emit(Inst::ReturnCall { - callee: Box::new(name), - info, + let info = Box::new(ReturnCallInfo { + dest: name, + uses, + new_stack_arg_size, }); + ctx.emit(Inst::ReturnCall { info }); } CallDest::ExtName(name, RelocDistance::Far) => { let callee = ctx.alloc_tmp(ir::types::I64).only_reg().unwrap(); @@ -792,12 +755,21 @@ impl Riscv64ABICallSite { name: Box::new(name), offset: 0, }); - ctx.emit(Inst::ReturnCallInd { - callee: callee.to_reg(), - info, + let info = Box::new(ReturnCallInfo { + dest: callee.to_reg(), + uses, + new_stack_arg_size, + }); + ctx.emit(Inst::ReturnCallInd { info }); + } + CallDest::Reg(callee) => { + let info = Box::new(ReturnCallInfo { + dest: callee, + uses, + new_stack_arg_size, }); + ctx.emit(Inst::ReturnCallInd { info }); } - CallDest::Reg(callee) => ctx.emit(Inst::ReturnCallInd { callee, info }), } } } diff --git a/cranelift/codegen/src/isa/riscv64/inst.isle b/cranelift/codegen/src/isa/riscv64/inst.isle index 1e9e004b35e9..f6e457094153 100644 --- a/cranelift/codegen/src/isa/riscv64/inst.isle +++ b/cranelift/codegen/src/isa/riscv64/inst.isle @@ -111,22 +111,16 @@ (from_bits u8) (to_bits u8)) - (Call - (info BoxCallInfo)) + (Call (info BoxCallInfo)) ;; A machine indirect-call instruction. - (CallInd - (info BoxCallIndInfo)) + (CallInd (info BoxCallIndInfo)) ;; A direct return-call macro instruction. - (ReturnCall - (callee BoxExternalName) - (info BoxReturnCallInfo)) + (ReturnCall (info BoxReturnCallInfo)) ;; An indirect return-call macro instruction. - (ReturnCallInd - (callee Reg) - (info BoxReturnCallInfo)) + (ReturnCallInd (info BoxReturnCallIndInfo)) ;; Emits a trap with the given trap code if the comparison succeeds (TrapIf @@ -780,6 +774,7 @@ (type BoxCallInfo (primitive BoxCallInfo)) (type BoxCallIndInfo (primitive BoxCallIndInfo)) (type BoxReturnCallInfo (primitive BoxReturnCallInfo)) +(type BoxReturnCallIndInfo (primitive BoxReturnCallIndInfo)) (type IntegerCompare (primitive IntegerCompare)) (type AMode (primitive AMode)) (type OptionReg (primitive OptionReg)) diff --git a/cranelift/codegen/src/isa/riscv64/inst/emit.rs b/cranelift/codegen/src/isa/riscv64/inst/emit.rs index bbce4ec90fdd..9282abb2e086 100644 --- a/cranelift/codegen/src/isa/riscv64/inst/emit.rs +++ b/cranelift/codegen/src/isa/riscv64/inst/emit.rs @@ -1137,7 +1137,7 @@ impl Inst { &Inst::CallInd { ref info } => { Inst::Jalr { rd: writable_link_reg(), - base: info.rn, + base: info.dest, offset: Imm12::ZERO, } .emit(sink, emit_info, state); @@ -1157,25 +1157,22 @@ impl Inst { } } - &Inst::ReturnCall { - ref callee, - ref info, - } => { + &Inst::ReturnCall { ref info } => { emit_return_call_common_sequence(sink, emit_info, state, info); sink.add_call_site(); - sink.add_reloc(Reloc::RiscvCallPlt, &**callee, 0); + sink.add_reloc(Reloc::RiscvCallPlt, &info.dest, 0); Inst::construct_auipc_and_jalr(None, writable_spilltmp_reg(), 0) .into_iter() .for_each(|i| i.emit_uncompressed(sink, emit_info, state, start_off)); } - &Inst::ReturnCallInd { callee, ref info } => { + &Inst::ReturnCallInd { ref info } => { emit_return_call_common_sequence(sink, emit_info, state, &info); Inst::Jalr { rd: writable_zero_reg(), - base: callee, + base: info.dest, offset: Imm12::ZERO, } .emit(sink, emit_info, state); @@ -1986,15 +1983,10 @@ impl Inst { .emit_uncompressed(sink, emit_info, state, start_off); Inst::Call { - info: Box::new(CallInfo { - dest: ExternalName::LibCall(LibCall::ElfTlsGetAddr), - uses: smallvec![], - defs: smallvec![], - caller_callconv: CallConv::SystemV, - callee_callconv: CallConv::SystemV, - callee_pop_size: 0, - clobbers: PRegSet::empty(), - }), + info: Box::new(CallInfo::empty( + ExternalName::LibCall(LibCall::ElfTlsGetAddr), + CallConv::SystemV, + )), } .emit_uncompressed(sink, emit_info, state, start_off); } @@ -2589,11 +2581,11 @@ impl Inst { } } -fn emit_return_call_common_sequence( +fn emit_return_call_common_sequence( sink: &mut MachBuffer, emit_info: &EmitInfo, state: &mut EmitState, - info: &ReturnCallInfo, + info: &ReturnCallInfo, ) { // The return call sequence can potentially emit a lot of instructions (up to 634 bytes!) // So lets emit an island here if we need it. @@ -2623,11 +2615,11 @@ fn emit_return_call_common_sequence( } /// This should not be called directly, Instead prefer to call [emit_return_call_common_sequence]. -fn return_call_emit_impl( +fn return_call_emit_impl( sink: &mut MachBuffer, emit_info: &EmitInfo, state: &mut EmitState, - info: &ReturnCallInfo, + info: &ReturnCallInfo, ) { let sp_to_fp_offset = { let frame_layout = state.frame_layout(); diff --git a/cranelift/codegen/src/isa/riscv64/inst/mod.rs b/cranelift/codegen/src/isa/riscv64/inst/mod.rs index 292892b9fe81..14d5fd4fc764 100644 --- a/cranelift/codegen/src/isa/riscv64/inst/mod.rs +++ b/cranelift/codegen/src/isa/riscv64/inst/mod.rs @@ -13,7 +13,7 @@ use crate::{settings, CodegenError, CodegenResult}; pub use crate::ir::condcodes::FloatCC; use alloc::vec::Vec; -use regalloc2::{PRegSet, RegClass}; +use regalloc2::RegClass; use smallvec::{smallvec, SmallVec}; use std::boxed::Box; use std::fmt::Write; @@ -51,36 +51,11 @@ pub use crate::isa::riscv64::lower::isle::generated_code::{ }; use crate::isa::riscv64::lower::isle::generated_code::{CjOp, MInst, VecAluOpRRImm5, VecAluOpRRR}; -/// Additional information for (direct) Call instructions, left out of line to lower the size of -/// the Inst enum. -#[derive(Clone, Debug)] -pub struct CallInfo { - pub dest: ExternalName, - pub uses: CallArgList, - pub defs: CallRetList, - pub caller_callconv: CallConv, - pub callee_callconv: CallConv, - pub clobbers: PRegSet, - pub callee_pop_size: u32, -} - -/// Additional information for CallInd instructions, left out of line to lower the size of the Inst -/// enum. -#[derive(Clone, Debug)] -pub struct CallIndInfo { - pub rn: Reg, - pub uses: CallArgList, - pub defs: CallRetList, - pub caller_callconv: CallConv, - pub callee_callconv: CallConv, - pub clobbers: PRegSet, - pub callee_pop_size: u32, -} - /// Additional information for `return_call[_ind]` instructions, left out of /// line to lower the size of the `Inst` enum. #[derive(Clone, Debug)] -pub struct ReturnCallInfo { +pub struct ReturnCallInfo { + pub dest: T, pub uses: CallArgList, pub new_stack_arg_size: u32, } @@ -355,7 +330,7 @@ fn riscv64_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) { collector.reg_use(rn); collector.reg_def(rd); } - Inst::Call { info } => { + Inst::Call { info, .. } => { let CallInfo { uses, defs, .. } = &mut **info; for CallArgPair { vreg, preg } in uses { collector.reg_fixed_use(vreg, *preg); @@ -366,8 +341,10 @@ fn riscv64_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) { collector.reg_clobbers(info.clobbers); } Inst::CallInd { info } => { - let CallIndInfo { rn, uses, defs, .. } = &mut **info; - collector.reg_use(rn); + let CallInfo { + dest, uses, defs, .. + } = &mut **info; + collector.reg_use(dest); for CallArgPair { vreg, preg } in uses { collector.reg_fixed_use(vreg, *preg); } @@ -376,15 +353,15 @@ fn riscv64_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) { } collector.reg_clobbers(info.clobbers); } - Inst::ReturnCall { info, .. } => { + Inst::ReturnCall { info } => { for CallArgPair { vreg, preg } in &mut info.uses { collector.reg_fixed_use(vreg, *preg); } } - Inst::ReturnCallInd { info, callee } => { + Inst::ReturnCallInd { info } => { // TODO(https://github.com/bytecodealliance/regalloc2/issues/145): // This shouldn't be a fixed register constraint. - collector.reg_fixed_use(callee, x_reg(5)); + collector.reg_fixed_use(&mut info.dest, x_reg(5)); for CallArgPair { vreg, preg } in &mut info.uses { collector.reg_fixed_use(vreg, *preg); @@ -1323,16 +1300,13 @@ impl Inst { } &MInst::Call { ref info } => format!("call {}", info.dest.display(None)), &MInst::CallInd { ref info } => { - let rd = format_reg(info.rn); + let rd = format_reg(info.dest); format!("callind {rd}") } - &MInst::ReturnCall { - ref callee, - ref info, - } => { + &MInst::ReturnCall { ref info } => { let mut s = format!( - "return_call {callee:?} new_stack_arg_size:{}", - info.new_stack_arg_size + "return_call {:?} new_stack_arg_size:{}", + info.dest, info.new_stack_arg_size ); for ret in &info.uses { let preg = format_reg(ret.preg); @@ -1341,8 +1315,8 @@ impl Inst { } s } - &MInst::ReturnCallInd { callee, ref info } => { - let callee = format_reg(callee); + &MInst::ReturnCallInd { ref info } => { + let callee = format_reg(info.dest); let mut s = format!( "return_call_ind {callee} new_stack_arg_size:{}", info.new_stack_arg_size diff --git a/cranelift/codegen/src/isa/riscv64/lower/isle.rs b/cranelift/codegen/src/isa/riscv64/lower/isle.rs index 3c0090d0b2d2..28a0bd064667 100644 --- a/cranelift/codegen/src/isa/riscv64/lower/isle.rs +++ b/cranelift/codegen/src/isa/riscv64/lower/isle.rs @@ -14,7 +14,7 @@ use crate::isa::riscv64::lower::args::{ }; use crate::isa::riscv64::Riscv64Backend; use crate::machinst::Reg; -use crate::machinst::{isle::*, MachInst}; +use crate::machinst::{isle::*, CallInfo, MachInst}; use crate::machinst::{VCodeConstant, VCodeConstantData}; use crate::{ ir::{ @@ -28,9 +28,10 @@ use regalloc2::PReg; use std::boxed::Box; use std::vec::Vec; -type BoxCallInfo = Box; -type BoxCallIndInfo = Box; -type BoxReturnCallInfo = Box; +type BoxCallInfo = Box>; +type BoxCallIndInfo = Box>; +type BoxReturnCallInfo = Box>; +type BoxReturnCallIndInfo = Box>; type BoxExternalName = Box; type VecMachLabel = Vec; type VecArgPair = Vec; diff --git a/cranelift/codegen/src/isa/s390x/abi.rs b/cranelift/codegen/src/isa/s390x/abi.rs index 20082a80b758..f6de879b1a43 100644 --- a/cranelift/codegen/src/isa/s390x/abi.rs +++ b/cranelift/codegen/src/isa/s390x/abi.rs @@ -801,16 +801,7 @@ impl ABIMachineSpec for S390xMachineDeps { insts } - fn gen_call( - _dest: &CallDest, - _uses: CallArgList, - _defs: CallRetList, - _clobbers: PRegSet, - _tmp: Writable, - _callee_conv: isa::CallConv, - _caller_conv: isa::CallConv, - _callee_pop_size: u32, - ) -> SmallVec<[Inst; 2]> { + fn gen_call(_dest: &CallDest, _tmp: Writable, _info: CallInfo<()>) -> SmallVec<[Inst; 2]> { unreachable!(); } diff --git a/cranelift/codegen/src/isa/s390x/inst.isle b/cranelift/codegen/src/isa/s390x/inst.isle index f4a88a254343..36af4ba75cc5 100644 --- a/cranelift/codegen/src/isa/s390x/inst.isle +++ b/cranelift/codegen/src/isa/s390x/inst.isle @@ -1014,6 +1014,15 @@ ;; machine at this program point. (Unwind (inst UnwindInst)) + + ;; Pseudo-instruction used for `tls_value` to call the libcall of the same + ;; name. + (ElfTlsGetOffset + (tls_offset WritableReg) + (got Reg) + (got_offset Reg) + (link WritableReg) + (symbol BoxSymbolReloc)) )) ;; Primitive types used in instruction formats. @@ -2912,7 +2921,8 @@ (rule 1 (memcpy dst src (len_minus_one len)) (mvc (memarg_pair dst) (memarg_pair src) len)) (rule (memcpy dst src len) - (lib_call (lib_call_info_memcpy (load_addr dst) (load_addr src) (imm $I64 len)))) + (lib_call + (lib_call_info_memcpy (load_addr dst) (load_addr src) (imm $I64 len)))) ;; Prepare a stack copy of a single (oversized) argument. (decl copy_to_buffer (MemArg ABIArg Value) InstOutput) @@ -3605,18 +3615,12 @@ (type LibCallInfo extern (enum)) -(decl lib_call_info_memcpy (Reg Reg Reg) LibCallInfo) +(decl lib_call_info_memcpy (Reg Reg Reg) BoxCallInfo) (extern constructor lib_call_info_memcpy lib_call_info_memcpy) -(decl lib_call_info_tls_get_offset (WritableReg Reg Reg SymbolReloc) LibCallInfo) -(extern constructor lib_call_info_tls_get_offset lib_call_info_tls_get_offset) - -(decl lib_call_info (LibCallInfo) BoxCallInfo) -(extern constructor lib_call_info lib_call_info) - -(decl lib_call (LibCallInfo) SideEffectNoResult) -(rule (lib_call libcall) - (call_impl (writable_link_reg) (lib_call_info libcall))) +(decl lib_call (BoxCallInfo) SideEffectNoResult) +(rule (lib_call info) + (call_impl (writable_link_reg) info)) ;; Helpers for generating vector pack and unpack instructions ;;;;;;;;;;;;;;;;;; diff --git a/cranelift/codegen/src/isa/s390x/inst/emit.rs b/cranelift/codegen/src/isa/s390x/inst/emit.rs index 7d9fcc3e7e02..9fe3fed81875 100644 --- a/cranelift/codegen/src/isa/s390x/inst/emit.rs +++ b/cranelift/codegen/src/isa/s390x/inst/emit.rs @@ -1,6 +1,6 @@ //! S390x ISA: binary code emission. -use crate::ir::{self, MemFlags, TrapCode}; +use crate::ir::{self, LibCall, MemFlags, TrapCode}; use crate::isa::s390x::inst::*; use crate::isa::s390x::settings as s390x_settings; use cranelift_control::ControlPlane; @@ -3267,15 +3267,6 @@ impl Inst { // works correctly. sink.add_reloc_at_offset(2, Reloc::S390xPLTRel32Dbl, &info.dest, 2); - // Add relocation for TLS libcalls to enable linker optimizations. - match &info.tls_symbol { - None => {} - Some(SymbolReloc::TlsGd { name }) => { - sink.add_reloc(Reloc::S390xTlsGdCall, name, 0) - } - _ => unreachable!(), - } - if let Some(s) = state.take_stack_map() { let offset = sink.cur_offset() + 6; sink.push_user_stack_map(state, offset, s); @@ -3288,7 +3279,6 @@ impl Inst { } &Inst::CallInd { link, ref info } => { debug_assert_eq!(link.to_reg(), gpr(14)); - let rn = info.rn; if let Some(s) = state.take_stack_map() { let offset = sink.cur_offset() + 2; @@ -3296,7 +3286,7 @@ impl Inst { } let opcode = 0x0d; // BASR - put(sink, &enc_rr(opcode, link.to_reg(), rn)); + put(sink, &enc_rr(opcode, link.to_reg(), info.dest)); sink.add_call_site(); state.nominal_sp_offset -= info.callee_pop_size; @@ -3316,7 +3306,7 @@ impl Inst { sink.add_call_site(); } &Inst::ReturnCallInd { ref info } => { - let mut rn = info.rn; + let mut rn = info.dest; for inst in S390xMachineDeps::gen_tail_epilogue( state.frame_layout(), info.callee_pop_size, @@ -3329,6 +3319,26 @@ impl Inst { put(sink, &enc_rr(opcode, gpr(15), rn)); sink.add_call_site(); } + &Inst::ElfTlsGetOffset { + ref symbol, link, .. + } => { + debug_assert_eq!(link.to_reg(), gpr(14)); + + let opcode = 0xc05; // BRASL + + // Add relocation for target function. This has to be done + // *before* the S390xTlsGdCall, to ensure linker relaxation + // works correctly. + let dest = ExternalName::LibCall(LibCall::ElfTlsGetOffset); + sink.add_reloc_at_offset(2, Reloc::S390xPLTRel32Dbl, &dest, 2); + match &**symbol { + SymbolReloc::TlsGd { name } => sink.add_reloc(Reloc::S390xTlsGdCall, name, 0), + _ => unreachable!(), + } + + put(sink, &enc_ril_b(opcode, gpr(14), 0)); + sink.add_call_site(); + } &Inst::Args { .. } => {} &Inst::Rets { .. } => {} &Inst::Ret { link } => { diff --git a/cranelift/codegen/src/isa/s390x/inst/emit_tests.rs b/cranelift/codegen/src/isa/s390x/inst/emit_tests.rs index 9108b1fc0e7c..159301f736b8 100644 --- a/cranelift/codegen/src/isa/s390x/inst/emit_tests.rs +++ b/cranelift/codegen/src/isa/s390x/inst/emit_tests.rs @@ -1,7 +1,6 @@ use crate::ir::{MemFlags, TrapCode}; use crate::isa::s390x::inst::*; use crate::isa::s390x::settings as s390x_settings; -use smallvec::smallvec; #[cfg(test)] fn simm20_zero() -> SImm20 { @@ -6984,16 +6983,10 @@ fn test_s390x_binemit() { insns.push(( Inst::Call { link: writable_gpr(14), - info: Box::new(CallInfo { - dest: ExternalName::testcase("test0"), - uses: smallvec![], - defs: smallvec![], - clobbers: PRegSet::empty(), - callee_pop_size: 0, - caller_callconv: CallConv::SystemV, - callee_callconv: CallConv::SystemV, - tls_symbol: None, - }), + info: Box::new(CallInfo::empty( + ExternalName::testcase("test0"), + CallConv::SystemV, + )), }, "C0E500000000", "brasl %r14, %test0", @@ -7002,15 +6995,7 @@ fn test_s390x_binemit() { insns.push(( Inst::CallInd { link: writable_gpr(14), - info: Box::new(CallIndInfo { - rn: gpr(1), - uses: smallvec![], - defs: smallvec![], - clobbers: PRegSet::empty(), - callee_pop_size: 0, - caller_callconv: CallConv::SystemV, - callee_callconv: CallConv::SystemV, - }), + info: Box::new(CallInfo::empty(gpr(1), CallConv::SystemV)), }, "0DE1", "basr %r14, %r1", diff --git a/cranelift/codegen/src/isa/s390x/inst/mod.rs b/cranelift/codegen/src/isa/s390x/inst/mod.rs index 763c35a51ece..bccd0057bf62 100644 --- a/cranelift/codegen/src/isa/s390x/inst/mod.rs +++ b/cranelift/codegen/src/isa/s390x/inst/mod.rs @@ -34,47 +34,11 @@ pub use crate::isa::s390x::lower::isle::generated_code::{ VecUnaryOp, }; -/// Additional information for (direct) Call instructions, left out of line to lower the size of -/// the Inst enum. -#[derive(Clone, Debug)] -pub struct CallInfo { - pub dest: ExternalName, - pub uses: CallArgList, - pub defs: CallRetList, - pub clobbers: PRegSet, - pub callee_pop_size: u32, - pub caller_callconv: CallConv, - pub callee_callconv: CallConv, - pub tls_symbol: Option, -} - -/// Additional information for CallInd instructions, left out of line to lower the size of the Inst -/// enum. -#[derive(Clone, Debug)] -pub struct CallIndInfo { - pub rn: Reg, - pub uses: CallArgList, - pub defs: CallRetList, - pub clobbers: PRegSet, - pub callee_pop_size: u32, - pub caller_callconv: CallConv, - pub callee_callconv: CallConv, -} - /// Additional information for (direct) ReturnCall instructions, left out of line to lower the size of /// the Inst enum. #[derive(Clone, Debug)] -pub struct ReturnCallInfo { - pub dest: ExternalName, - pub uses: CallArgList, - pub callee_pop_size: u32, -} - -/// Additional information for ReturnCallInd instructions, left out of line to lower the size of the Inst -/// enum. -#[derive(Clone, Debug)] -pub struct ReturnCallIndInfo { - pub rn: Reg, +pub struct ReturnCallInfo { + pub dest: T, pub uses: CallArgList, pub callee_pop_size: u32, } @@ -267,7 +231,8 @@ impl Inst { | Inst::LoadAddr { .. } | Inst::Loop { .. } | Inst::CondBreak { .. } - | Inst::Unwind { .. } => InstructionSet::Base, + | Inst::Unwind { .. } + | Inst::ElfTlsGetOffset { .. } => InstructionSet::Base, // These depend on the opcode Inst::AluRRR { alu_op, .. } => match alu_op { @@ -908,7 +873,7 @@ fn s390x_get_operands(inst: &mut Inst, collector: &mut DenyReuseVisitor {} - Inst::Call { link, info } => { + Inst::Call { link, info, .. } => { let CallInfo { uses, defs, @@ -927,14 +892,14 @@ fn s390x_get_operands(inst: &mut Inst, collector: &mut DenyReuseVisitor { - let CallIndInfo { - rn, + let CallInfo { + dest, uses, defs, clobbers, .. } = &mut **info; - collector.reg_use(rn); + collector.reg_use(dest); for CallArgPair { vreg, preg } in uses { collector.reg_fixed_use(vreg, *preg); } @@ -953,12 +918,28 @@ fn s390x_get_operands(inst: &mut Inst, collector: &mut DenyReuseVisitor { - let ReturnCallIndInfo { rn, uses, .. } = &mut **info; - collector.reg_use(rn); + let ReturnCallInfo { dest, uses, .. } = &mut **info; + collector.reg_use(dest); for CallArgPair { vreg, preg } in uses { collector.reg_fixed_use(vreg, *preg); } } + Inst::ElfTlsGetOffset { + tls_offset, + got, + got_offset, + link, + .. + } => { + collector.reg_fixed_use(got, gpr(12)); + collector.reg_fixed_use(got_offset, gpr(2)); + collector.reg_fixed_def(tls_offset, gpr(2)); + + let mut clobbers = S390xMachineDeps::get_regs_clobbered_by_call(CallConv::SystemV); + clobbers.add(link.to_reg().to_real_reg().unwrap().into()); + clobbers.remove(gpr_preg(2)); + collector.reg_clobbers(clobbers); + } Inst::Args { args } => { for ArgPair { vreg, preg } in args { collector.reg_fixed_def(vreg, *preg); @@ -1082,8 +1063,9 @@ impl MachInst for Inst { // registers. match self { &Inst::Args { .. } => false, - &Inst::Call { ref info, .. } => info.caller_callconv != info.callee_callconv, - &Inst::CallInd { ref info, .. } => info.caller_callconv != info.callee_callconv, + &Inst::Call { ref info, .. } => info.caller_conv != info.callee_conv, + &Inst::CallInd { ref info, .. } => info.caller_conv != info.callee_conv, + &Inst::ElfTlsGetOffset { .. } => false, _ => true, } } @@ -3159,15 +3141,8 @@ impl Inst { format!("slgfi {}, {}", show_reg(stack_reg()), size) } } - &Inst::Call { link, ref info, .. } => { + &Inst::Call { link, ref info } => { let link = link.to_reg(); - let tls_symbol = match &info.tls_symbol { - None => "".to_string(), - Some(SymbolReloc::TlsGd { name }) => { - format!(":tls_gdcall:{}", name.display(None)) - } - _ => unreachable!(), - }; let callee_pop_size = if info.callee_pop_size > 0 { format!(" ; callee_pop_size {}", info.callee_pop_size) } else { @@ -3175,16 +3150,15 @@ impl Inst { }; debug_assert_eq!(link, gpr(14)); format!( - "brasl {}, {}{}{}", + "brasl {}, {}{}", show_reg(link), info.dest.display(None), - tls_symbol, callee_pop_size ) } &Inst::CallInd { link, ref info, .. } => { let link = link.to_reg(); - let rn = pretty_print_reg(info.rn); + let rn = pretty_print_reg(info.dest); let callee_pop_size = if info.callee_pop_size > 0 { format!(" ; callee_pop_size {}", info.callee_pop_size) } else { @@ -3193,7 +3167,7 @@ impl Inst { debug_assert_eq!(link, gpr(14)); format!("basr {}, {}{}", show_reg(link), rn, callee_pop_size) } - &Inst::ReturnCall { ref info, .. } => { + &Inst::ReturnCall { ref info } => { let callee_pop_size = if info.callee_pop_size > 0 { format!(" ; callee_pop_size {}", info.callee_pop_size) } else { @@ -3201,8 +3175,8 @@ impl Inst { }; format!("return_call {}{}", info.dest.display(None), callee_pop_size) } - &Inst::ReturnCallInd { ref info, .. } => { - let rn = pretty_print_reg(info.rn); + &Inst::ReturnCallInd { ref info } => { + let rn = pretty_print_reg(info.dest); let callee_pop_size = if info.callee_pop_size > 0 { format!(" ; callee_pop_size {}", info.callee_pop_size) } else { @@ -3210,6 +3184,21 @@ impl Inst { }; format!("return_call_ind {rn}{callee_pop_size}") } + &Inst::ElfTlsGetOffset { + ref symbol, + ref link, + .. + } => { + let link = link.to_reg(); + let dest = match &**symbol { + SymbolReloc::TlsGd { name } => { + format!("tls_gdcall:{}", name.display(None)) + } + _ => unreachable!(), + }; + debug_assert_eq!(link, gpr(14)); + format!("brasl {}, {}", show_reg(link), dest) + } &Inst::Args { ref args } => { let mut s = "args".to_string(); for arg in args { diff --git a/cranelift/codegen/src/isa/s390x/lower.isle b/cranelift/codegen/src/isa/s390x/lower.isle index d0efcd25fc1b..330cb3910cfc 100644 --- a/cranelift/codegen/src/isa/s390x/lower.isle +++ b/cranelift/codegen/src/isa/s390x/lower.isle @@ -2269,10 +2269,13 @@ (decl lib_call_tls_get_offset (Reg Reg SymbolReloc) Reg) (rule (lib_call_tls_get_offset got got_offset symbol) (let ((tls_offset WritableReg (temp_writable_reg $I64)) - (libcall LibCallInfo (lib_call_info_tls_get_offset tls_offset got got_offset symbol)) - (_ InstOutput (side_effect (lib_call libcall)))) + (_ Unit (abi_for_elf_tls_get_offset)) + (_ Unit (emit (MInst.ElfTlsGetOffset tls_offset got got_offset (writable_link_reg) symbol)))) tls_offset)) +(decl abi_for_elf_tls_get_offset () Unit) +(extern constructor abi_for_elf_tls_get_offset abi_for_elf_tls_get_offset) + ;; Helper to extract the current thread pointer from %a0/%a1. (decl thread_pointer () Reg) (rule (thread_pointer) diff --git a/cranelift/codegen/src/isa/s390x/lower/isle.rs b/cranelift/codegen/src/isa/s390x/lower/isle.rs index 42c769536ca6..6d219da85e17 100644 --- a/cranelift/codegen/src/isa/s390x/lower/isle.rs +++ b/cranelift/codegen/src/isa/s390x/lower/isle.rs @@ -7,13 +7,12 @@ use crate::ir::ExternalName; // Types that the generated ISLE code uses via `use super::*`. use crate::isa::s390x::abi::{S390xMachineDeps, REG_SAVE_AREA_SIZE}; use crate::isa::s390x::inst::{ - gpr, stack_reg, writable_gpr, zero_reg, CallIndInfo, CallInfo, Cond, Inst as MInst, LaneOrder, - MemArg, MemArgPair, RegPair, ReturnCallIndInfo, ReturnCallInfo, SymbolReloc, UImm12, - UImm16Shifted, UImm32Shifted, WritableRegPair, + gpr, stack_reg, writable_gpr, zero_reg, Cond, Inst as MInst, LaneOrder, MemArg, MemArgPair, + RegPair, ReturnCallInfo, SymbolReloc, UImm12, UImm16Shifted, UImm32Shifted, WritableRegPair, }; use crate::isa::s390x::S390xBackend; use crate::machinst::isle::*; -use crate::machinst::{MachLabel, Reg}; +use crate::machinst::{CallInfo, MachLabel, Reg}; use crate::{ ir::{ condcodes::*, immediates::*, types::*, ArgumentExtension, ArgumentPurpose, AtomicRmwOp, @@ -33,18 +32,10 @@ use std::boxed::Box; use std::cell::Cell; use std::vec::Vec; -/// Information describing a library call to be emitted. -pub struct LibCallInfo { - libcall: LibCall, - uses: CallArgList, - defs: CallRetList, - tls_symbol: Option, -} - -type BoxCallInfo = Box; -type BoxCallIndInfo = Box; -type BoxReturnCallInfo = Box; -type BoxReturnCallIndInfo = Box; +type BoxCallInfo = Box>; +type BoxCallIndInfo = Box>; +type BoxReturnCallInfo = Box>; +type BoxReturnCallIndInfo = Box>; type VecMachLabel = Vec; type BoxExternalName = Box; type BoxSymbolReloc = Box; @@ -271,56 +262,21 @@ impl generated_code::Context for IsleContext<'_, '_, MInst, S390xBackend> { fn abi_call_info( &mut self, abi: Sig, - name: ExternalName, + dest: ExternalName, uses: &CallArgList, defs: &CallRetList, ) -> BoxCallInfo { - let sig_data = &self.lower_ctx.sigs()[abi]; - // Get clobbers: all caller-saves. These may include return value - // regs, which we will remove from the clobber set later. - let clobbers = S390xMachineDeps::get_regs_clobbered_by_call(sig_data.call_conv()); - let callee_pop_size = if sig_data.call_conv() == CallConv::Tail { - sig_data.sized_stack_arg_space() as u32 - } else { - 0 - }; - Box::new(CallInfo { - dest: name.clone(), - uses: uses.clone(), - defs: defs.clone(), - clobbers, - callee_pop_size, - caller_callconv: self.lower_ctx.abi().call_conv(self.lower_ctx.sigs()), - callee_callconv: self.lower_ctx.sigs()[abi].call_conv(), - tls_symbol: None, - }) + Box::new(self.abi_call_info_no_dest(abi, uses, defs).map(|()| dest)) } fn abi_call_ind_info( &mut self, abi: Sig, - target: Reg, + dest: Reg, uses: &CallArgList, defs: &CallRetList, ) -> BoxCallIndInfo { - let sig_data = &self.lower_ctx.sigs()[abi]; - // Get clobbers: all caller-saves. These may include return value - // regs, which we will remove from the clobber set later. - let clobbers = S390xMachineDeps::get_regs_clobbered_by_call(sig_data.call_conv()); - let callee_pop_size = if sig_data.call_conv() == CallConv::Tail { - sig_data.sized_stack_arg_space() as u32 - } else { - 0 - }; - Box::new(CallIndInfo { - rn: target, - uses: uses.clone(), - defs: defs.clone(), - clobbers, - callee_pop_size, - caller_callconv: self.lower_ctx.abi().call_conv(self.lower_ctx.sigs()), - callee_callconv: self.lower_ctx.sigs()[abi].call_conv(), - }) + Box::new(self.abi_call_info_no_dest(abi, uses, defs).map(|()| dest)) } fn abi_return_call_info( @@ -346,16 +302,32 @@ impl generated_code::Context for IsleContext<'_, '_, MInst, S390xBackend> { ) -> BoxReturnCallIndInfo { let sig_data = &self.lower_ctx.sigs()[abi]; let callee_pop_size = sig_data.sized_stack_arg_space() as u32; - Box::new(ReturnCallIndInfo { - rn: target, + Box::new(ReturnCallInfo { + dest: target, uses: uses.clone(), callee_pop_size, }) } - fn lib_call_info_memcpy(&mut self, dst: Reg, src: Reg, len: Reg) -> LibCallInfo { - LibCallInfo { - libcall: LibCall::Memcpy, + fn lib_call_info_memcpy( + &mut self, + dst: Reg, + src: Reg, + len: Reg, + ) -> Box> { + let caller_conv = self.lower_ctx.abi().call_conv(self.lower_ctx.sigs()); + let callee_conv = CallConv::for_libcall(&self.backend.flags, caller_conv); + + // Clobbers are defined by the calling convention. We will remove return value regs later. + let clobbers = S390xMachineDeps::get_regs_clobbered_by_call(callee_conv); + + // Libcalls only require the register save area. + self.lower_ctx + .abi_mut() + .accumulate_outgoing_args_size(REG_SAVE_AREA_SIZE); + + Box::new(CallInfo { + dest: ExternalName::LibCall(LibCall::Memcpy), uses: smallvec![ CallArgPair { vreg: dst, @@ -371,59 +343,17 @@ impl generated_code::Context for IsleContext<'_, '_, MInst, S390xBackend> { }, ], defs: smallvec![], - tls_symbol: None, - } - } - - fn lib_call_info_tls_get_offset( - &mut self, - tls_offset: WritableReg, - got: Reg, - got_offset: Reg, - tls_symbol: &SymbolReloc, - ) -> LibCallInfo { - LibCallInfo { - libcall: LibCall::ElfTlsGetOffset, - uses: smallvec![ - CallArgPair { - vreg: got, - preg: gpr(12), - }, - CallArgPair { - vreg: got_offset, - preg: gpr(2), - }, - ], - defs: smallvec![CallRetPair { - vreg: tls_offset, - preg: gpr(2), - },], - tls_symbol: Some(tls_symbol.clone()), - } + clobbers, + caller_conv, + callee_conv, + callee_pop_size: 0, + }) } - fn lib_call_info(&mut self, info: &LibCallInfo) -> BoxCallInfo { - let caller_callconv = self.lower_ctx.abi().call_conv(self.lower_ctx.sigs()); - let callee_callconv = CallConv::for_libcall(&self.backend.flags, caller_callconv); - - // Clobbers are defined by the calling convention. We will remove return value regs later. - let clobbers = S390xMachineDeps::get_regs_clobbered_by_call(callee_callconv); - - // Libcalls only require the register save area. + fn abi_for_elf_tls_get_offset(&mut self) { self.lower_ctx .abi_mut() .accumulate_outgoing_args_size(REG_SAVE_AREA_SIZE); - - Box::new(CallInfo { - dest: ExternalName::LibCall(info.libcall), - uses: info.uses.clone(), - defs: info.defs.clone(), - clobbers, - callee_pop_size: 0, - caller_callconv, - callee_callconv, - tls_symbol: info.tls_symbol.clone(), - }) } #[inline] @@ -1098,6 +1028,34 @@ impl generated_code::Context for IsleContext<'_, '_, MInst, S390xBackend> { } } +impl IsleContext<'_, '_, MInst, S390xBackend> { + fn abi_call_info_no_dest( + &mut self, + abi: Sig, + uses: &CallArgList, + defs: &CallRetList, + ) -> CallInfo<()> { + let sig_data = &self.lower_ctx.sigs()[abi]; + // Get clobbers: all caller-saves. These may include return value + // regs, which we will remove from the clobber set later. + let clobbers = S390xMachineDeps::get_regs_clobbered_by_call(sig_data.call_conv()); + let callee_pop_size = if sig_data.call_conv() == CallConv::Tail { + sig_data.sized_stack_arg_space() as u32 + } else { + 0 + }; + CallInfo { + dest: (), + uses: uses.clone(), + defs: defs.clone(), + clobbers, + callee_pop_size, + caller_conv: self.lower_ctx.abi().call_conv(self.lower_ctx.sigs()), + callee_conv: self.lower_ctx.sigs()[abi].call_conv(), + } + } +} + /// Lane order to be used for a given calling convention. #[inline] fn lane_order_for_call_conv(call_conv: CallConv) -> LaneOrder { diff --git a/cranelift/codegen/src/isa/x64/abi.rs b/cranelift/codegen/src/isa/x64/abi.rs index 9d3c9fd55b92..163a3b8a7583 100644 --- a/cranelift/codegen/src/isa/x64/abi.rs +++ b/cranelift/codegen/src/isa/x64/abi.rs @@ -617,16 +617,12 @@ impl ABIMachineSpec for X64ABIMachineSpec { Writable::from_reg(regs::rax()), )); insts.push(Inst::CallKnown { - dest: ExternalName::LibCall(LibCall::Probestack), - info: Some(Box::new(CallInfo { - // No need to include arg here: we are post-regalloc - // so no constraints will be seen anyway. - uses: smallvec![], - defs: smallvec![], - clobbers: PRegSet::empty(), - callee_pop_size: 0, - callee_conv: CallConv::Probestack, - })), + // No need to include arg here: we are post-regalloc + // so no constraints will be seen anyway. + info: Box::new(CallInfo::empty( + ExternalName::LibCall(LibCall::Probestack), + CallConv::Probestack, + )), }); } @@ -822,27 +818,12 @@ impl ABIMachineSpec for X64ABIMachineSpec { } /// Generate a call instruction/sequence. - fn gen_call( - dest: &CallDest, - uses: CallArgList, - defs: CallRetList, - clobbers: PRegSet, - tmp: Writable, - callee_conv: isa::CallConv, - _caller_conv: isa::CallConv, - callee_pop_size: u32, - ) -> SmallVec<[Self::I; 2]> { + fn gen_call(dest: &CallDest, tmp: Writable, info: CallInfo<()>) -> SmallVec<[Self::I; 2]> { let mut insts = SmallVec::new(); match dest { &CallDest::ExtName(ref name, RelocDistance::Near) => { - insts.push(Inst::call_known( - name.clone(), - uses, - defs, - clobbers, - callee_pop_size, - callee_conv, - )); + let info = Box::new(info.map(|()| name.clone())); + insts.push(Inst::call_known(info)); } &CallDest::ExtName(ref name, RelocDistance::Far) => { insts.push(Inst::LoadExtName { @@ -851,24 +832,12 @@ impl ABIMachineSpec for X64ABIMachineSpec { offset: 0, distance: RelocDistance::Far, }); - insts.push(Inst::call_unknown( - RegMem::reg(tmp.to_reg()), - uses, - defs, - clobbers, - callee_pop_size, - callee_conv, - )); + let info = Box::new(info.map(|()| RegMem::reg(tmp.to_reg()))); + insts.push(Inst::call_unknown(info)); } &CallDest::Reg(reg) => { - insts.push(Inst::call_unknown( - RegMem::reg(reg), - uses, - defs, - clobbers, - callee_pop_size, - callee_conv, - )); + let info = Box::new(info.map(|()| RegMem::reg(reg))); + insts.push(Inst::call_unknown(info)); } } insts @@ -898,10 +867,9 @@ impl ABIMachineSpec for X64ABIMachineSpec { distance: RelocDistance::Far, }); let callee_pop_size = 0; - insts.push(Inst::call_unknown( - RegMem::reg(temp2.to_reg()), - /* uses = */ - smallvec![ + insts.push(Inst::call_unknown(Box::new(CallInfo { + dest: RegMem::reg(temp2.to_reg()), + uses: smallvec![ CallArgPair { vreg: dst, preg: arg0 @@ -915,11 +883,12 @@ impl ABIMachineSpec for X64ABIMachineSpec { preg: arg2 }, ], - /* defs = */ smallvec![], - /* clobbers = */ Self::get_regs_clobbered_by_call(call_conv), + defs: smallvec![], + clobbers: Self::get_regs_clobbered_by_call(call_conv), callee_pop_size, - call_conv, - )); + callee_conv: call_conv, + caller_conv: call_conv, + }))); insts } @@ -1028,14 +997,17 @@ impl X64CallSite { // Finally, do the actual tail call! let dest = self.dest().clone(); - let info = Box::new(ReturnCallInfo { - new_stack_arg_size, - uses: self.take_uses(), - tmp: ctx.temp_writable_gpr(), - }); + let uses = self.take_uses(); + let tmp = ctx.temp_writable_gpr(); match dest { CallDest::ExtName(callee, RelocDistance::Near) => { - ctx.emit(Inst::ReturnCallKnown { callee, info }); + let info = Box::new(ReturnCallInfo { + dest: callee, + uses, + tmp, + new_stack_arg_size, + }); + ctx.emit(Inst::ReturnCallKnown { info }); } CallDest::ExtName(callee, RelocDistance::Far) => { let tmp2 = ctx.temp_writable_gpr(); @@ -1045,15 +1017,23 @@ impl X64CallSite { offset: 0, distance: RelocDistance::Far, }); - ctx.emit(Inst::ReturnCallUnknown { - callee: tmp2.to_reg().to_reg(), - info, + let info = Box::new(ReturnCallInfo { + dest: tmp2.to_reg().to_reg().into(), + uses, + tmp, + new_stack_arg_size, + }); + ctx.emit(Inst::ReturnCallUnknown { info }); + } + CallDest::Reg(callee) => { + let info = Box::new(ReturnCallInfo { + dest: callee.into(), + uses, + tmp, + new_stack_arg_size, }); + ctx.emit(Inst::ReturnCallUnknown { info }); } - CallDest::Reg(callee) => ctx.emit(Inst::ReturnCallUnknown { - callee: callee.into(), - info, - }), } } } diff --git a/cranelift/codegen/src/isa/x64/inst.isle b/cranelift/codegen/src/isa/x64/inst.isle index 80e46ec44bd9..385afe68d481 100644 --- a/cranelift/codegen/src/isa/x64/inst.isle +++ b/cranelift/codegen/src/isa/x64/inst.isle @@ -531,20 +531,16 @@ ;; Control flow instructions. ;; Direct call: call simm32. - (CallKnown (dest ExternalName) - (info BoxCallInfo)) + (CallKnown (info BoxCallInfo)) ;; Indirect call: callq (reg mem) - (CallUnknown (dest RegMem) - (info BoxCallInfo)) + (CallUnknown (info BoxCallIndInfo)) ;; Tail call to a direct destination. - (ReturnCallKnown (callee ExternalName) - (info BoxReturnCallInfo)) + (ReturnCallKnown (info BoxReturnCallInfo)) ;; Tail call to an indirect destination. - (ReturnCallUnknown (callee Reg) - (info BoxReturnCallInfo)) + (ReturnCallUnknown (info BoxReturnCallIndInfo)) ;; A pseudo-instruction that captures register arguments in vregs. (Args @@ -748,7 +744,9 @@ SFence)) (type BoxCallInfo extern (enum)) +(type BoxCallIndInfo extern (enum)) (type BoxReturnCallInfo extern (enum)) +(type BoxReturnCallIndInfo extern (enum)) ;; Get the `OperandSize` for a given `Type`, rounding smaller types up to 32 bits. (decl operand_size_of_type_32_64 (Type) OperandSize) diff --git a/cranelift/codegen/src/isa/x64/inst/emit.rs b/cranelift/codegen/src/isa/x64/inst/emit.rs index 9bc374ce507c..77e0ec45c6bd 100644 --- a/cranelift/codegen/src/isa/x64/inst/emit.rs +++ b/cranelift/codegen/src/isa/x64/inst/emit.rs @@ -1593,11 +1593,7 @@ pub(crate) fn emit( inst.emit(sink, info, state); } - Inst::CallKnown { - dest, - info: call_info, - .. - } => { + Inst::CallKnown { info: call_info } => { if let Some(s) = state.take_stack_map() { let offset = sink.cur_offset() + 5; sink.push_user_stack_map(state, offset, s); @@ -1606,29 +1602,24 @@ pub(crate) fn emit( sink.put1(0xE8); // The addend adjusts for the difference between the end of the instruction and the // beginning of the immediate field. - emit_reloc(sink, Reloc::X86CallPCRel4, &dest, -4); + emit_reloc(sink, Reloc::X86CallPCRel4, &call_info.dest, -4); sink.put4(0); sink.add_call_site(); // Reclaim the outgoing argument area that was released by the callee, to ensure that // StackAMode values are always computed from a consistent SP. - if let Some(call_info) = call_info { - if call_info.callee_pop_size > 0 { - Inst::alu_rmi_r( - OperandSize::Size64, - AluRmiROpcode::Sub, - RegMemImm::imm(call_info.callee_pop_size), - Writable::from_reg(regs::rsp()), - ) - .emit(sink, info, state); - } + if call_info.callee_pop_size > 0 { + Inst::alu_rmi_r( + OperandSize::Size64, + AluRmiROpcode::Sub, + RegMemImm::imm(call_info.callee_pop_size), + Writable::from_reg(regs::rsp()), + ) + .emit(sink, info, state); } } - Inst::ReturnCallKnown { - callee, - info: call_info, - } => { + Inst::ReturnCallKnown { info: call_info } => { emit_return_call_common_sequence(sink, info, state, &call_info); // Finally, jump to the callee! @@ -1639,16 +1630,13 @@ pub(crate) fn emit( sink.put1(0xE9); // The addend adjusts for the difference between the end of the instruction and the // beginning of the immediate field. - emit_reloc(sink, Reloc::X86CallPCRel4, &callee, -4); + emit_reloc(sink, Reloc::X86CallPCRel4, &call_info.dest, -4); sink.put4(0); sink.add_call_site(); } - Inst::ReturnCallUnknown { - callee, - info: call_info, - } => { - let callee = *callee; + Inst::ReturnCallUnknown { info: call_info } => { + let callee = call_info.dest; emit_return_call_common_sequence(sink, info, state, &call_info); @@ -1660,11 +1648,9 @@ pub(crate) fn emit( } Inst::CallUnknown { - dest, - info: call_info, - .. + info: call_info, .. } => { - let dest = dest.clone(); + let dest = call_info.dest.clone(); match dest { RegMem::Reg { reg } => { @@ -1704,16 +1690,14 @@ pub(crate) fn emit( // Reclaim the outgoing argument area that was released by the callee, to ensure that // StackAMode values are always computed from a consistent SP. - if let Some(call_info) = call_info { - if call_info.callee_pop_size > 0 { - Inst::alu_rmi_r( - OperandSize::Size64, - AluRmiROpcode::Sub, - RegMemImm::imm(call_info.callee_pop_size), - Writable::from_reg(regs::rsp()), - ) - .emit(sink, info, state); - } + if call_info.callee_pop_size > 0 { + Inst::alu_rmi_r( + OperandSize::Size64, + AluRmiROpcode::Sub, + RegMemImm::imm(call_info.callee_pop_size), + Writable::from_reg(regs::rsp()), + ) + .emit(sink, info, state); } } @@ -4364,11 +4348,11 @@ pub(crate) fn emit( /// arguments). /// /// * Move the return address into its stack slot. -fn emit_return_call_common_sequence( +fn emit_return_call_common_sequence( sink: &mut MachBuffer, info: &EmitInfo, state: &mut EmitState, - call_info: &ReturnCallInfo, + call_info: &ReturnCallInfo, ) { assert!( info.flags.preserve_frame_pointers(), diff --git a/cranelift/codegen/src/isa/x64/inst/emit_tests.rs b/cranelift/codegen/src/isa/x64/inst/emit_tests.rs index cf2150fc356c..777d85830485 100644 --- a/cranelift/codegen/src/isa/x64/inst/emit_tests.rs +++ b/cranelift/codegen/src/isa/x64/inst/emit_tests.rs @@ -3747,14 +3747,10 @@ fn test_x64_emit() { // ======================================================== // CallKnown insns.push(( - Inst::call_known( + Inst::call_known(Box::new(CallInfo::empty( ExternalName::User(UserExternalNameRef::new(0)), - smallvec![], - smallvec![], - PRegSet::default(), - 0, CallConv::SystemV, - ), + ))), "E800000000", "call User(userextname0)", )); @@ -3762,14 +3758,7 @@ fn test_x64_emit() { // ======================================================== // CallUnknown fn call_unknown(rm: RegMem) -> Inst { - Inst::call_unknown( - rm, - smallvec![], - smallvec![], - PRegSet::default(), - 0, - CallConv::SystemV, - ) + Inst::call_unknown(Box::new(CallInfo::empty(rm, CallConv::SystemV))) } insns.push((call_unknown(RegMem::reg(rbp)), "FFD5", "call *%rbp")); diff --git a/cranelift/codegen/src/isa/x64/inst/mod.rs b/cranelift/codegen/src/isa/x64/inst/mod.rs index 705d8151b071..208cf98ace4d 100644 --- a/cranelift/codegen/src/isa/x64/inst/mod.rs +++ b/cranelift/codegen/src/isa/x64/inst/mod.rs @@ -11,7 +11,6 @@ use crate::isa::{CallConv, FunctionAlignment}; use crate::{machinst::*, trace}; use crate::{settings, CodegenError, CodegenResult}; use alloc::boxed::Box; -use regalloc2::PRegSet; use smallvec::{smallvec, SmallVec}; use std::fmt::{self, Write}; use std::string::{String, ToString}; @@ -33,26 +32,12 @@ use args::*; // `Inst` is defined inside ISLE as `MInst`. We publicly re-export it here. pub use super::lower::isle::generated_code::MInst as Inst; -/// Out-of-line data for calls, to keep the size of `Inst` down. -#[derive(Clone, Debug)] -pub struct CallInfo { - /// Register uses of this call. - pub uses: CallArgList, - /// Register defs of this call. - pub defs: CallRetList, - /// Registers clobbered by this call, as per its calling convention. - pub clobbers: PRegSet, - /// The number of bytes that the callee will pop from the stack for the - /// caller, if any. (Used for popping stack arguments with the `tail` - /// calling convention.) - pub callee_pop_size: u32, - /// The calling convention of the callee. - pub callee_conv: CallConv, -} - /// Out-of-line data for return-calls, to keep the size of `Inst` down. #[derive(Clone, Debug)] -pub struct ReturnCallInfo { +pub struct ReturnCallInfo { + /// Where this call is going. + pub dest: T, + /// The size of the argument area for this return-call, potentially smaller than that of the /// caller, but never larger. pub new_stack_arg_size: u32, @@ -538,45 +523,13 @@ impl Inst { Inst::Pop64 { dst } } - pub(crate) fn call_known( - dest: ExternalName, - uses: CallArgList, - defs: CallRetList, - clobbers: PRegSet, - callee_pop_size: u32, - callee_conv: CallConv, - ) -> Inst { - Inst::CallKnown { - dest, - info: Some(Box::new(CallInfo { - uses, - defs, - clobbers, - callee_pop_size, - callee_conv, - })), - } + pub(crate) fn call_known(info: Box>) -> Inst { + Inst::CallKnown { info } } - pub(crate) fn call_unknown( - dest: RegMem, - uses: CallArgList, - defs: CallRetList, - clobbers: PRegSet, - callee_pop_size: u32, - callee_conv: CallConv, - ) -> Inst { - dest.assert_regclass_is(RegClass::Int); - Inst::CallUnknown { - dest, - info: Some(Box::new(CallInfo { - uses, - defs, - clobbers, - callee_pop_size, - callee_conv, - })), - } + pub(crate) fn call_unknown(info: Box>) -> Inst { + info.dest.assert_regclass_is(RegClass::Int); + Inst::CallUnknown { info } } pub(crate) fn ret(stack_bytes_to_pop: u32) -> Inst { @@ -1660,26 +1613,26 @@ impl PrettyPrint for Inst { format!("{op} {dst}") } - Inst::CallKnown { dest, .. } => { + Inst::CallKnown { info } => { let op = ljustify("call".to_string()); - format!("{op} {dest:?}") + format!("{op} {:?}", info.dest) } - Inst::CallUnknown { dest, .. } => { - let dest = dest.pretty_print(8); + Inst::CallUnknown { info } => { + let dest = info.dest.pretty_print(8); let op = ljustify("call".to_string()); format!("{op} *{dest}") } - Inst::ReturnCallKnown { callee, info } => { + Inst::ReturnCallKnown { info } => { let ReturnCallInfo { uses, new_stack_arg_size, tmp, + dest, } = &**info; let tmp = pretty_print_reg(tmp.to_reg().to_reg(), 8); - let mut s = - format!("return_call_known {callee:?} ({new_stack_arg_size}) tmp={tmp}"); + let mut s = format!("return_call_known {dest:?} ({new_stack_arg_size}) tmp={tmp}"); for ret in uses { let preg = regs::show_reg(ret.preg); let vreg = pretty_print_reg(ret.vreg, 8); @@ -1688,13 +1641,14 @@ impl PrettyPrint for Inst { s } - Inst::ReturnCallUnknown { callee, info } => { + Inst::ReturnCallUnknown { info } => { let ReturnCallInfo { uses, new_stack_arg_size, tmp, + dest, } = &**info; - let callee = pretty_print_reg(*callee, 8); + let callee = pretty_print_reg(*dest, 8); let tmp = pretty_print_reg(tmp.to_reg().to_reg(), 8); let mut s = format!("return_call_unknown {callee} ({new_stack_arg_size}) tmp={tmp}"); @@ -2340,7 +2294,7 @@ fn x64_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) { collector.reg_early_def(tmp); } - Inst::CallKnown { dest, info, .. } => { + Inst::CallKnown { info } => { // Probestack is special and is only inserted after // regalloc, so we do not need to represent its ABI to the // register allocator. Assert that we don't alter that @@ -2349,8 +2303,9 @@ fn x64_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) { uses, defs, clobbers, + dest, .. - } = &mut **info.as_mut().expect("CallInfo is expected in this path"); + } = &mut **info; debug_assert_ne!(*dest, ExternalName::LibCall(LibCall::Probestack)); for CallArgPair { vreg, preg } in uses { collector.reg_fixed_use(vreg, *preg); @@ -2361,14 +2316,15 @@ fn x64_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) { collector.reg_clobbers(*clobbers); } - Inst::CallUnknown { info, dest, .. } => { + Inst::CallUnknown { info } => { let CallInfo { uses, defs, clobbers, callee_conv, + dest, .. - } = &mut **info.as_mut().expect("CallInfo is expected in this path"); + } = &mut **info; match dest { RegMem::Reg { reg } if *callee_conv == CallConv::Winch => { // TODO(https://github.com/bytecodealliance/regalloc2/issues/145): @@ -2408,25 +2364,29 @@ fn x64_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) { collector.reg_clobbers(clobbers); } - Inst::ReturnCallKnown { callee, info } => { - let ReturnCallInfo { uses, tmp, .. } = &mut **info; + Inst::ReturnCallKnown { info } => { + let ReturnCallInfo { + dest, uses, tmp, .. + } = &mut **info; collector.reg_fixed_def(tmp, regs::r11()); // Same as in the `Inst::CallKnown` branch. - debug_assert_ne!(*callee, ExternalName::LibCall(LibCall::Probestack)); + debug_assert_ne!(*dest, ExternalName::LibCall(LibCall::Probestack)); for CallArgPair { vreg, preg } in uses { collector.reg_fixed_use(vreg, *preg); } } - Inst::ReturnCallUnknown { callee, info } => { - let ReturnCallInfo { uses, tmp, .. } = &mut **info; + Inst::ReturnCallUnknown { info } => { + let ReturnCallInfo { + dest, uses, tmp, .. + } = &mut **info; // TODO(https://github.com/bytecodealliance/regalloc2/issues/145): // This shouldn't be a fixed register constraint, but it's not clear how to // pick a register that won't be clobbered by the callee-save restore code // emitted with a return_call_indirect. r10 is caller-saved, so this should be // safe to use. - collector.reg_fixed_use(callee, regs::r10()); + collector.reg_fixed_use(dest, regs::r10()); collector.reg_fixed_def(tmp, regs::r11()); for CallArgPair { vreg, preg } in uses { diff --git a/cranelift/codegen/src/isa/x64/lower/isle.rs b/cranelift/codegen/src/isa/x64/lower/isle.rs index 54a5ad518519..2c49b73acae6 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle.rs +++ b/cranelift/codegen/src/isa/x64/lower/isle.rs @@ -19,10 +19,10 @@ use crate::{ }, isa::x64::{ abi::X64CallSite, - inst::{args::*, regs, CallInfo, ReturnCallInfo}, + inst::{args::*, regs, ReturnCallInfo}, }, machinst::{ - isle::*, ArgPair, InsnInput, InstOutput, IsTailCall, MachAtomicRmwOp, MachInst, + isle::*, ArgPair, CallInfo, InsnInput, InstOutput, IsTailCall, MachAtomicRmwOp, MachInst, VCodeConstant, VCodeConstantData, }, }; @@ -34,8 +34,10 @@ use std::boxed::Box; /// call instruction is also used by Winch to emit calls, but the /// `Box` field is not used, it's only used by Cranelift. By making it /// optional, we reduce the number of heap allocations in Winch. -type BoxCallInfo = Option>; -type BoxReturnCallInfo = Box; +type BoxCallInfo = Box>; +type BoxCallIndInfo = Box>; +type BoxReturnCallInfo = Box>; +type BoxReturnCallIndInfo = Box>; type VecArgPair = Vec; pub struct SinkableLoad { diff --git a/cranelift/codegen/src/isa/x64/mod.rs b/cranelift/codegen/src/isa/x64/mod.rs index 0b6371582b43..7545a6f14835 100644 --- a/cranelift/codegen/src/isa/x64/mod.rs +++ b/cranelift/codegen/src/isa/x64/mod.rs @@ -1,6 +1,6 @@ //! X86_64-bit Instruction Set Architecture. -pub use self::inst::{args, CallInfo, EmitInfo, EmitState, Inst}; +pub use self::inst::{args, EmitInfo, EmitState, Inst}; use super::{OwnedTargetIsa, TargetIsa}; use crate::dominator_tree::DominatorTree; diff --git a/cranelift/codegen/src/isa/x64/pcc.rs b/cranelift/codegen/src/isa/x64/pcc.rs index 402c16301432..813016165948 100644 --- a/cranelift/codegen/src/isa/x64/pcc.rs +++ b/cranelift/codegen/src/isa/x64/pcc.rs @@ -843,8 +843,14 @@ pub(crate) fn check( Inst::ReturnCallUnknown { .. } => Ok(()), - Inst::CallUnknown { ref dest, .. } - | Inst::JmpUnknown { + Inst::CallUnknown { ref info } => match <&RegMem>::from(&info.dest) { + RegMem::Mem { ref addr } => { + check_load(ctx, None, addr, vcode, I64, 64)?; + Ok(()) + } + RegMem::Reg { .. } => Ok(()), + }, + Inst::JmpUnknown { target: ref dest, .. } => match <&RegMem>::from(dest) { RegMem::Mem { ref addr } => { diff --git a/cranelift/codegen/src/lib.rs b/cranelift/codegen/src/lib.rs index 1c78ac9a3752..475014e0f550 100644 --- a/cranelift/codegen/src/lib.rs +++ b/cranelift/codegen/src/lib.rs @@ -60,7 +60,7 @@ pub use crate::machinst::buffer::{ MachTrap, OpenPatchRegion, PatchRegion, }; pub use crate::machinst::{ - CompiledCode, Final, MachBuffer, MachBufferFinalized, MachInst, MachInstEmit, + CallInfo, CompiledCode, Final, MachBuffer, MachBufferFinalized, MachInst, MachInstEmit, MachInstEmitState, MachLabel, RealReg, Reg, RelocDistance, TextSectionBuilder, VCodeConstantData, VCodeConstants, Writable, }; diff --git a/cranelift/codegen/src/machinst/abi.rs b/cranelift/codegen/src/machinst/abi.rs index 7760aef208dd..475f7c466f91 100644 --- a/cranelift/codegen/src/machinst/abi.rs +++ b/cranelift/codegen/src/machinst/abi.rs @@ -541,16 +541,7 @@ pub trait ABIMachineSpec { /// Generate a call instruction/sequence. This method is provided one /// temporary register to use to synthesize the called address, if needed. - fn gen_call( - dest: &CallDest, - uses: CallArgList, - defs: CallRetList, - clobbers: PRegSet, - tmp: Writable, - callee_conv: isa::CallConv, - caller_conv: isa::CallConv, - callee_pop_size: u32, - ) -> SmallVec<[Self::I; 2]>; + fn gen_call(dest: &CallDest, tmp: Writable, info: CallInfo<()>) -> SmallVec<[Self::I; 2]>; /// Generate a memcpy invocation. Used to set up struct /// args. Takes `src`, `dst` as read-only inputs and passes a temporary @@ -588,6 +579,56 @@ pub trait ABIMachineSpec { ) -> ir::ArgumentExtension; } +/// Out-of-line data for calls, to keep the size of `Inst` down. +#[derive(Clone, Debug)] +pub struct CallInfo { + /// Receiver of this call + pub dest: T, + /// Register uses of this call. + pub uses: CallArgList, + /// Register defs of this call. + pub defs: CallRetList, + /// Registers clobbered by this call, as per its calling convention. + pub clobbers: PRegSet, + /// The calling convention of the callee. + pub callee_conv: isa::CallConv, + /// The calling convention of the caller. + pub caller_conv: isa::CallConv, + /// The number of bytes that the callee will pop from the stack for the + /// caller, if any. (Used for popping stack arguments with the `tail` + /// calling convention.) + pub callee_pop_size: u32, +} + +impl CallInfo { + /// Creates an empty set of info with no clobbers/uses/etc with the + /// specified ABI + pub fn empty(dest: T, call_conv: isa::CallConv) -> CallInfo { + CallInfo { + dest, + uses: smallvec![], + defs: smallvec![], + clobbers: PRegSet::empty(), + caller_conv: call_conv, + callee_conv: call_conv, + callee_pop_size: 0, + } + } + + /// Change the `T` payload on this info to `U`. + pub fn map(self, f: impl FnOnce(T) -> U) -> CallInfo { + CallInfo { + dest: f(self.dest), + uses: self.uses, + defs: self.defs, + clobbers: self.clobbers, + caller_conv: self.caller_conv, + callee_conv: self.callee_conv, + callee_pop_size: self.callee_pop_size, + } + } +} + /// The id of an ABI signature within the `SigSet`. #[derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] pub struct Sig(u32); @@ -2345,13 +2386,16 @@ impl CallSite { // references to SP offsets.) for inst in M::gen_call( &self.dest, - uses, - defs, - clobbers, tmp, - call_conv, - self.caller_conv, - callee_pop_size, + CallInfo { + dest: (), + uses, + defs, + clobbers, + callee_conv: call_conv, + caller_conv: self.caller_conv, + callee_pop_size, + }, ) .into_iter() { diff --git a/cranelift/filetests/filetests/isa/pulley32/call.clif b/cranelift/filetests/filetests/isa/pulley32/call.clif index d02583907c31..a40254d3a0e3 100644 --- a/cranelift/filetests/filetests/isa/pulley32/call.clif +++ b/cranelift/filetests/filetests/isa/pulley32/call.clif @@ -19,7 +19,7 @@ block0: ; x29 = xmov x27 ; block0: ; x0 = xconst8 0 -; call TestCase(%g), CallInfo { uses: [CallArgPair { vreg: p0i, preg: p0i }], defs: [CallRetPair { vreg: Writable { reg: p0i }, preg: p0i }], clobbers: PRegSet { bits: [65534, 65279, 4294967295, 0] }, callee_pop_size: 0 } +; call CallInfo { dest: TestCase(%g), uses: [CallArgPair { vreg: p0i, preg: p0i }], defs: [CallRetPair { vreg: Writable { reg: p0i }, preg: p0i }], clobbers: PRegSet { bits: [65534, 65279, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0 } ; x0 = xconst8 1 ; x28 = load64_u sp+8 // flags = notrap aligned ; x29 = load64_u sp+0 // flags = notrap aligned @@ -60,7 +60,7 @@ block0: ; x29 = xmov x27 ; block0: ; x0 = xconst8 0 -; call TestCase(%g), CallInfo { uses: [CallArgPair { vreg: p0i, preg: p0i }], defs: [CallRetPair { vreg: Writable { reg: p0i }, preg: p0i }], clobbers: PRegSet { bits: [65534, 65279, 4294967295, 0] }, callee_pop_size: 0 } +; call CallInfo { dest: TestCase(%g), uses: [CallArgPair { vreg: p0i, preg: p0i }], defs: [CallRetPair { vreg: Writable { reg: p0i }, preg: p0i }], clobbers: PRegSet { bits: [65534, 65279, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0 } ; x0 = xconst8 1 ; x28 = load64_u sp+8 // flags = notrap aligned ; x29 = load64_u sp+0 // flags = notrap aligned @@ -106,7 +106,7 @@ block0: ; x1 = xconst8 1 ; x2 = xconst8 2 ; x3 = xconst8 3 -; call TestCase(%g), CallInfo { uses: [CallArgPair { vreg: p0i, preg: p0i }, CallArgPair { vreg: p1i, preg: p1i }, CallArgPair { vreg: p2i, preg: p2i }, CallArgPair { vreg: p3i, preg: p3i }], defs: [], clobbers: PRegSet { bits: [65535, 65279, 4294967295, 0] }, callee_pop_size: 0 } +; call CallInfo { dest: TestCase(%g), uses: [CallArgPair { vreg: p0i, preg: p0i }, CallArgPair { vreg: p1i, preg: p1i }, CallArgPair { vreg: p2i, preg: p2i }, CallArgPair { vreg: p3i, preg: p3i }], defs: [], clobbers: PRegSet { bits: [65535, 65279, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0 } ; x28 = load64_u sp+8 // flags = notrap aligned ; x29 = load64_u sp+0 // flags = notrap aligned ; x30 = xconst8 16 @@ -148,7 +148,7 @@ block0: ; store64 sp+0, x29 // flags = notrap aligned ; x29 = xmov x27 ; block0: -; call TestCase(%g), CallInfo { uses: [], defs: [CallRetPair { vreg: Writable { reg: p0i }, preg: p0i }, CallRetPair { vreg: Writable { reg: p1i }, preg: p1i }, CallRetPair { vreg: Writable { reg: p2i }, preg: p2i }, CallRetPair { vreg: Writable { reg: p3i }, preg: p3i }], clobbers: PRegSet { bits: [65520, 65279, 4294967295, 0] }, callee_pop_size: 0 } +; call CallInfo { dest: TestCase(%g), uses: [], defs: [CallRetPair { vreg: Writable { reg: p0i }, preg: p0i }, CallRetPair { vreg: Writable { reg: p1i }, preg: p1i }, CallRetPair { vreg: Writable { reg: p2i }, preg: p2i }, CallRetPair { vreg: Writable { reg: p3i }, preg: p3i }], clobbers: PRegSet { bits: [65520, 65279, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0 } ; x4 = xadd64 x0, x2 ; x3 = xadd64 x1, x3 ; x0 = xadd64 x4, x3 @@ -214,7 +214,7 @@ block0: ; x12 = xmov x15 ; x13 = xmov x15 ; x14 = xmov x15 -; call TestCase(%g), CallInfo { uses: [CallArgPair { vreg: p0i, preg: p0i }, CallArgPair { vreg: p1i, preg: p1i }, CallArgPair { vreg: p2i, preg: p2i }, CallArgPair { vreg: p3i, preg: p3i }, CallArgPair { vreg: p4i, preg: p4i }, CallArgPair { vreg: p5i, preg: p5i }, CallArgPair { vreg: p6i, preg: p6i }, CallArgPair { vreg: p7i, preg: p7i }, CallArgPair { vreg: p8i, preg: p8i }, CallArgPair { vreg: p9i, preg: p9i }, CallArgPair { vreg: p10i, preg: p10i }, CallArgPair { vreg: p11i, preg: p11i }, CallArgPair { vreg: p12i, preg: p12i }, CallArgPair { vreg: p13i, preg: p13i }, CallArgPair { vreg: p14i, preg: p14i }, CallArgPair { vreg: p15i, preg: p15i }], defs: [], clobbers: PRegSet { bits: [65535, 65279, 4294967295, 0] }, callee_pop_size: 0 } +; call CallInfo { dest: TestCase(%g), uses: [CallArgPair { vreg: p0i, preg: p0i }, CallArgPair { vreg: p1i, preg: p1i }, CallArgPair { vreg: p2i, preg: p2i }, CallArgPair { vreg: p3i, preg: p3i }, CallArgPair { vreg: p4i, preg: p4i }, CallArgPair { vreg: p5i, preg: p5i }, CallArgPair { vreg: p6i, preg: p6i }, CallArgPair { vreg: p7i, preg: p7i }, CallArgPair { vreg: p8i, preg: p8i }, CallArgPair { vreg: p9i, preg: p9i }, CallArgPair { vreg: p10i, preg: p10i }, CallArgPair { vreg: p11i, preg: p11i }, CallArgPair { vreg: p12i, preg: p12i }, CallArgPair { vreg: p13i, preg: p13i }, CallArgPair { vreg: p14i, preg: p14i }, CallArgPair { vreg: p15i, preg: p15i }], defs: [], clobbers: PRegSet { bits: [65535, 65279, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0 } ; x30 = xconst8 48 ; x27 = xadd32 x27, x30 ; x28 = load64_u sp+8 // flags = notrap aligned @@ -310,7 +310,7 @@ block0: ; store64 sp+48, x18 // flags = notrap aligned ; block0: ; x0 = load_addr OutgoingArg(0) -; call TestCase(%g), CallInfo { uses: [CallArgPair { vreg: p0i, preg: p0i }], defs: [CallRetPair { vreg: Writable { reg: p0i }, preg: p0i }, CallRetPair { vreg: Writable { reg: p1i }, preg: p1i }, CallRetPair { vreg: Writable { reg: p2i }, preg: p2i }, CallRetPair { vreg: Writable { reg: p3i }, preg: p3i }, CallRetPair { vreg: Writable { reg: p4i }, preg: p4i }, CallRetPair { vreg: Writable { reg: p5i }, preg: p5i }, CallRetPair { vreg: Writable { reg: p6i }, preg: p6i }, CallRetPair { vreg: Writable { reg: p7i }, preg: p7i }, CallRetPair { vreg: Writable { reg: p8i }, preg: p8i }, CallRetPair { vreg: Writable { reg: p9i }, preg: p9i }, CallRetPair { vreg: Writable { reg: p10i }, preg: p10i }, CallRetPair { vreg: Writable { reg: p11i }, preg: p11i }, CallRetPair { vreg: Writable { reg: p12i }, preg: p12i }, CallRetPair { vreg: Writable { reg: p13i }, preg: p13i }, CallRetPair { vreg: Writable { reg: p14i }, preg: p14i }, CallRetPair { vreg: Writable { reg: p15i }, preg: p15i }], clobbers: PRegSet { bits: [0, 65279, 4294967295, 0] }, callee_pop_size: 0 } +; call CallInfo { dest: TestCase(%g), uses: [CallArgPair { vreg: p0i, preg: p0i }], defs: [CallRetPair { vreg: Writable { reg: p0i }, preg: p0i }, CallRetPair { vreg: Writable { reg: p1i }, preg: p1i }, CallRetPair { vreg: Writable { reg: p2i }, preg: p2i }, CallRetPair { vreg: Writable { reg: p3i }, preg: p3i }, CallRetPair { vreg: Writable { reg: p4i }, preg: p4i }, CallRetPair { vreg: Writable { reg: p5i }, preg: p5i }, CallRetPair { vreg: Writable { reg: p6i }, preg: p6i }, CallRetPair { vreg: Writable { reg: p7i }, preg: p7i }, CallRetPair { vreg: Writable { reg: p8i }, preg: p8i }, CallRetPair { vreg: Writable { reg: p9i }, preg: p9i }, CallRetPair { vreg: Writable { reg: p10i }, preg: p10i }, CallRetPair { vreg: Writable { reg: p11i }, preg: p11i }, CallRetPair { vreg: Writable { reg: p12i }, preg: p12i }, CallRetPair { vreg: Writable { reg: p13i }, preg: p13i }, CallRetPair { vreg: Writable { reg: p14i }, preg: p14i }, CallRetPair { vreg: Writable { reg: p15i }, preg: p15i }], clobbers: PRegSet { bits: [0, 65279, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0 } ; x16 = xmov x13 ; x18 = xmov x11 ; x25 = load64_u OutgoingArg(0) // flags = notrap aligned diff --git a/cranelift/filetests/filetests/isa/pulley64/call.clif b/cranelift/filetests/filetests/isa/pulley64/call.clif index daefab228804..2a031f18219b 100644 --- a/cranelift/filetests/filetests/isa/pulley64/call.clif +++ b/cranelift/filetests/filetests/isa/pulley64/call.clif @@ -19,7 +19,7 @@ block0: ; x29 = xmov x27 ; block0: ; x0 = xconst8 0 -; call TestCase(%g), CallInfo { uses: [CallArgPair { vreg: p0i, preg: p0i }], defs: [CallRetPair { vreg: Writable { reg: p0i }, preg: p0i }], clobbers: PRegSet { bits: [65534, 65279, 4294967295, 0] }, callee_pop_size: 0 } +; call CallInfo { dest: TestCase(%g), uses: [CallArgPair { vreg: p0i, preg: p0i }], defs: [CallRetPair { vreg: Writable { reg: p0i }, preg: p0i }], clobbers: PRegSet { bits: [65534, 65279, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0 } ; x0 = xconst8 1 ; x28 = load64_u sp+8 // flags = notrap aligned ; x29 = load64_u sp+0 // flags = notrap aligned @@ -60,7 +60,7 @@ block0: ; x29 = xmov x27 ; block0: ; x0 = xconst8 0 -; call TestCase(%g), CallInfo { uses: [CallArgPair { vreg: p0i, preg: p0i }], defs: [CallRetPair { vreg: Writable { reg: p0i }, preg: p0i }], clobbers: PRegSet { bits: [65534, 65279, 4294967295, 0] }, callee_pop_size: 0 } +; call CallInfo { dest: TestCase(%g), uses: [CallArgPair { vreg: p0i, preg: p0i }], defs: [CallRetPair { vreg: Writable { reg: p0i }, preg: p0i }], clobbers: PRegSet { bits: [65534, 65279, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0 } ; x0 = xconst8 1 ; x28 = load64_u sp+8 // flags = notrap aligned ; x29 = load64_u sp+0 // flags = notrap aligned @@ -106,7 +106,7 @@ block0: ; x1 = xconst8 1 ; x2 = xconst8 2 ; x3 = xconst8 3 -; call TestCase(%g), CallInfo { uses: [CallArgPair { vreg: p0i, preg: p0i }, CallArgPair { vreg: p1i, preg: p1i }, CallArgPair { vreg: p2i, preg: p2i }, CallArgPair { vreg: p3i, preg: p3i }], defs: [], clobbers: PRegSet { bits: [65535, 65279, 4294967295, 0] }, callee_pop_size: 0 } +; call CallInfo { dest: TestCase(%g), uses: [CallArgPair { vreg: p0i, preg: p0i }, CallArgPair { vreg: p1i, preg: p1i }, CallArgPair { vreg: p2i, preg: p2i }, CallArgPair { vreg: p3i, preg: p3i }], defs: [], clobbers: PRegSet { bits: [65535, 65279, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0 } ; x28 = load64_u sp+8 // flags = notrap aligned ; x29 = load64_u sp+0 // flags = notrap aligned ; x30 = xconst8 16 @@ -148,7 +148,7 @@ block0: ; store64 sp+0, x29 // flags = notrap aligned ; x29 = xmov x27 ; block0: -; call TestCase(%g), CallInfo { uses: [], defs: [CallRetPair { vreg: Writable { reg: p0i }, preg: p0i }, CallRetPair { vreg: Writable { reg: p1i }, preg: p1i }, CallRetPair { vreg: Writable { reg: p2i }, preg: p2i }, CallRetPair { vreg: Writable { reg: p3i }, preg: p3i }], clobbers: PRegSet { bits: [65520, 65279, 4294967295, 0] }, callee_pop_size: 0 } +; call CallInfo { dest: TestCase(%g), uses: [], defs: [CallRetPair { vreg: Writable { reg: p0i }, preg: p0i }, CallRetPair { vreg: Writable { reg: p1i }, preg: p1i }, CallRetPair { vreg: Writable { reg: p2i }, preg: p2i }, CallRetPair { vreg: Writable { reg: p3i }, preg: p3i }], clobbers: PRegSet { bits: [65520, 65279, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0 } ; x4 = xadd64 x0, x2 ; x3 = xadd64 x1, x3 ; x0 = xadd64 x4, x3 @@ -214,7 +214,7 @@ block0: ; x12 = xmov x15 ; x13 = xmov x15 ; x14 = xmov x15 -; call TestCase(%g), CallInfo { uses: [CallArgPair { vreg: p0i, preg: p0i }, CallArgPair { vreg: p1i, preg: p1i }, CallArgPair { vreg: p2i, preg: p2i }, CallArgPair { vreg: p3i, preg: p3i }, CallArgPair { vreg: p4i, preg: p4i }, CallArgPair { vreg: p5i, preg: p5i }, CallArgPair { vreg: p6i, preg: p6i }, CallArgPair { vreg: p7i, preg: p7i }, CallArgPair { vreg: p8i, preg: p8i }, CallArgPair { vreg: p9i, preg: p9i }, CallArgPair { vreg: p10i, preg: p10i }, CallArgPair { vreg: p11i, preg: p11i }, CallArgPair { vreg: p12i, preg: p12i }, CallArgPair { vreg: p13i, preg: p13i }, CallArgPair { vreg: p14i, preg: p14i }, CallArgPair { vreg: p15i, preg: p15i }], defs: [], clobbers: PRegSet { bits: [65535, 65279, 4294967295, 0] }, callee_pop_size: 0 } +; call CallInfo { dest: TestCase(%g), uses: [CallArgPair { vreg: p0i, preg: p0i }, CallArgPair { vreg: p1i, preg: p1i }, CallArgPair { vreg: p2i, preg: p2i }, CallArgPair { vreg: p3i, preg: p3i }, CallArgPair { vreg: p4i, preg: p4i }, CallArgPair { vreg: p5i, preg: p5i }, CallArgPair { vreg: p6i, preg: p6i }, CallArgPair { vreg: p7i, preg: p7i }, CallArgPair { vreg: p8i, preg: p8i }, CallArgPair { vreg: p9i, preg: p9i }, CallArgPair { vreg: p10i, preg: p10i }, CallArgPair { vreg: p11i, preg: p11i }, CallArgPair { vreg: p12i, preg: p12i }, CallArgPair { vreg: p13i, preg: p13i }, CallArgPair { vreg: p14i, preg: p14i }, CallArgPair { vreg: p15i, preg: p15i }], defs: [], clobbers: PRegSet { bits: [65535, 65279, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0 } ; x30 = xconst8 48 ; x27 = xadd32 x27, x30 ; x28 = load64_u sp+8 // flags = notrap aligned @@ -310,7 +310,7 @@ block0: ; store64 sp+48, x18 // flags = notrap aligned ; block0: ; x0 = load_addr OutgoingArg(0) -; call TestCase(%g), CallInfo { uses: [CallArgPair { vreg: p0i, preg: p0i }], defs: [CallRetPair { vreg: Writable { reg: p0i }, preg: p0i }, CallRetPair { vreg: Writable { reg: p1i }, preg: p1i }, CallRetPair { vreg: Writable { reg: p2i }, preg: p2i }, CallRetPair { vreg: Writable { reg: p3i }, preg: p3i }, CallRetPair { vreg: Writable { reg: p4i }, preg: p4i }, CallRetPair { vreg: Writable { reg: p5i }, preg: p5i }, CallRetPair { vreg: Writable { reg: p6i }, preg: p6i }, CallRetPair { vreg: Writable { reg: p7i }, preg: p7i }, CallRetPair { vreg: Writable { reg: p8i }, preg: p8i }, CallRetPair { vreg: Writable { reg: p9i }, preg: p9i }, CallRetPair { vreg: Writable { reg: p10i }, preg: p10i }, CallRetPair { vreg: Writable { reg: p11i }, preg: p11i }, CallRetPair { vreg: Writable { reg: p12i }, preg: p12i }, CallRetPair { vreg: Writable { reg: p13i }, preg: p13i }, CallRetPair { vreg: Writable { reg: p14i }, preg: p14i }, CallRetPair { vreg: Writable { reg: p15i }, preg: p15i }], clobbers: PRegSet { bits: [0, 65279, 4294967295, 0] }, callee_pop_size: 0 } +; call CallInfo { dest: TestCase(%g), uses: [CallArgPair { vreg: p0i, preg: p0i }], defs: [CallRetPair { vreg: Writable { reg: p0i }, preg: p0i }, CallRetPair { vreg: Writable { reg: p1i }, preg: p1i }, CallRetPair { vreg: Writable { reg: p2i }, preg: p2i }, CallRetPair { vreg: Writable { reg: p3i }, preg: p3i }, CallRetPair { vreg: Writable { reg: p4i }, preg: p4i }, CallRetPair { vreg: Writable { reg: p5i }, preg: p5i }, CallRetPair { vreg: Writable { reg: p6i }, preg: p6i }, CallRetPair { vreg: Writable { reg: p7i }, preg: p7i }, CallRetPair { vreg: Writable { reg: p8i }, preg: p8i }, CallRetPair { vreg: Writable { reg: p9i }, preg: p9i }, CallRetPair { vreg: Writable { reg: p10i }, preg: p10i }, CallRetPair { vreg: Writable { reg: p11i }, preg: p11i }, CallRetPair { vreg: Writable { reg: p12i }, preg: p12i }, CallRetPair { vreg: Writable { reg: p13i }, preg: p13i }, CallRetPair { vreg: Writable { reg: p14i }, preg: p14i }, CallRetPair { vreg: Writable { reg: p15i }, preg: p15i }], clobbers: PRegSet { bits: [0, 65279, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0 } ; x16 = xmov x13 ; x18 = xmov x11 ; x25 = load64_u OutgoingArg(0) // flags = notrap aligned diff --git a/cranelift/filetests/filetests/isa/s390x/tls_elf.clif b/cranelift/filetests/filetests/isa/s390x/tls_elf.clif index 250f1c50c1c7..b31136506915 100644 --- a/cranelift/filetests/filetests/isa/s390x/tls_elf.clif +++ b/cranelift/filetests/filetests/isa/s390x/tls_elf.clif @@ -16,7 +16,7 @@ block0(v0: i32): ; block0: ; larl %r12, %ElfGlobalOffsetTable + 0 ; bras %r1, 12 ; data userextname0@tlsgd ; lg %r2, 0(%r1) -; brasl %r14, %ElfTlsGetOffset:tls_gdcall:userextname0 +; brasl %r14, tls_gdcall:userextname0 ; ear %r4, %a0 ; sllg %r3, %r4, 32 ; ear %r3, %a1 diff --git a/winch/codegen/src/isa/x64/asm.rs b/winch/codegen/src/isa/x64/asm.rs index 79eab2a8078b..b48cb5e0dd09 100644 --- a/winch/codegen/src/isa/x64/asm.rs +++ b/winch/codegen/src/isa/x64/asm.rs @@ -21,9 +21,10 @@ use cranelift_codegen::{ encoding::rex::{encode_modrm, RexFlags}, settings as x64_settings, EmitInfo, EmitState, Inst, }, + CallConv, }, - settings, Final, MachBuffer, MachBufferFinalized, MachInstEmit, MachInstEmitState, MachLabel, - PatchRegion, RelocDistance, VCodeConstantData, VCodeConstants, Writable, + settings, CallInfo, Final, MachBuffer, MachBufferFinalized, MachInstEmit, MachInstEmitState, + MachLabel, PatchRegion, RelocDistance, VCodeConstantData, VCodeConstants, Writable, }; use super::address::Address; @@ -1250,15 +1251,18 @@ impl Assembler { /// Emit a call to an unknown location through a register. pub fn call_with_reg(&mut self, callee: Reg) { self.emit(Inst::CallUnknown { - dest: RegMem::reg(callee.into()), - info: None, + info: Box::new(CallInfo::empty( + RegMem::reg(callee.into()), + CallConv::SystemV, + )), }); } /// Emit a call to a locally defined function through an index. pub fn call_with_name(&mut self, name: UserExternalNameRef) { - let dest = ExternalName::user(name); - self.emit(Inst::CallKnown { dest, info: None }); + self.emit(Inst::CallKnown { + info: Box::new(CallInfo::empty(ExternalName::user(name), CallConv::SystemV)), + }); } /// Emit a call to a well-known libcall.