From 067e404f1a87f09985d07a47538263438a99bff1 Mon Sep 17 00:00:00 2001 From: "Levytskyy, Vyacheslav" Date: Mon, 16 Sep 2024 15:00:55 -0700 Subject: [PATCH 01/16] allow intrinsics with aggregate return type to reach GlobalISel --- llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp | 48 ++++++++++++++++++- llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp | 3 +- llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp | 8 +++- .../Target/SPIRV/SPIRVPrepareFunctions.cpp | 19 ++++++++ 4 files changed, 74 insertions(+), 4 deletions(-) diff --git a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp index 795ddf47c40dab..6da132dcf7e20c 100644 --- a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp @@ -188,6 +188,40 @@ bool isConvergenceIntrinsic(const Instruction *I) { II->getIntrinsicID() == Intrinsic::experimental_convergence_loop || II->getIntrinsicID() == Intrinsic::experimental_convergence_anchor; } + +bool isInternalNonVoidIntrinsic(const Value *I) { + if (const auto *II = dyn_cast(I)) + switch (II->getIntrinsicID()) { + case Intrinsic::spv_cmpxchg: + case Intrinsic::spv_const_composite: + case Intrinsic::spv_track_constant: + case Intrinsic::spv_load: + case Intrinsic::spv_extractv: + case Intrinsic::spv_insertv: + case Intrinsic::spv_extractelt: + case Intrinsic::spv_insertelt: + case Intrinsic::spv_bitcast: + case Intrinsic::spv_ptrcast: + case Intrinsic::spv_alloca: + case Intrinsic::spv_alloca_array: + case Intrinsic::spv_undef: + return true; + } + return false; +} + +bool allowEmitFakeUse(const Value *Arg) { + if (isInternalNonVoidIntrinsic(Arg)) + return false; + if (dyn_cast(Arg) || dyn_cast(Arg) || + dyn_cast(Arg)) + return false; + if (const auto *LI = dyn_cast(Arg)) + if (LI->getType()->isAggregateType()) + return false; + return true; +} + } // namespace char SPIRVEmitIntrinsics::ID = 0; @@ -283,8 +317,16 @@ static inline Type *reconstructType(SPIRVGlobalRegistry *GR, Value *Op) { void SPIRVEmitIntrinsics::buildAssignType(IRBuilder<> &B, Type *Ty, Value *Arg) { Value *OfType = PoisonValue::get(Ty); - CallInst *AssignCI = buildIntrWithMD(Intrinsic::spv_assign_type, - {Arg->getType()}, OfType, Arg, {}, B); + CallInst *AssignCI = nullptr; + if (Ty->isAggregateType() && allowEmitFakeUse(Arg)) { + AssignCI = B.CreateIntrinsic(Intrinsic::fake_use, {}, {Arg}); + AssignCI->setMetadata( + "spirv.__BE.assign_type", + MDNode::get(Arg->getContext(), ValueAsMetadata::getConstant(OfType))); + } else { + AssignCI = buildIntrWithMD(Intrinsic::spv_assign_type, {Arg->getType()}, + OfType, Arg, {}, B); + } GR->addAssignPtrTypeInstr(Arg, AssignCI); } @@ -1268,6 +1310,8 @@ Instruction *SPIRVEmitIntrinsics::visitInsertValueInst(InsertValueInst &I) { } Instruction *SPIRVEmitIntrinsics::visitExtractValueInst(ExtractValueInst &I) { + if (I.getAggregateOperand()->getType()->isAggregateType()) + return &I; IRBuilder<> B(I.getParent()); B.SetInsertPoint(&I); SmallVector Args; diff --git a/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp b/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp index 9fe4d8a16bc32a..189b84369d1c33 100644 --- a/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp @@ -287,7 +287,8 @@ SPIRVLegalizerInfo::SPIRVLegalizerInfo(const SPIRVSubtarget &ST) { // TODO: add proper legalization rules. getActionDefinitionsBuilder(G_ATOMIC_CMPXCHG).alwaysLegal(); - getActionDefinitionsBuilder({G_UADDO, G_USUBO, G_SMULO, G_UMULO}) + getActionDefinitionsBuilder( + {G_UADDO, G_SADDO, G_USUBO, G_SSUBO, G_SMULO, G_UMULO}) .alwaysLegal(); // FP conversions. diff --git a/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp b/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp index f1b10e264781f2..bd29e28920e692 100644 --- a/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp @@ -376,7 +376,13 @@ Register insertAssignInstr(Register Reg, Type *Ty, SPIRVType *SpvType, .addUse(NewReg) .addUse(GR->getSPIRVTypeID(SpvType)) .setMIFlags(Flags); - Def->getOperand(0).setReg(NewReg); + for (unsigned I = 0, E = Def->getNumExplicitDefs(); I != E; ++I) { + MachineOperand &MO = Def->getOperand(I); + if (MO.isReg() && MO.isDef() && MO.getReg() == Reg) { + MO.setReg(NewReg); + break; + } + } return NewReg; } diff --git a/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp b/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp index eb5139ac5383a9..5e311589103e50 100644 --- a/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp @@ -478,6 +478,24 @@ bool SPIRVPrepareFunctions::substituteIntrinsicCalls(Function *F) { // noted in 'spv.cloned_funcs' metadata for later restoration. Function * SPIRVPrepareFunctions::removeAggregateTypesFromSignature(Function *F) { + if (F->isIntrinsic()) + // Allow intrinsics with aggregate return type to reach GlobalISel + switch (F->getIntrinsicID()) { + // Standard C/C++ Library Intrinsics + case Intrinsic::frexp: + // Vector Reduction Intrinsics + case Intrinsic::vector_interleave2: + case Intrinsic::vector_deinterleave2: + // Arithmetic with Overflow Intrinsics + case Intrinsic::smul_with_overflow: + case Intrinsic::umul_with_overflow: + case Intrinsic::sadd_with_overflow: + case Intrinsic::uadd_with_overflow: + case Intrinsic::ssub_with_overflow: + case Intrinsic::usub_with_overflow: + return F; + } + IRBuilder<> B(F->getContext()); bool IsRetAggr = F->getReturnType()->isAggregateType(); @@ -563,6 +581,7 @@ bool SPIRVPrepareFunctions::runOnModule(Module &M) { Changed = true; } } + return Changed; } From 416eafee576c01430778e2dd484c6a6559418a6f Mon Sep 17 00:00:00 2001 From: "Levytskyy, Vyacheslav" Date: Tue, 17 Sep 2024 04:58:06 -0700 Subject: [PATCH 02/16] implement keeping and referencing aggregate result attributes between passes; implement selection of Arithmetic with Overflow Intrinsics --- llvm/include/llvm/IR/IntrinsicsSPIRV.td | 1 + llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp | 11 +-- llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h | 17 +++++ .../Target/SPIRV/SPIRVInstructionSelector.cpp | 70 +++++++++++++++++++ llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp | 2 +- llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp | 22 +++++- .../Target/SPIRV/SPIRVPrepareFunctions.cpp | 17 +++-- 7 files changed, 126 insertions(+), 14 deletions(-) diff --git a/llvm/include/llvm/IR/IntrinsicsSPIRV.td b/llvm/include/llvm/IR/IntrinsicsSPIRV.td index 7ac479f31386f9..c5c60963ed6fda 100644 --- a/llvm/include/llvm/IR/IntrinsicsSPIRV.td +++ b/llvm/include/llvm/IR/IntrinsicsSPIRV.td @@ -15,6 +15,7 @@ let TargetPrefix = "spv" in { def int_spv_assign_ptr_type : Intrinsic<[], [llvm_any_ty, llvm_metadata_ty, llvm_i32_ty], [ImmArg>]>; def int_spv_assign_name : Intrinsic<[], [llvm_any_ty, llvm_vararg_ty]>; def int_spv_assign_decoration : Intrinsic<[], [llvm_any_ty, llvm_metadata_ty]>; + def int_spv_value_md : Intrinsic<[], [llvm_metadata_ty]>; def int_spv_track_constant : Intrinsic<[llvm_any_ty], [llvm_any_ty, llvm_metadata_ty]>; def int_spv_init_global : Intrinsic<[], [llvm_any_ty, llvm_any_ty]>; diff --git a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp index 6da132dcf7e20c..73fb5c207a26cf 100644 --- a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp @@ -319,10 +319,13 @@ void SPIRVEmitIntrinsics::buildAssignType(IRBuilder<> &B, Type *Ty, Value *OfType = PoisonValue::get(Ty); CallInst *AssignCI = nullptr; if (Ty->isAggregateType() && allowEmitFakeUse(Arg)) { + LLVMContext &Ctx = Arg->getContext(); + SmallVector ArgMDs{ + MDNode::get(Ctx, ValueAsMetadata::getConstant(OfType)), + MDString::get(Ctx, Arg->getName())}; + B.CreateIntrinsic(Intrinsic::spv_value_md, {}, + {MetadataAsValue::get(Ctx, MDTuple::get(Ctx, ArgMDs))}); AssignCI = B.CreateIntrinsic(Intrinsic::fake_use, {}, {Arg}); - AssignCI->setMetadata( - "spirv.__BE.assign_type", - MDNode::get(Arg->getContext(), ValueAsMetadata::getConstant(OfType))); } else { AssignCI = buildIntrWithMD(Intrinsic::spv_assign_type, {Arg->getType()}, OfType, Arg, {}, B); @@ -1577,7 +1580,7 @@ void SPIRVEmitIntrinsics::processInstrAfterVisit(Instruction *I, I->setOperand(OpNo, NewOp); } } - if (I->hasName()) { + if (I->hasName() && !I->getType()->isAggregateType()) { reportFatalOnTokenType(I); setInsertPointAfterDef(B, I); std::vector Args = {I}; diff --git a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h index ed9cfc07132430..ba104a87064b2f 100644 --- a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h +++ b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h @@ -55,6 +55,8 @@ class SPIRVGlobalRegistry { // created during substitution of aggregate arguments // (see `SPIRVPrepareFunctions::removeAggregateTypesFromSignature()`) DenseMap MutatedAggRet; + // map an instruction to its value's attributes (type, name) + DenseMap> ValueAttrs; // Look for an equivalent of the newType in the map. Return the equivalent // if it's found, otherwise insert newType to the map and return the type. @@ -177,6 +179,21 @@ class SPIRVGlobalRegistry { return It == MutatedAggRet.end() ? nullptr : It->second; } + // A registry of value's attributes (type, name) + // - Add a record. + void addValueAttrs(MachineInstr *Key, std::pair Val) { + ValueAttrs[Key] = Val; + } + // - Find a record. + bool findValueAttrs(const MachineInstr *Key, Type *&Ty, StringRef &Name) { + auto It = ValueAttrs.find(Key); + if (It == ValueAttrs.end()) + return false; + Ty = It->second.first; + Name = It->second.second; + return true; + } + // Deduced element types of untyped pointers and composites: // - Add a record to the map of deduced element types. void addDeducedElementType(Value *Val, Type *Ty) { DeducedElTys[Val] = Ty; } diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp index 7af92b87ce00cf..05a63aba902a58 100644 --- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp @@ -191,6 +191,9 @@ class SPIRVInstructionSelector : public InstructionSelector { bool selectFloatDot(Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const; + bool selectOverflowArith(Register ResVReg, const SPIRVType *ResType, + MachineInstr &I, unsigned Opcode) const; + bool selectIntegerDot(Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const; @@ -590,6 +593,23 @@ bool SPIRVInstructionSelector::spvSelect(Register ResVReg, case TargetOpcode::G_USUBSAT: return selectExtInst(ResVReg, ResType, I, CL::u_sub_sat); + case TargetOpcode::G_SADDO: + report_fatal_error( + "Selection of a signed addition with overflow is not implemented", + false); + case TargetOpcode::G_UADDO: + return selectOverflowArith(ResVReg, ResType, I, SPIRV::OpIAddCarry); + case TargetOpcode::G_SSUBO: + report_fatal_error( + "Selection of a signed subtraction with overflow is not implemented", + false); + case TargetOpcode::G_USUBO: + return selectOverflowArith(ResVReg, ResType, I, SPIRV::OpISubBorrow); + case TargetOpcode::G_UMULO: + return selectOverflowArith(ResVReg, ResType, I, SPIRV::OpUMulExtended); + case TargetOpcode::G_SMULO: + return selectOverflowArith(ResVReg, ResType, I, SPIRV::OpSMulExtended); + case TargetOpcode::G_SEXT: return selectExt(ResVReg, ResType, I, true); case TargetOpcode::G_ANYEXT: @@ -1101,6 +1121,53 @@ bool SPIRVInstructionSelector::selectFence(MachineInstr &I) const { .constrainAllUses(TII, TRI, RBI); } +bool SPIRVInstructionSelector::selectOverflowArith(Register ResVReg, + const SPIRVType *ResType, + MachineInstr &I, + unsigned Opcode) const { + Type *ResTy = nullptr; + StringRef ResName; + if (!GR.findValueAttrs(&I, ResTy, ResName)) + report_fatal_error( + "Not enough info to select the arithmetic with overflow instruction"); + if (!ResTy->isStructTy()) + report_fatal_error("Expect struct type result for the arithmetic " + "with overflow instruction"); + // "Result Type must be from OpTypeStruct. The struct must have two members, + // and the two members must be the same type." + Type *ResElemTy = cast(ResTy)->getElementType(0); + ResTy = StructType::create(SmallVector{ResElemTy, ResElemTy}); + // Build SPIR-V type if needed. + MachineIRBuilder MIRBuilder(I); + SPIRVType *StructType = GR.getOrCreateSPIRVType( + ResTy, MIRBuilder, SPIRV::AccessQualifier::ReadWrite, false); + // A new virtual register to store the result struct. + Register StructVReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); + MRI->setRegClass(StructVReg, &SPIRV::IDRegClass); + // Build the result name of needed. + if (ResName.size() > 0) + buildOpName(StructVReg, ResName, MIRBuilder); + // Build the arithmetic with overflow instruction. + MachineBasicBlock &BB = *I.getParent(); + auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) + .addDef(StructVReg) + .addUse(GR.getSPIRVTypeID(StructType)); + for (unsigned i = I.getNumDefs(); i < I.getNumOperands(); ++i) + MIB.addUse(I.getOperand(i).getReg()); + bool Status = MIB.constrainAllUses(TII, TRI, RBI); + // Build instructions to extract fields of the instruction's result. + for (unsigned i = 0; i < I.getNumDefs(); ++i) { + auto MIB = + BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) + .addDef(I.getOperand(i).getReg()) + .addUse(GR.getSPIRVTypeID(ResType)) + .addUse(StructVReg) + .addImm(i); + Status &= MIB.constrainAllUses(TII, TRI, RBI); + } + return Status; +} + bool SPIRVInstructionSelector::selectAtomicCmpXchg(Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const { @@ -2505,6 +2572,9 @@ bool SPIRVInstructionSelector::selectIntrinsic(Register ResVReg, } case Intrinsic::spv_step: return selectStep(ResVReg, ResType, I); + case Intrinsic::spv_value_md: + // ignore the intrinsic + break; default: { std::string DiagMsg; raw_string_ostream OS(DiagMsg); diff --git a/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp b/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp index 189b84369d1c33..de9c495d4cbacc 100644 --- a/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp @@ -288,7 +288,7 @@ SPIRVLegalizerInfo::SPIRVLegalizerInfo(const SPIRVSubtarget &ST) { getActionDefinitionsBuilder(G_ATOMIC_CMPXCHG).alwaysLegal(); getActionDefinitionsBuilder( - {G_UADDO, G_SADDO, G_USUBO, G_SSUBO, G_SMULO, G_UMULO}) + {G_UADDO, G_SADDO, G_USUBO, G_SSUBO, G_UMULO, G_SMULO}) .alwaysLegal(); // FP conversions. diff --git a/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp b/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp index bd29e28920e692..a67b80605f62c7 100644 --- a/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp @@ -376,9 +376,9 @@ Register insertAssignInstr(Register Reg, Type *Ty, SPIRVType *SpvType, .addUse(NewReg) .addUse(GR->getSPIRVTypeID(SpvType)) .setMIFlags(Flags); - for (unsigned I = 0, E = Def->getNumExplicitDefs(); I != E; ++I) { + for (unsigned I = 0, E = Def->getNumDefs(); I != E; ++I) { MachineOperand &MO = Def->getOperand(I); - if (MO.isReg() && MO.isDef() && MO.getReg() == Reg) { + if (MO.getReg() == Reg) { MO.setReg(NewReg); break; } @@ -468,6 +468,24 @@ generateAssignInstrs(MachineFunction &MF, SPIRVGlobalRegistry *GR, Def->getOpcode() != SPIRV::ASSIGN_TYPE) insertAssignInstr(Reg, Ty, nullptr, GR, MIB, MF.getRegInfo()); ToErase.push_back(&MI); + } else if (MIOp == TargetOpcode::FAKE_USE && MI.getNumOperands() > 0) { + MachineInstr *MdMI = MI.getPrevNode(); + if (MdMI && isSpvIntrinsic(*MdMI, Intrinsic::spv_value_md)) { + // It's an internal service info from before IRTranslator passes. + MachineInstr *Def = MRI.getVRegDef(MI.getOperand(0).getReg()); + for (unsigned I = 1, E = MI.getNumOperands(); I != E && Def; ++I) + if (MRI.getVRegDef(MI.getOperand(I).getReg()) != Def) + Def = nullptr; + if (Def) { + const MDNode *MD = MdMI->getOperand(1).getMetadata(); + Type *ValueTy = cast(MD->getOperand(0))->getType(); + StringRef ValueName = + cast(MD->getOperand(1))->getString(); + GR->addValueAttrs(Def, std::make_pair(ValueTy, ValueName)); + } + } + ToErase.push_back(&MI); + ToErase.push_back(MdMI); } else if (MIOp == TargetOpcode::G_CONSTANT || MIOp == TargetOpcode::G_FCONSTANT || MIOp == TargetOpcode::G_BUILD_VECTOR) { diff --git a/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp b/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp index 5e311589103e50..e1d6f8923c51d5 100644 --- a/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp @@ -444,10 +444,10 @@ bool SPIRVPrepareFunctions::substituteIntrinsicCalls(Function *F) { lowerFunnelShifts(II); Changed = true; break; - case Intrinsic::umul_with_overflow: - lowerUMulWithOverflow(II); - Changed = true; - break; +// case Intrinsic::umul_with_overflow: +// lowerUMulWithOverflow(II); +// Changed = true; +// break; case Intrinsic::assume: case Intrinsic::expect: { const SPIRVSubtarget &STI = TM.getSubtarget(*F); @@ -478,7 +478,11 @@ bool SPIRVPrepareFunctions::substituteIntrinsicCalls(Function *F) { // noted in 'spv.cloned_funcs' metadata for later restoration. Function * SPIRVPrepareFunctions::removeAggregateTypesFromSignature(Function *F) { - if (F->isIntrinsic()) + bool IsRetAggr = F->getReturnType()->isAggregateType(); + // Allow intrinsics with aggregate return type to reach GlobalISel + if (F->isIntrinsic() && IsRetAggr) + return F; + /* // Allow intrinsics with aggregate return type to reach GlobalISel switch (F->getIntrinsicID()) { // Standard C/C++ Library Intrinsics @@ -495,10 +499,10 @@ SPIRVPrepareFunctions::removeAggregateTypesFromSignature(Function *F) { case Intrinsic::usub_with_overflow: return F; } +*/ IRBuilder<> B(F->getContext()); - bool IsRetAggr = F->getReturnType()->isAggregateType(); bool HasAggrArg = std::any_of(F->arg_begin(), F->arg_end(), [](Argument &Arg) { return Arg.getType()->isAggregateType(); @@ -581,7 +585,6 @@ bool SPIRVPrepareFunctions::runOnModule(Module &M) { Changed = true; } } - return Changed; } From 45ea3fd8cb281d4bf1a0eebff63c253c60ff0a65 Mon Sep 17 00:00:00 2001 From: "Levytskyy, Vyacheslav" Date: Tue, 17 Sep 2024 05:36:01 -0700 Subject: [PATCH 03/16] fix passing info and access to metadata --- llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp | 3 ++- llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp | 11 ++++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp index 73fb5c207a26cf..3ca1ea5024b2ee 100644 --- a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp @@ -318,7 +318,8 @@ void SPIRVEmitIntrinsics::buildAssignType(IRBuilder<> &B, Type *Ty, Value *Arg) { Value *OfType = PoisonValue::get(Ty); CallInst *AssignCI = nullptr; - if (Ty->isAggregateType() && allowEmitFakeUse(Arg)) { + if (Arg->getType()->isAggregateType() && Ty->isAggregateType() && + allowEmitFakeUse(Arg)) { LLVMContext &Ctx = Arg->getContext(); SmallVector ArgMDs{ MDNode::get(Ctx, ValueAsMetadata::getConstant(OfType)), diff --git a/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp b/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp index a67b80605f62c7..4215710dbfda62 100644 --- a/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp @@ -473,14 +473,19 @@ generateAssignInstrs(MachineFunction &MF, SPIRVGlobalRegistry *GR, if (MdMI && isSpvIntrinsic(*MdMI, Intrinsic::spv_value_md)) { // It's an internal service info from before IRTranslator passes. MachineInstr *Def = MRI.getVRegDef(MI.getOperand(0).getReg()); - for (unsigned I = 1, E = MI.getNumOperands(); I != E && Def; ++I) - if (MRI.getVRegDef(MI.getOperand(I).getReg()) != Def) + for (unsigned I = 1, E = MI.getNumOperands(); I != E && Def; ++I) { + MachineInstr *MaybeDef = MRI.getVRegDef(MI.getOperand(I).getReg()); + if (MaybeDef && MaybeDef->getOpcode() == SPIRV::ASSIGN_TYPE) + MaybeDef = MRI.getVRegDef(MaybeDef->getOperand(1).getReg()); + if (MaybeDef != Def) Def = nullptr; + } if (Def) { const MDNode *MD = MdMI->getOperand(1).getMetadata(); - Type *ValueTy = cast(MD->getOperand(0))->getType(); StringRef ValueName = cast(MD->getOperand(1))->getString(); + const MDNode *TypeMD = cast(MD->getOperand(0)); + Type *ValueTy = getMDOperandAsType(TypeMD, 0); GR->addValueAttrs(Def, std::make_pair(ValueTy, ValueName)); } } From 063030d43573ae5576bfd6d2cf92896536370ea7 Mon Sep 17 00:00:00 2001 From: "Levytskyy, Vyacheslav" Date: Tue, 17 Sep 2024 06:03:39 -0700 Subject: [PATCH 04/16] fixes --- llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp | 10 +++------- llvm/lib/Target/SPIRV/SPIRVUtils.cpp | 5 +++++ llvm/lib/Target/SPIRV/SPIRVUtils.h | 2 ++ 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp b/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp index 4215710dbfda62..90020fdc44a548 100644 --- a/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp @@ -472,14 +472,10 @@ generateAssignInstrs(MachineFunction &MF, SPIRVGlobalRegistry *GR, MachineInstr *MdMI = MI.getPrevNode(); if (MdMI && isSpvIntrinsic(*MdMI, Intrinsic::spv_value_md)) { // It's an internal service info from before IRTranslator passes. - MachineInstr *Def = MRI.getVRegDef(MI.getOperand(0).getReg()); - for (unsigned I = 1, E = MI.getNumOperands(); I != E && Def; ++I) { - MachineInstr *MaybeDef = MRI.getVRegDef(MI.getOperand(I).getReg()); - if (MaybeDef && MaybeDef->getOpcode() == SPIRV::ASSIGN_TYPE) - MaybeDef = MRI.getVRegDef(MaybeDef->getOperand(1).getReg()); - if (MaybeDef != Def) + MachineInstr *Def = getVRegDef(MRI, MI.getOperand(0).getReg()); + for (unsigned I = 1, E = MI.getNumOperands(); I != E && Def; ++I) + if (getVRegDef(MRI, MI.getOperand(I).getReg()) != Def) Def = nullptr; - } if (Def) { const MDNode *MD = MdMI->getOperand(1).getMetadata(); StringRef ValueName = diff --git a/llvm/lib/Target/SPIRV/SPIRVUtils.cpp b/llvm/lib/Target/SPIRV/SPIRVUtils.cpp index 53601e402c737e..aaa0572f912d65 100644 --- a/llvm/lib/Target/SPIRV/SPIRVUtils.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVUtils.cpp @@ -589,6 +589,11 @@ bool sortBlocks(Function &F) { } return Modified; +MachineInstr *getVRegDef(MachineRegisterInfo &MRI, Register Reg) { + MachineInstr *MaybeDef = MRI.getVRegDef(Reg); + if (MaybeDef && MaybeDef->getOpcode() == SPIRV::ASSIGN_TYPE) + MaybeDef = MRI.getVRegDef(MaybeDef->getOperand(1).getReg()); + return MaybeDef; } } // namespace llvm diff --git a/llvm/lib/Target/SPIRV/SPIRVUtils.h b/llvm/lib/Target/SPIRV/SPIRVUtils.h index 93d64a7f435e92..f66a2b716443d1 100644 --- a/llvm/lib/Target/SPIRV/SPIRVUtils.h +++ b/llvm/lib/Target/SPIRV/SPIRVUtils.h @@ -313,5 +313,7 @@ inline const Type *unifyPtrType(const Type *Ty) { return toTypedPointer(const_cast(Ty)); } +MachineInstr *getVRegDef(MachineRegisterInfo &MRI, Register Reg); + } // namespace llvm #endif // LLVM_LIB_TARGET_SPIRV_SPIRVUTILS_H From abcf8e64017f55209774dc4c241d31428b6a788a Mon Sep 17 00:00:00 2001 From: "Levytskyy, Vyacheslav" Date: Tue, 17 Sep 2024 08:37:34 -0700 Subject: [PATCH 05/16] implement 'Arithmetic with Overflow' intrinsics --- .../Target/SPIRV/SPIRVInstructionSelector.cpp | 17 +++++- .../Target/SPIRV/SPIRVPrepareFunctions.cpp | 60 ------------------- .../llvm-intrinsics/umul.with.overflow.ll | 4 ++ 3 files changed, 20 insertions(+), 61 deletions(-) diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp index 05a63aba902a58..413817025d8987 100644 --- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp @@ -1156,15 +1156,30 @@ bool SPIRVInstructionSelector::selectOverflowArith(Register ResVReg, MIB.addUse(I.getOperand(i).getReg()); bool Status = MIB.constrainAllUses(TII, TRI, RBI); // Build instructions to extract fields of the instruction's result. + // A new virtual register to store the higher part of the result struct. + Register HigherVReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); + MRI->setRegClass(HigherVReg, &SPIRV::iIDRegClass); for (unsigned i = 0; i < I.getNumDefs(); ++i) { auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) - .addDef(I.getOperand(i).getReg()) + .addDef(i == 1 ? HigherVReg : I.getOperand(i).getReg()) .addUse(GR.getSPIRVTypeID(ResType)) .addUse(StructVReg) .addImm(i); Status &= MIB.constrainAllUses(TII, TRI, RBI); } + // Build boolean value from the higher part. + assert(I.getNumDefs() > 1 && "Not enought operands"); + SPIRVType *BoolType = GR.getOrCreateSPIRVBoolType(I, TII); + unsigned N = GR.getScalarOrVectorComponentCount(ResType); + if (N > 1) + BoolType = GR.getOrCreateSPIRVVectorType(BoolType, N, I, TII); + Status &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpINotEqual)) + .addDef(I.getOperand(1).getReg()) + .addUse(GR.getSPIRVTypeID(BoolType)) + .addUse(HigherVReg) + .addUse(buildZerosVal(ResType, I)) + .constrainAllUses(TII, TRI, RBI); return Status; } diff --git a/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp b/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp index e1d6f8923c51d5..1872b238d1077a 100644 --- a/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp @@ -342,30 +342,6 @@ static void lowerFunnelShifts(IntrinsicInst *FSHIntrinsic) { FSHIntrinsic->setCalledFunction(FSHFunc); } -static void buildUMulWithOverflowFunc(Function *UMulFunc) { - // The function body is already created. - if (!UMulFunc->empty()) - return; - - BasicBlock *EntryBB = BasicBlock::Create(UMulFunc->getParent()->getContext(), - "entry", UMulFunc); - IRBuilder<> IRB(EntryBB); - // Build the actual unsigned multiplication logic with the overflow - // indication. Do unsigned multiplication Mul = A * B. Then check - // if unsigned division Div = Mul / A is not equal to B. If so, - // then overflow has happened. - Value *Mul = IRB.CreateNUWMul(UMulFunc->getArg(0), UMulFunc->getArg(1)); - Value *Div = IRB.CreateUDiv(Mul, UMulFunc->getArg(0)); - Value *Overflow = IRB.CreateICmpNE(UMulFunc->getArg(0), Div); - - // umul.with.overflow intrinsic return a structure, where the first element - // is the multiplication result, and the second is an overflow bit. - Type *StructTy = UMulFunc->getReturnType(); - Value *Agg = IRB.CreateInsertValue(PoisonValue::get(StructTy), Mul, {0}); - Value *Res = IRB.CreateInsertValue(Agg, Overflow, {1}); - IRB.CreateRet(Res); -} - static void lowerExpectAssume(IntrinsicInst *II) { // If we cannot use the SPV_KHR_expect_assume extension, then we need to // ignore the intrinsic and move on. It should be removed later on by LLVM. @@ -407,20 +383,6 @@ static bool toSpvOverloadedIntrinsic(IntrinsicInst *II, Intrinsic::ID NewID, return true; } -static void lowerUMulWithOverflow(IntrinsicInst *UMulIntrinsic) { - // Get a separate function - otherwise, we'd have to rework the CFG of the - // current one. Then simply replace the intrinsic uses with a call to the new - // function. - Module *M = UMulIntrinsic->getModule(); - FunctionType *UMulFuncTy = UMulIntrinsic->getFunctionType(); - Type *FSHLRetTy = UMulFuncTy->getReturnType(); - const std::string FuncName = lowerLLVMIntrinsicName(UMulIntrinsic); - Function *UMulFunc = - getOrCreateFunction(M, FSHLRetTy, UMulFuncTy->params(), FuncName); - buildUMulWithOverflowFunc(UMulFunc); - UMulIntrinsic->setCalledFunction(UMulFunc); -} - // Substitutes calls to LLVM intrinsics with either calls to SPIR-V intrinsics // or calls to proper generated functions. Returns True if F was modified. bool SPIRVPrepareFunctions::substituteIntrinsicCalls(Function *F) { @@ -444,10 +406,6 @@ bool SPIRVPrepareFunctions::substituteIntrinsicCalls(Function *F) { lowerFunnelShifts(II); Changed = true; break; -// case Intrinsic::umul_with_overflow: -// lowerUMulWithOverflow(II); -// Changed = true; -// break; case Intrinsic::assume: case Intrinsic::expect: { const SPIRVSubtarget &STI = TM.getSubtarget(*F); @@ -482,24 +440,6 @@ SPIRVPrepareFunctions::removeAggregateTypesFromSignature(Function *F) { // Allow intrinsics with aggregate return type to reach GlobalISel if (F->isIntrinsic() && IsRetAggr) return F; - /* - // Allow intrinsics with aggregate return type to reach GlobalISel - switch (F->getIntrinsicID()) { - // Standard C/C++ Library Intrinsics - case Intrinsic::frexp: - // Vector Reduction Intrinsics - case Intrinsic::vector_interleave2: - case Intrinsic::vector_deinterleave2: - // Arithmetic with Overflow Intrinsics - case Intrinsic::smul_with_overflow: - case Intrinsic::umul_with_overflow: - case Intrinsic::sadd_with_overflow: - case Intrinsic::uadd_with_overflow: - case Intrinsic::ssub_with_overflow: - case Intrinsic::usub_with_overflow: - return F; - } -*/ IRBuilder<> B(F->getContext()); diff --git a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/umul.with.overflow.ll b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/umul.with.overflow.ll index c34771bf381ea9..047c8eff014205 100644 --- a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/umul.with.overflow.ll +++ b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/umul.with.overflow.ll @@ -1,5 +1,9 @@ ; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV +; This test fails due to change in code patterns that SPIR-V Backend emits for +; "Arithmetic with Overflow" intrinsics. +; XFAIL: * + ; CHECK-SPIRV: OpName %[[#NAME_UMUL_FUNC_8:]] "spirv.llvm_umul_with_overflow_i8" ; CHECK-SPIRV: OpName %[[#NAME_UMUL_FUNC_32:]] "spirv.llvm_umul_with_overflow_i32" ; CHECK-SPIRV: OpName %[[#NAME_UMUL_FUNC_VEC_I64:]] "spirv.llvm_umul_with_overflow_v2i64" From ff8e18e4e8556524bb159287265d533b20787567 Mon Sep 17 00:00:00 2001 From: "Levytskyy, Vyacheslav" Date: Tue, 17 Sep 2024 08:46:45 -0700 Subject: [PATCH 06/16] clang-format --- llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp index 413817025d8987..b291cc8212ccdb 100644 --- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp @@ -1175,11 +1175,11 @@ bool SPIRVInstructionSelector::selectOverflowArith(Register ResVReg, if (N > 1) BoolType = GR.getOrCreateSPIRVVectorType(BoolType, N, I, TII); Status &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpINotEqual)) - .addDef(I.getOperand(1).getReg()) - .addUse(GR.getSPIRVTypeID(BoolType)) - .addUse(HigherVReg) - .addUse(buildZerosVal(ResType, I)) - .constrainAllUses(TII, TRI, RBI); + .addDef(I.getOperand(1).getReg()) + .addUse(GR.getSPIRVTypeID(BoolType)) + .addUse(HigherVReg) + .addUse(buildZerosVal(ResType, I)) + .constrainAllUses(TII, TRI, RBI); return Status; } From a65fd2ebc759fa3272921114d8cfa025896c7126 Mon Sep 17 00:00:00 2001 From: "Levytskyy, Vyacheslav" Date: Tue, 17 Sep 2024 13:42:38 -0700 Subject: [PATCH 07/16] add a test case --- .../llvm-intrinsics/arith.with.overflow.ll | 89 +++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 llvm/test/CodeGen/SPIRV/llvm-intrinsics/arith.with.overflow.ll diff --git a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/arith.with.overflow.ll b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/arith.with.overflow.ll new file mode 100644 index 00000000000000..7113dd692f6ac4 --- /dev/null +++ b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/arith.with.overflow.ll @@ -0,0 +1,89 @@ +; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %} + +; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %} + +; CHECK-DAG: %[[Char:.*]] = OpTypeInt 8 0 +; CHECK-DAG: %[[Void:.*]] = OpTypeVoid +; CHECK-DAG: %[[PtrChar:.*]] = OpTypePointer Function %[[Char]] +; CHECK-DAG: %[[StructChar:.*]] = OpTypeStruct %[[Char]] %[[Char]] +; CHECK-DAG: %[[ZeroChar:.*]] = OpConstant %[[Char]] 0 +; CHECK-DAG: %[[Int:.*]] = OpTypeInt 32 0 +; CHECK-DAG: %[[PtrInt:.*]] = OpTypePointer Function %[[Int]] +; CHECK-DAG: %[[StructInt:.*]] = OpTypeStruct %[[Int]] %[[Int]] +; CHECK-DAG: %[[ZeroInt:.*]] = OpConstant %[[Int]] 0 +; CHECK-DAG: %[[Bool:.*]] = OpTypeBool +; CHECK-DAG: %[[V2Bool:.*]] = OpTypeVector %[[Bool]] 2 +; CHECK-DAG: %[[Long:.*]] = OpTypeInt 64 0 +; CHECK-DAG: %[[V2Long:.*]] = OpTypeVector %[[Long]] 2 +; CHECK-DAG: %[[PtrV2Long:.*]] = OpTypePointer Function %[[V2Long]] +; CHECK-DAG: %[[StructV2Long:.*]] = OpTypeStruct %[[V2Long]] %[[V2Long]] +; CHECK-DAG: %[[ZeroV2Long:.*]] = OpConstantNull %[[V2Long]] + +; CHECK: OpFunction +; CHECK: %[[A:.*]] = OpFunctionParameter %[[Char]] +; CHECK: %[[B:.*]] = OpFunctionParameter %[[Char]] +; CHECK: %[[Ptr:.*]] = OpFunctionParameter %[[PtrChar]] +; CHECK: %[[Struct:.*]] = OpUMulExtended %[[StructChar]] %[[A]] %[[B]] +; CHECK: %[[Val:.*]] = OpCompositeExtract %[[Char]] %[[Struct]] 0 +; CHECK: %[[Over:.*]] = OpCompositeExtract %[[Char]] %[[Struct]] 1 +; CHECK: %[[IsOver:.*]] = OpINotEqual %[[Bool]] %[[Over]] %[[ZeroChar]] +; CHECK: %[[Res:.*]] = OpSelect %[[Char]] %[[IsOver]] %[[ZeroChar]] %[[Val]] +; CHECK: OpStore %[[Ptr]] %[[Res]] Aligned 1 +; CHECK: OpReturn +define dso_local spir_func void @umulo_i8(i8 zeroext %a, i8 zeroext %b, ptr nocapture %c) local_unnamed_addr { +entry: + %umul = tail call { i8, i1 } @llvm.umul.with.overflow.i8(i8 %a, i8 %b) + %cmp = extractvalue { i8, i1 } %umul, 1 + %umul.value = extractvalue { i8, i1 } %umul, 0 + %storemerge = select i1 %cmp, i8 0, i8 %umul.value + store i8 %storemerge, ptr %c, align 1 + ret void +} + +; CHECK: OpFunction +; CHECK: %[[A2:.*]] = OpFunctionParameter %[[Int]] +; CHECK: %[[B2:.*]] = OpFunctionParameter %[[Int]] +; CHECK: %[[Ptr2:.*]] = OpFunctionParameter %[[PtrInt]] +; CHECK: %[[Struct2:.*]] = OpUMulExtended %[[StructInt]] %[[B2]] %[[A2]] +; CHECK: %[[Val2:.*]] = OpCompositeExtract %[[Int]] %[[Struct2]] 0 +; CHECK: %[[Over2:.*]] = OpCompositeExtract %[[Int]] %[[Struct2]] 1 +; CHECK: %[[IsOver2:.*]] = OpINotEqual %[[Bool]] %[[Over2]] %[[ZeroInt]] +; CHECK: %[[Res2:.*]] = OpSelect %[[Int]] %[[IsOver2]] %[[ZeroInt]] %[[Val2]] +; CHECK: OpStore %[[Ptr2]] %[[Res2]] Aligned 4 +; CHECK: OpReturn +define dso_local spir_func void @umulo_i32(i32 %a, i32 %b, ptr nocapture %c) local_unnamed_addr { +entry: + %umul = tail call { i32, i1 } @llvm.umul.with.overflow.i32(i32 %b, i32 %a) + %umul.val = extractvalue { i32, i1 } %umul, 0 + %umul.ov = extractvalue { i32, i1 } %umul, 1 + %spec.select = select i1 %umul.ov, i32 0, i32 %umul.val + store i32 %spec.select, ptr %c, align 4 + ret void +} + +; CHECK: OpFunction +; CHECK: %[[A3:.*]] = OpFunctionParameter %[[V2Long]] +; CHECK: %[[B3:.*]] = OpFunctionParameter %[[V2Long]] +; CHECK: %[[Ptr3:.*]] = OpFunctionParameter %[[PtrV2Long]] +; CHECK: %[[Struct3:.*]] = OpUMulExtended %[[StructV2Long]] %[[A3]] %[[B3]] +; CHECK: %[[Val3:.*]] = OpCompositeExtract %[[V2Long]] %[[Struct3]] 0 +; CHECK: %[[Over3:.*]] = OpCompositeExtract %[[V2Long]] %[[Struct3]] 1 +; CHECK: %[[IsOver3:.*]] = OpINotEqual %[[V2Bool]] %[[Over3]] %[[ZeroV2Long]] +; CHECK: %[[Res3:.*]] = OpSelect %[[V2Long]] %[[IsOver3]] %[[ZeroV2Long]] %[[Val3]] +; CHECK: OpStore %[[Ptr3]] %[[Res3]] Aligned 16 +; CHECK: OpReturn +define dso_local spir_func void @umulo_v2i64(<2 x i64> %a, <2 x i64> %b, ptr %p) nounwind { + %umul = call {<2 x i64>, <2 x i1>} @llvm.umul.with.overflow.v2i64(<2 x i64> %a, <2 x i64> %b) + %umul.val = extractvalue {<2 x i64>, <2 x i1>} %umul, 0 + %umul.ov = extractvalue {<2 x i64>, <2 x i1>} %umul, 1 + %zero = alloca <2 x i64>, align 16 + %spec.select = select <2 x i1> %umul.ov, <2 x i64> , <2 x i64> %umul.val + store <2 x i64> %spec.select, ptr %p + ret void +} + +declare {i8, i1} @llvm.umul.with.overflow.i8(i8, i8) +declare {i32, i1} @llvm.umul.with.overflow.i32(i32, i32) +declare {<2 x i64>, <2 x i1>} @llvm.umul.with.overflow.v2i64(<2 x i64>, <2 x i64>) From 4e365d59c0401c68699e3ea1f06d0b723a1f80b0 Mon Sep 17 00:00:00 2001 From: "Levytskyy, Vyacheslav" Date: Tue, 17 Sep 2024 14:04:16 -0700 Subject: [PATCH 08/16] add scalar/vector versions of OpIAddCarry/OpISubBorrow; add test cases --- llvm/lib/Target/SPIRV/SPIRVInstrInfo.td | 4 +- .../Target/SPIRV/SPIRVInstructionSelector.cpp | 10 +- ...with.overflow.ll => smul.with.overflow.ll} | 18 ++-- .../llvm-intrinsics/uadd.with.overflow.ll | 89 ++++++++++++++++++ .../llvm-intrinsics/umul.with.overflow.ll | 91 +++++++++++++------ .../llvm-intrinsics/usub.with.overflow.ll | 89 ++++++++++++++++++ 6 files changed, 258 insertions(+), 43 deletions(-) rename llvm/test/CodeGen/SPIRV/llvm-intrinsics/{arith.with.overflow.ll => smul.with.overflow.ll} (87%) create mode 100644 llvm/test/CodeGen/SPIRV/llvm-intrinsics/uadd.with.overflow.ll create mode 100644 llvm/test/CodeGen/SPIRV/llvm-intrinsics/usub.with.overflow.ll diff --git a/llvm/lib/Target/SPIRV/SPIRVInstrInfo.td b/llvm/lib/Target/SPIRV/SPIRVInstrInfo.td index 51bacb00b1c515..fe45be4daba650 100644 --- a/llvm/lib/Target/SPIRV/SPIRVInstrInfo.td +++ b/llvm/lib/Target/SPIRV/SPIRVInstrInfo.td @@ -519,8 +519,8 @@ def OpMatrixTimesMatrix: BinOp<"OpMatrixTimesMatrix", 146>; def OpOuterProduct: BinOp<"OpOuterProduct", 147>; def OpDot: BinOp<"OpDot", 148>; -def OpIAddCarry: BinOpTyped<"OpIAddCarry", 149, iID, addc>; -def OpISubBorrow: BinOpTyped<"OpISubBorrow", 150, iID, subc>; +defm OpIAddCarry: BinOpTypedGen<"OpIAddCarry", 149, addc, 0, 1>; +defm OpISubBorrow: BinOpTypedGen<"OpISubBorrow", 150, subc, 0, 1>; def OpUMulExtended: BinOp<"OpUMulExtended", 151>; def OpSMulExtended: BinOp<"OpSMulExtended", 152>; diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp index b291cc8212ccdb..9e3cc4fdf01259 100644 --- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp @@ -598,13 +598,19 @@ bool SPIRVInstructionSelector::spvSelect(Register ResVReg, "Selection of a signed addition with overflow is not implemented", false); case TargetOpcode::G_UADDO: - return selectOverflowArith(ResVReg, ResType, I, SPIRV::OpIAddCarry); + return selectOverflowArith(ResVReg, ResType, I, + ResType->getOpcode() == SPIRV::OpTypeVector + ? SPIRV::OpIAddCarryV + : SPIRV::OpIAddCarryS); case TargetOpcode::G_SSUBO: report_fatal_error( "Selection of a signed subtraction with overflow is not implemented", false); case TargetOpcode::G_USUBO: - return selectOverflowArith(ResVReg, ResType, I, SPIRV::OpISubBorrow); + return selectOverflowArith(ResVReg, ResType, I, + ResType->getOpcode() == SPIRV::OpTypeVector + ? SPIRV::OpISubBorrowV + : SPIRV::OpISubBorrowS); case TargetOpcode::G_UMULO: return selectOverflowArith(ResVReg, ResType, I, SPIRV::OpUMulExtended); case TargetOpcode::G_SMULO: diff --git a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/arith.with.overflow.ll b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/smul.with.overflow.ll similarity index 87% rename from llvm/test/CodeGen/SPIRV/llvm-intrinsics/arith.with.overflow.ll rename to llvm/test/CodeGen/SPIRV/llvm-intrinsics/smul.with.overflow.ll index 7113dd692f6ac4..2281ccf52bbb4e 100644 --- a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/arith.with.overflow.ll +++ b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/smul.with.overflow.ll @@ -25,7 +25,7 @@ ; CHECK: %[[A:.*]] = OpFunctionParameter %[[Char]] ; CHECK: %[[B:.*]] = OpFunctionParameter %[[Char]] ; CHECK: %[[Ptr:.*]] = OpFunctionParameter %[[PtrChar]] -; CHECK: %[[Struct:.*]] = OpUMulExtended %[[StructChar]] %[[A]] %[[B]] +; CHECK: %[[Struct:.*]] = OpSMulExtended %[[StructChar]] %[[A]] %[[B]] ; CHECK: %[[Val:.*]] = OpCompositeExtract %[[Char]] %[[Struct]] 0 ; CHECK: %[[Over:.*]] = OpCompositeExtract %[[Char]] %[[Struct]] 1 ; CHECK: %[[IsOver:.*]] = OpINotEqual %[[Bool]] %[[Over]] %[[ZeroChar]] @@ -34,7 +34,7 @@ ; CHECK: OpReturn define dso_local spir_func void @umulo_i8(i8 zeroext %a, i8 zeroext %b, ptr nocapture %c) local_unnamed_addr { entry: - %umul = tail call { i8, i1 } @llvm.umul.with.overflow.i8(i8 %a, i8 %b) + %umul = tail call { i8, i1 } @llvm.smul.with.overflow.i8(i8 %a, i8 %b) %cmp = extractvalue { i8, i1 } %umul, 1 %umul.value = extractvalue { i8, i1 } %umul, 0 %storemerge = select i1 %cmp, i8 0, i8 %umul.value @@ -46,7 +46,7 @@ entry: ; CHECK: %[[A2:.*]] = OpFunctionParameter %[[Int]] ; CHECK: %[[B2:.*]] = OpFunctionParameter %[[Int]] ; CHECK: %[[Ptr2:.*]] = OpFunctionParameter %[[PtrInt]] -; CHECK: %[[Struct2:.*]] = OpUMulExtended %[[StructInt]] %[[B2]] %[[A2]] +; CHECK: %[[Struct2:.*]] = OpSMulExtended %[[StructInt]] %[[B2]] %[[A2]] ; CHECK: %[[Val2:.*]] = OpCompositeExtract %[[Int]] %[[Struct2]] 0 ; CHECK: %[[Over2:.*]] = OpCompositeExtract %[[Int]] %[[Struct2]] 1 ; CHECK: %[[IsOver2:.*]] = OpINotEqual %[[Bool]] %[[Over2]] %[[ZeroInt]] @@ -55,7 +55,7 @@ entry: ; CHECK: OpReturn define dso_local spir_func void @umulo_i32(i32 %a, i32 %b, ptr nocapture %c) local_unnamed_addr { entry: - %umul = tail call { i32, i1 } @llvm.umul.with.overflow.i32(i32 %b, i32 %a) + %umul = tail call { i32, i1 } @llvm.smul.with.overflow.i32(i32 %b, i32 %a) %umul.val = extractvalue { i32, i1 } %umul, 0 %umul.ov = extractvalue { i32, i1 } %umul, 1 %spec.select = select i1 %umul.ov, i32 0, i32 %umul.val @@ -67,7 +67,7 @@ entry: ; CHECK: %[[A3:.*]] = OpFunctionParameter %[[V2Long]] ; CHECK: %[[B3:.*]] = OpFunctionParameter %[[V2Long]] ; CHECK: %[[Ptr3:.*]] = OpFunctionParameter %[[PtrV2Long]] -; CHECK: %[[Struct3:.*]] = OpUMulExtended %[[StructV2Long]] %[[A3]] %[[B3]] +; CHECK: %[[Struct3:.*]] = OpSMulExtended %[[StructV2Long]] %[[A3]] %[[B3]] ; CHECK: %[[Val3:.*]] = OpCompositeExtract %[[V2Long]] %[[Struct3]] 0 ; CHECK: %[[Over3:.*]] = OpCompositeExtract %[[V2Long]] %[[Struct3]] 1 ; CHECK: %[[IsOver3:.*]] = OpINotEqual %[[V2Bool]] %[[Over3]] %[[ZeroV2Long]] @@ -75,7 +75,7 @@ entry: ; CHECK: OpStore %[[Ptr3]] %[[Res3]] Aligned 16 ; CHECK: OpReturn define dso_local spir_func void @umulo_v2i64(<2 x i64> %a, <2 x i64> %b, ptr %p) nounwind { - %umul = call {<2 x i64>, <2 x i1>} @llvm.umul.with.overflow.v2i64(<2 x i64> %a, <2 x i64> %b) + %umul = call {<2 x i64>, <2 x i1>} @llvm.smul.with.overflow.v2i64(<2 x i64> %a, <2 x i64> %b) %umul.val = extractvalue {<2 x i64>, <2 x i1>} %umul, 0 %umul.ov = extractvalue {<2 x i64>, <2 x i1>} %umul, 1 %zero = alloca <2 x i64>, align 16 @@ -84,6 +84,6 @@ define dso_local spir_func void @umulo_v2i64(<2 x i64> %a, <2 x i64> %b, ptr %p) ret void } -declare {i8, i1} @llvm.umul.with.overflow.i8(i8, i8) -declare {i32, i1} @llvm.umul.with.overflow.i32(i32, i32) -declare {<2 x i64>, <2 x i1>} @llvm.umul.with.overflow.v2i64(<2 x i64>, <2 x i64>) +declare {i8, i1} @llvm.smul.with.overflow.i8(i8, i8) +declare {i32, i1} @llvm.smul.with.overflow.i32(i32, i32) +declare {<2 x i64>, <2 x i1>} @llvm.smul.with.overflow.v2i64(<2 x i64>, <2 x i64>) diff --git a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/uadd.with.overflow.ll b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/uadd.with.overflow.ll new file mode 100644 index 00000000000000..cecd6f60655dcf --- /dev/null +++ b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/uadd.with.overflow.ll @@ -0,0 +1,89 @@ +; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %} + +; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %} + +; CHECK-DAG: %[[Char:.*]] = OpTypeInt 8 0 +; CHECK-DAG: %[[Void:.*]] = OpTypeVoid +; CHECK-DAG: %[[PtrChar:.*]] = OpTypePointer Function %[[Char]] +; CHECK-DAG: %[[StructChar:.*]] = OpTypeStruct %[[Char]] %[[Char]] +; CHECK-DAG: %[[ZeroChar:.*]] = OpConstant %[[Char]] 0 +; CHECK-DAG: %[[Int:.*]] = OpTypeInt 32 0 +; CHECK-DAG: %[[PtrInt:.*]] = OpTypePointer Function %[[Int]] +; CHECK-DAG: %[[StructInt:.*]] = OpTypeStruct %[[Int]] %[[Int]] +; CHECK-DAG: %[[ZeroInt:.*]] = OpConstant %[[Int]] 0 +; CHECK-DAG: %[[Bool:.*]] = OpTypeBool +; CHECK-DAG: %[[V2Bool:.*]] = OpTypeVector %[[Bool]] 2 +; CHECK-DAG: %[[Long:.*]] = OpTypeInt 64 0 +; CHECK-DAG: %[[V2Long:.*]] = OpTypeVector %[[Long]] 2 +; CHECK-DAG: %[[PtrV2Long:.*]] = OpTypePointer Function %[[V2Long]] +; CHECK-DAG: %[[StructV2Long:.*]] = OpTypeStruct %[[V2Long]] %[[V2Long]] +; CHECK-DAG: %[[ZeroV2Long:.*]] = OpConstantNull %[[V2Long]] + +; CHECK: OpFunction +; CHECK: %[[A:.*]] = OpFunctionParameter %[[Char]] +; CHECK: %[[B:.*]] = OpFunctionParameter %[[Char]] +; CHECK: %[[Ptr:.*]] = OpFunctionParameter %[[PtrChar]] +; CHECK: %[[Struct:.*]] = OpIAddCarry %[[StructChar]] %[[A]] %[[B]] +; CHECK: %[[Val:.*]] = OpCompositeExtract %[[Char]] %[[Struct]] 0 +; CHECK: %[[Over:.*]] = OpCompositeExtract %[[Char]] %[[Struct]] 1 +; CHECK: %[[IsOver:.*]] = OpINotEqual %[[Bool]] %[[Over]] %[[ZeroChar]] +; CHECK: %[[Res:.*]] = OpSelect %[[Char]] %[[IsOver]] %[[ZeroChar]] %[[Val]] +; CHECK: OpStore %[[Ptr]] %[[Res]] Aligned 1 +; CHECK: OpReturn +define dso_local spir_func void @umulo_i8(i8 zeroext %a, i8 zeroext %b, ptr nocapture %c) local_unnamed_addr { +entry: + %umul = tail call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 %a, i8 %b) + %cmp = extractvalue { i8, i1 } %umul, 1 + %umul.value = extractvalue { i8, i1 } %umul, 0 + %storemerge = select i1 %cmp, i8 0, i8 %umul.value + store i8 %storemerge, ptr %c, align 1 + ret void +} + +; CHECK: OpFunction +; CHECK: %[[A2:.*]] = OpFunctionParameter %[[Int]] +; CHECK: %[[B2:.*]] = OpFunctionParameter %[[Int]] +; CHECK: %[[Ptr2:.*]] = OpFunctionParameter %[[PtrInt]] +; CHECK: %[[Struct2:.*]] = OpIAddCarry %[[StructInt]] %[[B2]] %[[A2]] +; CHECK: %[[Val2:.*]] = OpCompositeExtract %[[Int]] %[[Struct2]] 0 +; CHECK: %[[Over2:.*]] = OpCompositeExtract %[[Int]] %[[Struct2]] 1 +; CHECK: %[[IsOver2:.*]] = OpINotEqual %[[Bool]] %[[Over2]] %[[ZeroInt]] +; CHECK: %[[Res2:.*]] = OpSelect %[[Int]] %[[IsOver2]] %[[ZeroInt]] %[[Val2]] +; CHECK: OpStore %[[Ptr2]] %[[Res2]] Aligned 4 +; CHECK: OpReturn +define dso_local spir_func void @umulo_i32(i32 %a, i32 %b, ptr nocapture %c) local_unnamed_addr { +entry: + %umul = tail call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 %b, i32 %a) + %umul.val = extractvalue { i32, i1 } %umul, 0 + %umul.ov = extractvalue { i32, i1 } %umul, 1 + %spec.select = select i1 %umul.ov, i32 0, i32 %umul.val + store i32 %spec.select, ptr %c, align 4 + ret void +} + +; CHECK: OpFunction +; CHECK: %[[A3:.*]] = OpFunctionParameter %[[V2Long]] +; CHECK: %[[B3:.*]] = OpFunctionParameter %[[V2Long]] +; CHECK: %[[Ptr3:.*]] = OpFunctionParameter %[[PtrV2Long]] +; CHECK: %[[Struct3:.*]] = OpIAddCarry %[[StructV2Long]] %[[A3]] %[[B3]] +; CHECK: %[[Val3:.*]] = OpCompositeExtract %[[V2Long]] %[[Struct3]] 0 +; CHECK: %[[Over3:.*]] = OpCompositeExtract %[[V2Long]] %[[Struct3]] 1 +; CHECK: %[[IsOver3:.*]] = OpINotEqual %[[V2Bool]] %[[Over3]] %[[ZeroV2Long]] +; CHECK: %[[Res3:.*]] = OpSelect %[[V2Long]] %[[IsOver3]] %[[ZeroV2Long]] %[[Val3]] +; CHECK: OpStore %[[Ptr3]] %[[Res3]] Aligned 16 +; CHECK: OpReturn +define dso_local spir_func void @umulo_v2i64(<2 x i64> %a, <2 x i64> %b, ptr %p) nounwind { + %umul = call {<2 x i64>, <2 x i1>} @llvm.uadd.with.overflow.v2i64(<2 x i64> %a, <2 x i64> %b) + %umul.val = extractvalue {<2 x i64>, <2 x i1>} %umul, 0 + %umul.ov = extractvalue {<2 x i64>, <2 x i1>} %umul, 1 + %zero = alloca <2 x i64>, align 16 + %spec.select = select <2 x i1> %umul.ov, <2 x i64> , <2 x i64> %umul.val + store <2 x i64> %spec.select, ptr %p + ret void +} + +declare {i8, i1} @llvm.uadd.with.overflow.i8(i8, i8) +declare {i32, i1} @llvm.uadd.with.overflow.i32(i32, i32) +declare {<2 x i64>, <2 x i1>} @llvm.uadd.with.overflow.v2i64(<2 x i64>, <2 x i64>) diff --git a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/umul.with.overflow.ll b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/umul.with.overflow.ll index 047c8eff014205..7113dd692f6ac4 100644 --- a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/umul.with.overflow.ll +++ b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/umul.with.overflow.ll @@ -1,58 +1,89 @@ -; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV +; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %} -; This test fails due to change in code patterns that SPIR-V Backend emits for -; "Arithmetic with Overflow" intrinsics. -; XFAIL: * +; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %} -; CHECK-SPIRV: OpName %[[#NAME_UMUL_FUNC_8:]] "spirv.llvm_umul_with_overflow_i8" -; CHECK-SPIRV: OpName %[[#NAME_UMUL_FUNC_32:]] "spirv.llvm_umul_with_overflow_i32" -; CHECK-SPIRV: OpName %[[#NAME_UMUL_FUNC_VEC_I64:]] "spirv.llvm_umul_with_overflow_v2i64" +; CHECK-DAG: %[[Char:.*]] = OpTypeInt 8 0 +; CHECK-DAG: %[[Void:.*]] = OpTypeVoid +; CHECK-DAG: %[[PtrChar:.*]] = OpTypePointer Function %[[Char]] +; CHECK-DAG: %[[StructChar:.*]] = OpTypeStruct %[[Char]] %[[Char]] +; CHECK-DAG: %[[ZeroChar:.*]] = OpConstant %[[Char]] 0 +; CHECK-DAG: %[[Int:.*]] = OpTypeInt 32 0 +; CHECK-DAG: %[[PtrInt:.*]] = OpTypePointer Function %[[Int]] +; CHECK-DAG: %[[StructInt:.*]] = OpTypeStruct %[[Int]] %[[Int]] +; CHECK-DAG: %[[ZeroInt:.*]] = OpConstant %[[Int]] 0 +; CHECK-DAG: %[[Bool:.*]] = OpTypeBool +; CHECK-DAG: %[[V2Bool:.*]] = OpTypeVector %[[Bool]] 2 +; CHECK-DAG: %[[Long:.*]] = OpTypeInt 64 0 +; CHECK-DAG: %[[V2Long:.*]] = OpTypeVector %[[Long]] 2 +; CHECK-DAG: %[[PtrV2Long:.*]] = OpTypePointer Function %[[V2Long]] +; CHECK-DAG: %[[StructV2Long:.*]] = OpTypeStruct %[[V2Long]] %[[V2Long]] +; CHECK-DAG: %[[ZeroV2Long:.*]] = OpConstantNull %[[V2Long]] -define dso_local spir_func void @_Z4foo8hhPh(i8 zeroext %a, i8 zeroext %b, i8* nocapture %c) local_unnamed_addr { +; CHECK: OpFunction +; CHECK: %[[A:.*]] = OpFunctionParameter %[[Char]] +; CHECK: %[[B:.*]] = OpFunctionParameter %[[Char]] +; CHECK: %[[Ptr:.*]] = OpFunctionParameter %[[PtrChar]] +; CHECK: %[[Struct:.*]] = OpUMulExtended %[[StructChar]] %[[A]] %[[B]] +; CHECK: %[[Val:.*]] = OpCompositeExtract %[[Char]] %[[Struct]] 0 +; CHECK: %[[Over:.*]] = OpCompositeExtract %[[Char]] %[[Struct]] 1 +; CHECK: %[[IsOver:.*]] = OpINotEqual %[[Bool]] %[[Over]] %[[ZeroChar]] +; CHECK: %[[Res:.*]] = OpSelect %[[Char]] %[[IsOver]] %[[ZeroChar]] %[[Val]] +; CHECK: OpStore %[[Ptr]] %[[Res]] Aligned 1 +; CHECK: OpReturn +define dso_local spir_func void @umulo_i8(i8 zeroext %a, i8 zeroext %b, ptr nocapture %c) local_unnamed_addr { entry: - ; CHECK-SPIRV: %[[#]] = OpFunctionCall %[[#]] %[[#NAME_UMUL_FUNC_8]] %umul = tail call { i8, i1 } @llvm.umul.with.overflow.i8(i8 %a, i8 %b) %cmp = extractvalue { i8, i1 } %umul, 1 %umul.value = extractvalue { i8, i1 } %umul, 0 %storemerge = select i1 %cmp, i8 0, i8 %umul.value - store i8 %storemerge, i8* %c, align 1 + store i8 %storemerge, ptr %c, align 1 ret void } -define dso_local spir_func void @_Z5foo32jjPj(i32 %a, i32 %b, i32* nocapture %c) local_unnamed_addr { +; CHECK: OpFunction +; CHECK: %[[A2:.*]] = OpFunctionParameter %[[Int]] +; CHECK: %[[B2:.*]] = OpFunctionParameter %[[Int]] +; CHECK: %[[Ptr2:.*]] = OpFunctionParameter %[[PtrInt]] +; CHECK: %[[Struct2:.*]] = OpUMulExtended %[[StructInt]] %[[B2]] %[[A2]] +; CHECK: %[[Val2:.*]] = OpCompositeExtract %[[Int]] %[[Struct2]] 0 +; CHECK: %[[Over2:.*]] = OpCompositeExtract %[[Int]] %[[Struct2]] 1 +; CHECK: %[[IsOver2:.*]] = OpINotEqual %[[Bool]] %[[Over2]] %[[ZeroInt]] +; CHECK: %[[Res2:.*]] = OpSelect %[[Int]] %[[IsOver2]] %[[ZeroInt]] %[[Val2]] +; CHECK: OpStore %[[Ptr2]] %[[Res2]] Aligned 4 +; CHECK: OpReturn +define dso_local spir_func void @umulo_i32(i32 %a, i32 %b, ptr nocapture %c) local_unnamed_addr { entry: - ; CHECK-SPIRV: %[[#]] = OpFunctionCall %[[#]] %[[#NAME_UMUL_FUNC_32]] %umul = tail call { i32, i1 } @llvm.umul.with.overflow.i32(i32 %b, i32 %a) %umul.val = extractvalue { i32, i1 } %umul, 0 %umul.ov = extractvalue { i32, i1 } %umul, 1 %spec.select = select i1 %umul.ov, i32 0, i32 %umul.val - store i32 %spec.select, i32* %c, align 4 + store i32 %spec.select, ptr %c, align 4 ret void } -define dso_local spir_func void @umulo_v2i64(<2 x i64> %a, <2 x i64> %b, <2 x i64>* %p) nounwind { - ; CHECK-SPIRV: %[[#]] = OpFunctionCall %[[#]] %[[#NAME_UMUL_FUNC_VEC_I64]] +; CHECK: OpFunction +; CHECK: %[[A3:.*]] = OpFunctionParameter %[[V2Long]] +; CHECK: %[[B3:.*]] = OpFunctionParameter %[[V2Long]] +; CHECK: %[[Ptr3:.*]] = OpFunctionParameter %[[PtrV2Long]] +; CHECK: %[[Struct3:.*]] = OpUMulExtended %[[StructV2Long]] %[[A3]] %[[B3]] +; CHECK: %[[Val3:.*]] = OpCompositeExtract %[[V2Long]] %[[Struct3]] 0 +; CHECK: %[[Over3:.*]] = OpCompositeExtract %[[V2Long]] %[[Struct3]] 1 +; CHECK: %[[IsOver3:.*]] = OpINotEqual %[[V2Bool]] %[[Over3]] %[[ZeroV2Long]] +; CHECK: %[[Res3:.*]] = OpSelect %[[V2Long]] %[[IsOver3]] %[[ZeroV2Long]] %[[Val3]] +; CHECK: OpStore %[[Ptr3]] %[[Res3]] Aligned 16 +; CHECK: OpReturn +define dso_local spir_func void @umulo_v2i64(<2 x i64> %a, <2 x i64> %b, ptr %p) nounwind { %umul = call {<2 x i64>, <2 x i1>} @llvm.umul.with.overflow.v2i64(<2 x i64> %a, <2 x i64> %b) %umul.val = extractvalue {<2 x i64>, <2 x i1>} %umul, 0 %umul.ov = extractvalue {<2 x i64>, <2 x i1>} %umul, 1 %zero = alloca <2 x i64>, align 16 %spec.select = select <2 x i1> %umul.ov, <2 x i64> , <2 x i64> %umul.val - store <2 x i64> %spec.select, <2 x i64>* %p + store <2 x i64> %spec.select, ptr %p ret void } -; CHECK-SPIRV: %[[#NAME_UMUL_FUNC_8]] = OpFunction %[[#]] -; CHECK-SPIRV: %[[#VAR_A:]] = OpFunctionParameter %[[#]] -; CHECK-SPIRV: %[[#VAR_B:]] = OpFunctionParameter %[[#]] -; CHECK-SPIRV: %[[#MUL_RES:]] = OpIMul %[[#]] %[[#VAR_A]] %[[#VAR_B]] -; CHECK-SPIRV: %[[#DIV_RES:]] = OpUDiv %[[#]] %[[#MUL_RES]] %[[#VAR_A]] -; CHECK-SPIRV: %[[#CMP_RES:]] = OpINotEqual %[[#]] %[[#VAR_A]] %[[#DIV_RES]] -; CHECK-SPIRV: %[[#INSERT_RES:]] = OpCompositeInsert %[[#]] %[[#MUL_RES]] -; CHECK-SPIRV: %[[#INSERT_RES_1:]] = OpCompositeInsert %[[#]] %[[#CMP_RES]] %[[#INSERT_RES]] -; CHECK-SPIRV: OpReturnValue %[[#INSERT_RES_1]] - -declare { i8, i1 } @llvm.umul.with.overflow.i8(i8, i8) - -declare { i32, i1 } @llvm.umul.with.overflow.i32(i32, i32) - +declare {i8, i1} @llvm.umul.with.overflow.i8(i8, i8) +declare {i32, i1} @llvm.umul.with.overflow.i32(i32, i32) declare {<2 x i64>, <2 x i1>} @llvm.umul.with.overflow.v2i64(<2 x i64>, <2 x i64>) diff --git a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/usub.with.overflow.ll b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/usub.with.overflow.ll new file mode 100644 index 00000000000000..963dd70f606b6e --- /dev/null +++ b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/usub.with.overflow.ll @@ -0,0 +1,89 @@ +; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %} + +; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %} + +; CHECK-DAG: %[[Char:.*]] = OpTypeInt 8 0 +; CHECK-DAG: %[[Void:.*]] = OpTypeVoid +; CHECK-DAG: %[[PtrChar:.*]] = OpTypePointer Function %[[Char]] +; CHECK-DAG: %[[StructChar:.*]] = OpTypeStruct %[[Char]] %[[Char]] +; CHECK-DAG: %[[ZeroChar:.*]] = OpConstant %[[Char]] 0 +; CHECK-DAG: %[[Int:.*]] = OpTypeInt 32 0 +; CHECK-DAG: %[[PtrInt:.*]] = OpTypePointer Function %[[Int]] +; CHECK-DAG: %[[StructInt:.*]] = OpTypeStruct %[[Int]] %[[Int]] +; CHECK-DAG: %[[ZeroInt:.*]] = OpConstant %[[Int]] 0 +; CHECK-DAG: %[[Bool:.*]] = OpTypeBool +; CHECK-DAG: %[[V2Bool:.*]] = OpTypeVector %[[Bool]] 2 +; CHECK-DAG: %[[Long:.*]] = OpTypeInt 64 0 +; CHECK-DAG: %[[V2Long:.*]] = OpTypeVector %[[Long]] 2 +; CHECK-DAG: %[[PtrV2Long:.*]] = OpTypePointer Function %[[V2Long]] +; CHECK-DAG: %[[StructV2Long:.*]] = OpTypeStruct %[[V2Long]] %[[V2Long]] +; CHECK-DAG: %[[ZeroV2Long:.*]] = OpConstantNull %[[V2Long]] + +; CHECK: OpFunction +; CHECK: %[[A:.*]] = OpFunctionParameter %[[Char]] +; CHECK: %[[B:.*]] = OpFunctionParameter %[[Char]] +; CHECK: %[[Ptr:.*]] = OpFunctionParameter %[[PtrChar]] +; CHECK: %[[Struct:.*]] = OpISubBorrow %[[StructChar]] %[[A]] %[[B]] +; CHECK: %[[Val:.*]] = OpCompositeExtract %[[Char]] %[[Struct]] 0 +; CHECK: %[[Over:.*]] = OpCompositeExtract %[[Char]] %[[Struct]] 1 +; CHECK: %[[IsOver:.*]] = OpINotEqual %[[Bool]] %[[Over]] %[[ZeroChar]] +; CHECK: %[[Res:.*]] = OpSelect %[[Char]] %[[IsOver]] %[[ZeroChar]] %[[Val]] +; CHECK: OpStore %[[Ptr]] %[[Res]] Aligned 1 +; CHECK: OpReturn +define dso_local spir_func void @umulo_i8(i8 zeroext %a, i8 zeroext %b, ptr nocapture %c) local_unnamed_addr { +entry: + %umul = tail call { i8, i1 } @llvm.usub.with.overflow.i8(i8 %a, i8 %b) + %cmp = extractvalue { i8, i1 } %umul, 1 + %umul.value = extractvalue { i8, i1 } %umul, 0 + %storemerge = select i1 %cmp, i8 0, i8 %umul.value + store i8 %storemerge, ptr %c, align 1 + ret void +} + +; CHECK: OpFunction +; CHECK: %[[A2:.*]] = OpFunctionParameter %[[Int]] +; CHECK: %[[B2:.*]] = OpFunctionParameter %[[Int]] +; CHECK: %[[Ptr2:.*]] = OpFunctionParameter %[[PtrInt]] +; CHECK: %[[Struct2:.*]] = OpISubBorrow %[[StructInt]] %[[B2]] %[[A2]] +; CHECK: %[[Val2:.*]] = OpCompositeExtract %[[Int]] %[[Struct2]] 0 +; CHECK: %[[Over2:.*]] = OpCompositeExtract %[[Int]] %[[Struct2]] 1 +; CHECK: %[[IsOver2:.*]] = OpINotEqual %[[Bool]] %[[Over2]] %[[ZeroInt]] +; CHECK: %[[Res2:.*]] = OpSelect %[[Int]] %[[IsOver2]] %[[ZeroInt]] %[[Val2]] +; CHECK: OpStore %[[Ptr2]] %[[Res2]] Aligned 4 +; CHECK: OpReturn +define dso_local spir_func void @umulo_i32(i32 %a, i32 %b, ptr nocapture %c) local_unnamed_addr { +entry: + %umul = tail call { i32, i1 } @llvm.usub.with.overflow.i32(i32 %b, i32 %a) + %umul.val = extractvalue { i32, i1 } %umul, 0 + %umul.ov = extractvalue { i32, i1 } %umul, 1 + %spec.select = select i1 %umul.ov, i32 0, i32 %umul.val + store i32 %spec.select, ptr %c, align 4 + ret void +} + +; CHECK: OpFunction +; CHECK: %[[A3:.*]] = OpFunctionParameter %[[V2Long]] +; CHECK: %[[B3:.*]] = OpFunctionParameter %[[V2Long]] +; CHECK: %[[Ptr3:.*]] = OpFunctionParameter %[[PtrV2Long]] +; CHECK: %[[Struct3:.*]] = OpISubBorrow %[[StructV2Long]] %[[A3]] %[[B3]] +; CHECK: %[[Val3:.*]] = OpCompositeExtract %[[V2Long]] %[[Struct3]] 0 +; CHECK: %[[Over3:.*]] = OpCompositeExtract %[[V2Long]] %[[Struct3]] 1 +; CHECK: %[[IsOver3:.*]] = OpINotEqual %[[V2Bool]] %[[Over3]] %[[ZeroV2Long]] +; CHECK: %[[Res3:.*]] = OpSelect %[[V2Long]] %[[IsOver3]] %[[ZeroV2Long]] %[[Val3]] +; CHECK: OpStore %[[Ptr3]] %[[Res3]] Aligned 16 +; CHECK: OpReturn +define dso_local spir_func void @umulo_v2i64(<2 x i64> %a, <2 x i64> %b, ptr %p) nounwind { + %umul = call {<2 x i64>, <2 x i1>} @llvm.usub.with.overflow.v2i64(<2 x i64> %a, <2 x i64> %b) + %umul.val = extractvalue {<2 x i64>, <2 x i1>} %umul, 0 + %umul.ov = extractvalue {<2 x i64>, <2 x i1>} %umul, 1 + %zero = alloca <2 x i64>, align 16 + %spec.select = select <2 x i1> %umul.ov, <2 x i64> , <2 x i64> %umul.val + store <2 x i64> %spec.select, ptr %p + ret void +} + +declare {i8, i1} @llvm.usub.with.overflow.i8(i8, i8) +declare {i32, i1} @llvm.usub.with.overflow.i32(i32, i32) +declare {<2 x i64>, <2 x i1>} @llvm.usub.with.overflow.v2i64(<2 x i64>, <2 x i64>) From de45e6ebd9f0ebdd10be6e256684a55c4257f11b Mon Sep 17 00:00:00 2001 From: "Levytskyy, Vyacheslav" Date: Wed, 18 Sep 2024 01:31:00 -0700 Subject: [PATCH 09/16] update docs --- llvm/docs/SPIRVUsage.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/llvm/docs/SPIRVUsage.rst b/llvm/docs/SPIRVUsage.rst index 0f0b21fb237703..9e53bf66d0e627 100644 --- a/llvm/docs/SPIRVUsage.rst +++ b/llvm/docs/SPIRVUsage.rst @@ -275,6 +275,10 @@ SPIR-V backend, along with their descriptions and argument details. - None - `[Type, Vararg]` - Assigns names to types or values, enhancing readability and debuggability of SPIR-V code. Not emitted directly but used for metadata enrichment. + * - `int_spv_value_md` + - None + - `[Metadata]` + - Assigns a set of attributes (such as name and data type) to a value that is an argument of the associated `llvm.fake.use` intrinsic call. The latter is used as a mean to map virtual registers created by IRTranslator to the original value. * - `int_spv_assign_decoration` - None - `[Type, Metadata]` From f3e03048fa7dd4639d5c2266d33a0777a7b9e5b5 Mon Sep 17 00:00:00 2001 From: "Levytskyy, Vyacheslav" Date: Wed, 18 Sep 2024 02:24:32 -0700 Subject: [PATCH 10/16] add specific test cases: (1) example of llvm intrinsics originated from an optimization pass; (2) key translation steps with checks of LLVM IR / gMIR / SPIRV code patterns --- .../SPIRV/optimizations/add-check-overflow.ll | 56 ++++++++++++++++ .../SPIRV/passes/translate-aggregate-uaddo.ll | 64 +++++++++++++++++++ 2 files changed, 120 insertions(+) create mode 100644 llvm/test/CodeGen/SPIRV/optimizations/add-check-overflow.ll create mode 100644 llvm/test/CodeGen/SPIRV/passes/translate-aggregate-uaddo.ll diff --git a/llvm/test/CodeGen/SPIRV/optimizations/add-check-overflow.ll b/llvm/test/CodeGen/SPIRV/optimizations/add-check-overflow.ll new file mode 100644 index 00000000000000..22c51a56057966 --- /dev/null +++ b/llvm/test/CodeGen/SPIRV/optimizations/add-check-overflow.ll @@ -0,0 +1,56 @@ +; This test aims to check ability to support "Arithmetic with Overflow" intrinsics +; in the special case when those intrinsics are being generated by the CodeGenPrepare; +; pass during translations with optimization (note -O3 in llc arguments). + +; RUN: llc -O3 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s +; RUN: %if spirv-tools %{ llc -O3 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %} + +; RUN: llc -O3 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s +; RUN: %if spirv-tools %{ llc -O3 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %} + +; CHECK-DAG: OpName %[[Val:.*]] "math" +; CHECK-DAG: OpName %[[IsOver:.*]] "ov" +; CHECK-DAG: %[[Int:.*]] = OpTypeInt 32 0 +; CHECK-DAG: %[[Char:.*]] = OpTypeInt 8 0 +; CHECK-DAG: %[[PtrChar:.*]] = OpTypePointer Generic %[[Char]] +; CHECK-DAG: %[[Bool:.*]] = OpTypeBool +; CHECK-DAG: %[[Struct:.*]] = OpTypeStruct %[[Int]] %[[Int]] +; CHECK-DAG: %[[Const1:.*]] = OpConstant %[[Int]] 1 +; CHECK-DAG: %[[Const42:.*]] = OpConstant %[[Char]] 42 +; CHECK-DAG: %[[Zero:.*]] = OpConstantNull %[[Int]] + +; CHECK: OpFunction +; CHECK: %[[A:.*]] = OpFunctionParameter %[[Int]] +; CHECK: %[[Ptr:.*]] = OpFunctionParameter %[[PtrChar]] +; CHECK: %[[#]] = OpLabel +; CHECK: OpBranch %[[#]] +; CHECK: %[[#]] = OpLabel +; CHECK: %[[PhiRes:.*]] = OpPhi %[[Int]] %[[A]] %[[#]] %[[Val]] %[[#]] +; CHECK: %[[AggRes:.*]] = OpIAddCarry %[[Struct]] %[[PhiRes]] %[[Const1]] +; CHECK: %[[Val]] = OpCompositeExtract %[[Int]] %[[AggRes]] 0 +; CHECK: %[[Over:.*]] = OpCompositeExtract %[[Int]] %[[AggRes]] 1 +; CHECK: %[[IsOver]] = OpINotEqual %[[Bool:.*]] %[[Over]] %[[Zero]] +; CHECK: OpBranchConditional %[[IsOver]] %[[#]] %[[#]] +; CHECK: OpStore %[[Ptr]] %[[Const42]] Aligned 1 +; CHECK: OpBranch %[[#]] +; CHECK: %[[#]] = OpLabel +; CHECK: OpReturnValue %[[Val]] +; CHECK: OpFunctionEnd + +define spir_func i32 @foo(i32 %a, ptr addrspace(4) %p) { +entry: + br label %l1 + +l1: + %e = phi i32 [ %a, %entry ], [ %i, %body ] + %i = add nsw i32 %e, 1 + %fl = icmp eq i32 %i, 0 + br i1 %fl, label %exit, label %body + +body: + store i8 42, ptr addrspace(4) %p + br label %l1 + +exit: + ret i32 %i +} diff --git a/llvm/test/CodeGen/SPIRV/passes/translate-aggregate-uaddo.ll b/llvm/test/CodeGen/SPIRV/passes/translate-aggregate-uaddo.ll new file mode 100644 index 00000000000000..cd4d9325c76599 --- /dev/null +++ b/llvm/test/CodeGen/SPIRV/passes/translate-aggregate-uaddo.ll @@ -0,0 +1,64 @@ +; This test shows how value attributes are being passed during different translation steps. +; See also test/CodeGen/SPIRV/optimizations/add-check-overflow.ll. + +; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -print-after=prepare-functions 2>&1 | FileCheck %s --check-prefix=CHECK-PREPARE +; Intrinsics with aggregate return type are not substituted/removed. +; CHECK-PREPARE: @llvm.uadd.with.overflow.i32 + +; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -print-after=emit-intrinsics 2>&1 | FileCheck %s --check-prefix=CHECK-IR +; Aggregate data are wrapped into @llvm.fake.use(), +; and their attributes are packed into a metadata for @llvm.spv.value.md(). +; CHECK-IR: %[[R1:.*]] = call { i32, i1 } @llvm.uadd.with.overflow.i32 +; CHECK-IR: call void @llvm.spv.value.md(metadata !0) +; CHECK-IR: call void (...) @llvm.fake.use({ i32, i1 } %[[R1]]) +; CHECK-IR: %math = extractvalue { i32, i1 } %[[R1]], 0 +; CHECK-IR: %ov = extractvalue { i32, i1 } %[[R1]], 1 +; Type/Name attributes of the value. +; CHECK-IR: !0 = !{{[{]}}!1, !""{{[}]}} +; Origin data type of the value. +; CHECK-IR: !1 = !{{[{]}}{{[{]}} i32, i1 {{[}]}} poison{{[}]}} + +; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -print-after=irtranslator 2>&1 | FileCheck %s --check-prefix=CHECK-GMIR +; Required info succeeded to get through IRTranslator. +; CHECK-GMIR: %[[phires:.*]]:_(s32) = G_PHI +; CHECK-GMIR: %[[math:.*]]:id(s32), %[[ov:.*]]:_(s1) = G_UADDO %[[phires]]:_, %[[#]]:_ +; CHECK-GMIR: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.spv.value.md), !0 +; CHECK-GMIR: FAKE_USE %[[math]]:id(s32), %[[ov]]:_(s1) + +; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -print-after=spirv-prelegalizer 2>&1 | FileCheck %s --check-prefix=CHECK-PRE +; Internal service instructions are consumed. +; CHECK-PRE: G_UADDO +; CHECK-PRE-NO: llvm.spv.value.md +; CHECK-PRE-NO: FAKE_USE + +; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -print-after=instruction-select 2>&1 | FileCheck %s --check-prefix=CHECK-ISEL +; Names and types are restored and correctly encoded. Correct instruction selection is completed. +; CHECK-ISEL-DAG: %[[int32:.*]]:type = OpTypeInt 32, 0 +; CHECK-ISEL-DAG: %[[struct:.*]]:type = OpTypeStruct %[[int32]]:type, %[[int32]]:type +; CHECK-ISEL-DAG: %[[bool:.*]]:type = OpTypeBool +; CHECK-ISEL-DAG: %[[zero32:.*]]:iid = OpConstantNull %[[int32]]:type +; CHECK-ISEL-DAG: %[[res:.*]]:iid = OpIAddCarryS %[[struct]]:type +; CHECK-ISEL-DAG: %[[math:.*]]:id = OpCompositeExtract %[[int32]]:type, %[[res]]:iid, 0 +; CHECK-ISEL-DAG: %[[ov32:.*]]:iid = OpCompositeExtract %[[int32]]:type, %[[res]]:iid, 1 +; CHECK-ISEL-DAG: %[[ov:.*]]:iid = OpINotEqual %[[bool]]:type, %[[ov32]]:iid, %[[zero32:.*]]:iid +; CHECK-ISEL-DAG: OpName %[[math]]:id, 1752457581, 0 +; CHECK-ISEL-DAG: OpName %[[ov]]:iid, 30319 + +define spir_func i32 @foo(i32 %a, ptr addrspace(4) %p) { +entry: + br label %l1 + +l1: ; preds = %body, %entry + %e = phi i32 [ %a, %entry ], [ %math, %body ] + %0 = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 %e, i32 1) + %math = extractvalue { i32, i1 } %0, 0 + %ov = extractvalue { i32, i1 } %0, 1 + br i1 %ov, label %exit, label %body + +body: ; preds = %l1 + store i8 42, ptr addrspace(4) %p, align 1 + br label %l1 + +exit: ; preds = %l1 + ret i32 %math +} From 0e476118349776e9b26f3395fb08fa19a5233957 Mon Sep 17 00:00:00 2001 From: "Levytskyy, Vyacheslav" Date: Wed, 18 Sep 2024 03:12:09 -0700 Subject: [PATCH 11/16] correct insertion points for new instructions --- llvm/docs/SPIRVUsage.rst | 2 +- .../Target/SPIRV/SPIRVInstructionSelector.cpp | 25 +++++++++++-------- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/llvm/docs/SPIRVUsage.rst b/llvm/docs/SPIRVUsage.rst index 9e53bf66d0e627..bb12b05246afba 100644 --- a/llvm/docs/SPIRVUsage.rst +++ b/llvm/docs/SPIRVUsage.rst @@ -278,7 +278,7 @@ SPIR-V backend, along with their descriptions and argument details. * - `int_spv_value_md` - None - `[Metadata]` - - Assigns a set of attributes (such as name and data type) to a value that is an argument of the associated `llvm.fake.use` intrinsic call. The latter is used as a mean to map virtual registers created by IRTranslator to the original value. + - Assigns a set of attributes (such as name and data type) to a value that is the argument of the associated `llvm.fake.use` intrinsic call. The latter is used as a mean to map virtual registers created by IRTranslator to the original value. * - `int_spv_assign_decoration` - None - `[Type, Metadata]` diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp index 9e3cc4fdf01259..ceb7c9c26494ad 100644 --- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp @@ -1143,10 +1143,17 @@ bool SPIRVInstructionSelector::selectOverflowArith(Register ResVReg, // and the two members must be the same type." Type *ResElemTy = cast(ResTy)->getElementType(0); ResTy = StructType::create(SmallVector{ResElemTy, ResElemTy}); - // Build SPIR-V type if needed. + // Build SPIR-V types and constant(s) if needed. MachineIRBuilder MIRBuilder(I); SPIRVType *StructType = GR.getOrCreateSPIRVType( ResTy, MIRBuilder, SPIRV::AccessQualifier::ReadWrite, false); + assert(I.getNumDefs() > 1 && "Not enought operands"); + SPIRVType *BoolType = GR.getOrCreateSPIRVBoolType(I, TII); + unsigned N = GR.getScalarOrVectorComponentCount(ResType); + if (N > 1) + BoolType = GR.getOrCreateSPIRVVectorType(BoolType, N, I, TII); + Register BoolTypeReg = GR.getSPIRVTypeID(BoolType); + Register ZeroReg = buildZerosVal(ResType, I); // A new virtual register to store the result struct. Register StructVReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); MRI->setRegClass(StructVReg, &SPIRV::IDRegClass); @@ -1155,9 +1162,10 @@ bool SPIRVInstructionSelector::selectOverflowArith(Register ResVReg, buildOpName(StructVReg, ResName, MIRBuilder); // Build the arithmetic with overflow instruction. MachineBasicBlock &BB = *I.getParent(); - auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) - .addDef(StructVReg) - .addUse(GR.getSPIRVTypeID(StructType)); + auto MIB = + BuildMI(BB, MIRBuilder.getInsertPt(), I.getDebugLoc(), TII.get(Opcode)) + .addDef(StructVReg) + .addUse(GR.getSPIRVTypeID(StructType)); for (unsigned i = I.getNumDefs(); i < I.getNumOperands(); ++i) MIB.addUse(I.getOperand(i).getReg()); bool Status = MIB.constrainAllUses(TII, TRI, RBI); @@ -1175,16 +1183,11 @@ bool SPIRVInstructionSelector::selectOverflowArith(Register ResVReg, Status &= MIB.constrainAllUses(TII, TRI, RBI); } // Build boolean value from the higher part. - assert(I.getNumDefs() > 1 && "Not enought operands"); - SPIRVType *BoolType = GR.getOrCreateSPIRVBoolType(I, TII); - unsigned N = GR.getScalarOrVectorComponentCount(ResType); - if (N > 1) - BoolType = GR.getOrCreateSPIRVVectorType(BoolType, N, I, TII); Status &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpINotEqual)) .addDef(I.getOperand(1).getReg()) - .addUse(GR.getSPIRVTypeID(BoolType)) + .addUse(BoolTypeReg) .addUse(HigherVReg) - .addUse(buildZerosVal(ResType, I)) + .addUse(ZeroReg) .constrainAllUses(TII, TRI, RBI); return Status; } From e93b7d7363802b321deed7e9d21706bf74ceb9e5 Mon Sep 17 00:00:00 2001 From: "Levytskyy, Vyacheslav" Date: Thu, 19 Sep 2024 02:56:04 -0700 Subject: [PATCH 12/16] apply code review suggestions --- .../lib/Target/SPIRV/SPIRVInstructionSelector.cpp | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp index ceb7c9c26494ad..f9696cd2226c51 100644 --- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp @@ -412,11 +412,22 @@ bool SPIRVInstructionSelector::select(MachineInstr &I) { return false; } +static bool mayApplyGenericSelection(unsigned Opcode) { + switch (Opcode) { + case TargetOpcode::G_CONSTANT: + return false; + case TargetOpcode::G_SADDO: + case TargetOpcode::G_SSUBO: + return true; + } + return isTypeFoldingSupported(Opcode); +} + bool SPIRVInstructionSelector::spvSelect(Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const { const unsigned Opcode = I.getOpcode(); - if (isTypeFoldingSupported(Opcode) && Opcode != TargetOpcode::G_CONSTANT) + if (mayApplyGenericSelection(Opcode)) return selectImpl(I, *CoverageInfo); switch (Opcode) { case TargetOpcode::G_CONSTANT: @@ -1157,7 +1168,7 @@ bool SPIRVInstructionSelector::selectOverflowArith(Register ResVReg, // A new virtual register to store the result struct. Register StructVReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); MRI->setRegClass(StructVReg, &SPIRV::IDRegClass); - // Build the result name of needed. + // Build the result name if needed. if (ResName.size() > 0) buildOpName(StructVReg, ResName, MIRBuilder); // Build the arithmetic with overflow instruction. From 3758ff93cb7f4be28e7456b3c7839cf75b4bc47a Mon Sep 17 00:00:00 2001 From: "Levytskyy, Vyacheslav" Date: Thu, 19 Sep 2024 02:57:09 -0700 Subject: [PATCH 13/16] apply code review suggestions --- llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp | 8 -------- 1 file changed, 8 deletions(-) diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp index f9696cd2226c51..919831d30757f9 100644 --- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp @@ -604,19 +604,11 @@ bool SPIRVInstructionSelector::spvSelect(Register ResVReg, case TargetOpcode::G_USUBSAT: return selectExtInst(ResVReg, ResType, I, CL::u_sub_sat); - case TargetOpcode::G_SADDO: - report_fatal_error( - "Selection of a signed addition with overflow is not implemented", - false); case TargetOpcode::G_UADDO: return selectOverflowArith(ResVReg, ResType, I, ResType->getOpcode() == SPIRV::OpTypeVector ? SPIRV::OpIAddCarryV : SPIRV::OpIAddCarryS); - case TargetOpcode::G_SSUBO: - report_fatal_error( - "Selection of a signed subtraction with overflow is not implemented", - false); case TargetOpcode::G_USUBO: return selectOverflowArith(ResVReg, ResType, I, ResType->getOpcode() == SPIRV::OpTypeVector From ee044e34778676b6723e0bb9f723b7c01a28db24 Mon Sep 17 00:00:00 2001 From: "Levytskyy, Vyacheslav" Date: Thu, 19 Sep 2024 07:18:42 -0700 Subject: [PATCH 14/16] apply code review suggestions --- .../CodeGen/SPIRV/optimizations/add-check-overflow.ll | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/llvm/test/CodeGen/SPIRV/optimizations/add-check-overflow.ll b/llvm/test/CodeGen/SPIRV/optimizations/add-check-overflow.ll index 22c51a56057966..1a630f77a44c5d 100644 --- a/llvm/test/CodeGen/SPIRV/optimizations/add-check-overflow.ll +++ b/llvm/test/CodeGen/SPIRV/optimizations/add-check-overflow.ll @@ -41,16 +41,16 @@ define spir_func i32 @foo(i32 %a, ptr addrspace(4) %p) { entry: br label %l1 +body: + store i8 42, ptr addrspace(4) %p + br label %l1 + l1: %e = phi i32 [ %a, %entry ], [ %i, %body ] %i = add nsw i32 %e, 1 %fl = icmp eq i32 %i, 0 br i1 %fl, label %exit, label %body -body: - store i8 42, ptr addrspace(4) %p - br label %l1 - exit: ret i32 %i } From f26041e957e6ed3008dc1ef406b661d9b96f4f60 Mon Sep 17 00:00:00 2001 From: "Levytskyy, Vyacheslav" Date: Fri, 20 Sep 2024 02:28:45 -0700 Subject: [PATCH 15/16] apply code review suggestions --- llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp | 27 +++---------------- llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h | 4 +-- .../Target/SPIRV/SPIRVInstructionSelector.cpp | 2 +- llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp | 4 +-- 4 files changed, 9 insertions(+), 28 deletions(-) diff --git a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp index 3ca1ea5024b2ee..75e6f403239d74 100644 --- a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp @@ -189,30 +189,11 @@ bool isConvergenceIntrinsic(const Instruction *I) { II->getIntrinsicID() == Intrinsic::experimental_convergence_anchor; } -bool isInternalNonVoidIntrinsic(const Value *I) { - if (const auto *II = dyn_cast(I)) - switch (II->getIntrinsicID()) { - case Intrinsic::spv_cmpxchg: - case Intrinsic::spv_const_composite: - case Intrinsic::spv_track_constant: - case Intrinsic::spv_load: - case Intrinsic::spv_extractv: - case Intrinsic::spv_insertv: - case Intrinsic::spv_extractelt: - case Intrinsic::spv_insertelt: - case Intrinsic::spv_bitcast: - case Intrinsic::spv_ptrcast: - case Intrinsic::spv_alloca: - case Intrinsic::spv_alloca_array: - case Intrinsic::spv_undef: - return true; - } - return false; -} - bool allowEmitFakeUse(const Value *Arg) { - if (isInternalNonVoidIntrinsic(Arg)) - return false; + if (const auto *II = dyn_cast(Arg)) + if (Function *F = II->getCalledFunction()) + if (F->getName().starts_with("llvm.spv.")) + return false; if (dyn_cast(Arg) || dyn_cast(Arg) || dyn_cast(Arg)) return false; diff --git a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h index ba104a87064b2f..4849d061688a02 100644 --- a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h +++ b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h @@ -56,7 +56,7 @@ class SPIRVGlobalRegistry { // (see `SPIRVPrepareFunctions::removeAggregateTypesFromSignature()`) DenseMap MutatedAggRet; // map an instruction to its value's attributes (type, name) - DenseMap> ValueAttrs; + DenseMap> ValueAttrs; // Look for an equivalent of the newType in the map. Return the equivalent // if it's found, otherwise insert newType to the map and return the type. @@ -181,7 +181,7 @@ class SPIRVGlobalRegistry { // A registry of value's attributes (type, name) // - Add a record. - void addValueAttrs(MachineInstr *Key, std::pair Val) { + void addValueAttrs(MachineInstr *Key, std::pair Val) { ValueAttrs[Key] = Val; } // - Find a record. diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp index 919831d30757f9..3d6fa12af36d7e 100644 --- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp @@ -1139,7 +1139,7 @@ bool SPIRVInstructionSelector::selectOverflowArith(Register ResVReg, if (!GR.findValueAttrs(&I, ResTy, ResName)) report_fatal_error( "Not enough info to select the arithmetic with overflow instruction"); - if (!ResTy->isStructTy()) + if (!ResTy || !ResTy->isStructTy()) report_fatal_error("Expect struct type result for the arithmetic " "with overflow instruction"); // "Result Type must be from OpTypeStruct. The struct must have two members, diff --git a/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp b/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp index 90020fdc44a548..20f0e483d12b6c 100644 --- a/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp @@ -482,11 +482,11 @@ generateAssignInstrs(MachineFunction &MF, SPIRVGlobalRegistry *GR, cast(MD->getOperand(1))->getString(); const MDNode *TypeMD = cast(MD->getOperand(0)); Type *ValueTy = getMDOperandAsType(TypeMD, 0); - GR->addValueAttrs(Def, std::make_pair(ValueTy, ValueName)); + GR->addValueAttrs(Def, std::make_pair(ValueTy, ValueName.str())); } + ToErase.push_back(MdMI); } ToErase.push_back(&MI); - ToErase.push_back(MdMI); } else if (MIOp == TargetOpcode::G_CONSTANT || MIOp == TargetOpcode::G_FCONSTANT || MIOp == TargetOpcode::G_BUILD_VECTOR) { From c7a8975d2bebbf93744dfde1f161882ae76a2e9b Mon Sep 17 00:00:00 2001 From: "Levytskyy, Vyacheslav" Date: Fri, 20 Sep 2024 02:51:08 -0700 Subject: [PATCH 16/16] fix weird merge issue --- llvm/lib/Target/SPIRV/SPIRVUtils.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/llvm/lib/Target/SPIRV/SPIRVUtils.cpp b/llvm/lib/Target/SPIRV/SPIRVUtils.cpp index aaa0572f912d65..b8580fd2630822 100644 --- a/llvm/lib/Target/SPIRV/SPIRVUtils.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVUtils.cpp @@ -589,6 +589,8 @@ bool sortBlocks(Function &F) { } return Modified; +} + MachineInstr *getVRegDef(MachineRegisterInfo &MRI, Register Reg) { MachineInstr *MaybeDef = MRI.getVRegDef(Reg); if (MaybeDef && MaybeDef->getOpcode() == SPIRV::ASSIGN_TYPE)