Add iadd_overflow_trap

cranelift/codegen/meta/src/shared/formats.rs

@@ -20,6 +20,7 @@ pub(crate) struct Formats {
     pub(crate) int_compare: Rc<InstructionFormat>,
     pub(crate) int_compare_imm: Rc<InstructionFormat>,
     pub(crate) int_cond_trap: Rc<InstructionFormat>,
+    pub(crate) int_add_trap: Rc<InstructionFormat>,
     pub(crate) jump: Rc<InstructionFormat>,
     pub(crate) load: Rc<InstructionFormat>,
     pub(crate) load_no_offset: Rc<InstructionFormat>,
@@ -223,6 +224,12 @@ impl Formats {
                 .imm(&imm.trapcode)
                 .build(),
 
+            int_add_trap: Builder::new("IntAddTrap")
+                .value()
+                .value()
+                .imm(&imm.trapcode)
+                .build(),
+
             float_cond_trap: Builder::new("FloatCondTrap")
                 .imm(&imm.floatcc)
                 .value()

cranelift/codegen/meta/src/shared/instructions.rs

@@ -2174,6 +2174,34 @@ pub(crate) fn define(
         .operands_out(vec![a, c_if_out]),
     );
 
+    {
+        let code = &Operand::new("code", &imm.trapcode);
+
+        let i32_64 = &TypeVar::new(
+            "i32_64",
+            "A 32 or 64-bit scalar integer type",
+            TypeSetBuilder::new().ints(32..64).build(),
+        );
+
+        let a = &Operand::new("a", i32_64);
+        let x = &Operand::new("x", i32_64);
+        let y = &Operand::new("y", i32_64);
+        ig.push(
+            Inst::new(
+                "uadd_overflow_trap",
+                r#"
+            Unsigned addition of x and y, trapping if the result overflows.
+
+            Accepts 32 or 64-bit integers, and does not support vector types.
+            "#,
+                &formats.int_add_trap,
+            )
+            .operands_in(vec![x, y, code])
+            .operands_out(vec![a])
+            .can_trap(true),
+        );
+    }
+
     ig.push(
         Inst::new(
             "isub_bin",

cranelift/codegen/src/isa/aarch64/inst.isle

@@ -2772,6 +2772,14 @@
         )
         x))
 
+;; Check for unsigned overflow.
+(decl trap_if_overflow (ProducesFlags TrapCode) Reg)
+(rule (trap_if_overflow producer tc)
+      (with_flags_reg
+        producer
+        (ConsumesFlags.ConsumesFlagsSideEffect
+          (MInst.TrapIf (cond_br_cond (Cond.Hs)) tc))))
+
 (decl sink_atomic_load (Inst) Reg)
 (rule (sink_atomic_load x @ (atomic_load _ addr))
       (let ((_ Unit (sink_inst x)))

cranelift/codegen/src/isa/aarch64/lower.isle

@@ -2378,6 +2378,11 @@
        (add_with_flags ty a b)
        (invalid_reg)))
 
+;;; Rules for `uadd_overflow_trap` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(rule (lower (has_type (fits_in_64 ty) (uadd_overflow_trap a b tc)))
+      (trap_if_overflow (add_with_flags_paired ty a b) tc))
+
 ;;; Rules for `tls_value` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 (rule (lower (tls_value (symbol_value_data name _ _)))

cranelift/codegen/src/isa/aarch64/lower_inst.rs

@@ -259,6 +259,8 @@ pub(crate) fn lower_insn_to_regs(
 
         Opcode::IaddIfcout => implemented_in_isle(ctx),
 
+        Opcode::UaddOverflowTrap => implemented_in_isle(ctx),
+
         Opcode::IaddImm
         | Opcode::ImulImm
         | Opcode::UdivImm

cranelift/codegen/src/isa/riscv64/inst.isle

@@ -1847,22 +1847,22 @@
   (gen_fabs x ty)
   (fpu_rrr (fabs_copy_sign ty) ty x x))
 
-;;; right now only return if overflow.
-(decl lower_uadd_overflow (Reg Reg Type) Reg)
+;;; Returns the sum in the first register, and the overflow test in the second.
+(decl lower_uadd_overflow (Reg Reg Type) ValueRegs)
 
 (rule 1
   (lower_uadd_overflow x y $I64)
-  (let
-    ((tmp Reg (alu_add x y)))
-    (gen_icmp (IntCC.UnsignedLessThan) tmp x $I64)))
+  (let ((tmp Reg (alu_add x y))
+        (test Reg (gen_icmp (IntCC.UnsignedLessThan) tmp x $I64)))
+    (value_regs tmp test)))
 
 (rule
   (lower_uadd_overflow x y (fits_in_32 ty))
-  (let
-    ((tmp_x Reg (ext_int_if_need $false x ty))
-      (tmp_y Reg (ext_int_if_need $false y ty))
-      (sum Reg (alu_add tmp_x tmp_y)))
-    (alu_srli sum (ty_bits ty))))
+  (let ((tmp_x Reg (ext_int_if_need $false x ty))
+        (tmp_y Reg (ext_int_if_need $false y ty))
+        (sum Reg (alu_add tmp_x tmp_y))
+        (test Reg (alu_srli sum (ty_bits ty))))
+    (value_regs sum test)))
 
 (decl inst_output_get (InstOutput u8) ValueRegs)
 (extern constructor inst_output_get inst_output_get)

cranelift/codegen/src/isa/riscv64/lower.isle

@@ -48,6 +48,13 @@
   (lower (has_type (fits_in_64 ty) (iadd_ifcout x y)))
   (output_ifcout (alu_add x y)))
 
+;;; Rules for `uadd_overflow_trap` ;;;;;;;;;;;;;
+(rule
+  (lower (has_type (fits_in_64 ty) (uadd_overflow_trap x y tc)))
+  (let ((res ValueRegs (lower_uadd_overflow x y ty))
+        (_ InstOutput (gen_trapif (value_regs_get res 1) tc)))
+    (value_regs_get res 0)))
+
 
 ;;;; Rules for `isub` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; Base case, simply subtracting things in registers.
@@ -857,8 +864,8 @@
 (rule
   (lower (trapif _ (iadd_ifcout a @ (value_type ty) b) trap_code))
   (let
-    ((test Reg (lower_uadd_overflow a b ty)))
-    (gen_trapif test trap_code)))
+    ((res ValueRegs (lower_uadd_overflow a b ty)))
+    (gen_trapif (value_regs_get res 1) trap_code)))
 
 
 ;;;;;  Rules for `trapff`;;;;;;;;;

cranelift/codegen/src/isa/s390x/inst.isle

@@ -2022,6 +2022,13 @@
             (_ Unit (emit (MInst.AluRRR op dst src1 src2))))
         dst))
 
+;; Helper for emitting `MInst.AluRRR` instructions as flag producers.
+(decl alu_rrr_with_flags_paired (Type ALUOp Reg Reg) ProducesFlags)
+(rule (alu_rrr_with_flags_paired ty op src1 src2)
+      (let ((dst WritableReg (temp_writable_reg ty)))
+        (ProducesFlags.ProducesFlagsReturnsResultWithConsumer
+          (MInst.AluRRR op dst src1 src2) dst)))
+
 ;; Helper for emitting `MInst.AluRRSImm16` instructions.
 (decl alu_rrsimm16 (Type ALUOp Reg i16) Reg)
 (rule (alu_rrsimm16 ty op src imm)
@@ -2036,13 +2043,27 @@
             (_ Unit (emit (MInst.AluRR op dst src1 src2))))
         dst))
 
+;; Helper for emitting `MInst.AluRR` instructions as flag producers.
+(decl alu_rr_with_flags_paired (Type ALUOp Reg Reg) ProducesFlags)
+(rule (alu_rr_with_flags_paired ty op src1 src2)
+      (let ((dst WritableReg (temp_writable_reg ty)))
+        (ProducesFlags.ProducesFlagsReturnsResultWithConsumer
+            (MInst.AluRR op dst src1 src2) dst)))
+
 ;; Helper for emitting `MInst.AluRX` instructions.
 (decl alu_rx (Type ALUOp Reg MemArg) Reg)
 (rule (alu_rx ty op src mem)
       (let ((dst WritableReg (temp_writable_reg ty))
             (_ Unit (emit (MInst.AluRX op dst src mem))))
         dst))
 
+;; Helper for emitting `MInst.AluRX` instructions as flags producers.
+(decl alu_rx_with_flags_paired (Type ALUOp Reg MemArg) ProducesFlags)
+(rule (alu_rx_with_flags_paired ty op src mem)
+      (let ((dst WritableReg (temp_writable_reg ty)))
+        (ProducesFlags.ProducesFlagsReturnsResultWithConsumer
+          (MInst.AluRX op dst src mem) dst)))
+
 ;; Helper for emitting `MInst.AluRSImm16` instructions.
 (decl alu_rsimm16 (Type ALUOp Reg i16) Reg)
 (rule (alu_rsimm16 ty op src imm)
@@ -2064,6 +2085,13 @@
             (_ Unit (emit (MInst.AluRUImm32 op dst src imm))))
         dst))
 
+;; Helper for emitting `MInst.AluRUImm32` instructions as flag producers.
+(decl alu_ruimm32_with_flags_paired (Type ALUOp Reg u32) ProducesFlags)
+(rule (alu_ruimm32_with_flags_paired ty op src imm)
+      (let ((dst WritableReg (temp_writable_reg ty)))
+        (ProducesFlags.ProducesFlagsReturnsResultWithConsumer
+          (MInst.AluRUImm32 op dst src imm) dst)))
+
 ;; Helper for emitting `MInst.AluRUImm16Shifted` instructions.
 (decl alu_ruimm16shifted (Type ALUOp Reg UImm16Shifted) Reg)
 (rule (alu_ruimm16shifted ty op src imm)
@@ -3874,18 +3902,38 @@
 (decl add_logical_reg (Type Reg Reg) Reg)
 (rule (add_logical_reg ty x y) (alu_rrr ty (aluop_add_logical ty) x y))
 
+(decl add_logical_reg_with_flags_paired (Type Reg Reg) ProducesFlags)
+(rule (add_logical_reg_with_flags_paired ty x y)
+      (alu_rrr_with_flags_paired ty (aluop_add_logical ty) x y))
+
 (decl add_logical_reg_zext32 (Type Reg Reg) Reg)
 (rule (add_logical_reg_zext32 ty x y) (alu_rr ty (aluop_add_logical_zext32 ty) x y))
 
+(decl add_logical_reg_zext32_with_flags_paired (Type Reg Reg) ProducesFlags)
+(rule (add_logical_reg_zext32_with_flags_paired ty x y)
+      (alu_rr_with_flags_paired ty (aluop_add_logical_zext32 ty) x y))
+
 (decl add_logical_zimm32 (Type Reg u32) Reg)
 (rule (add_logical_zimm32 ty x y) (alu_ruimm32 ty (aluop_add_logical ty) x y))
 
+(decl add_logical_zimm32_with_flags_paired (Type Reg u32) ProducesFlags)
+(rule (add_logical_zimm32_with_flags_paired ty x y)
+      (alu_ruimm32_with_flags_paired ty (aluop_add_logical ty) x y))
+
 (decl add_logical_mem (Type Reg MemArg) Reg)
 (rule (add_logical_mem ty x y) (alu_rx ty (aluop_add_logical ty) x y))
 
+(decl add_logical_mem_with_flags_paired (Type Reg MemArg) ProducesFlags)
+(rule (add_logical_mem_with_flags_paired ty x y)
+      (alu_rx_with_flags_paired ty (aluop_add_logical ty) x y))
+
 (decl add_logical_mem_zext32 (Type Reg MemArg) Reg)
 (rule (add_logical_mem_zext32 ty x y) (alu_rx ty (aluop_add_logical_zext32 ty) x y))
 
+(decl add_logical_mem_zext32_with_flags_paired (Type Reg MemArg) ProducesFlags)
+(rule (add_logical_mem_zext32_with_flags_paired ty x y)
+      (alu_rx_with_flags_paired ty (aluop_add_logical_zext32 ty) x y))
+
 
 ;; Helpers for generating `sub` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 

cranelift/codegen/src/isa/s390x/lower.isle

@@ -3876,6 +3876,60 @@
         (trap_if_bool (bool (flags_to_producesflags flags) (mask_as_cond 3))
                       trap_code)))
 
+;;;; Rules for `uadd_overflow_trap` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(rule 0 (lower (has_type (fits_in_64 ty) (uadd_overflow_trap x y tc)))
+      (with_flags
+        (add_logical_reg_with_flags_paired ty x y)
+        (trap_if_impl (mask_as_cond 3) tc)))
+
+;; Add a register an a zero-extended register.
+(rule 4 (lower (has_type (fits_in_64 ty)
+                         (uadd_overflow_trap x (zext32_value y) tc)))
+      (with_flags
+        (add_logical_reg_zext32_with_flags_paired ty x y)
+        (trap_if_impl (mask_as_cond 3) tc)))
+(rule 8 (lower (has_type (fits_in_64 ty)
+                         (uadd_overflow_trap (zext32_value x) y tc)))
+      (with_flags
+        (add_logical_reg_zext32_with_flags_paired ty y x)
+        (trap_if_impl (mask_as_cond 3) tc)))
+
+;; Add a register and an immediate
+(rule 3 (lower (has_type (fits_in_64 ty)
+                         (uadd_overflow_trap x (u32_from_value y) tc)))
+      (with_flags
+        (add_logical_zimm32_with_flags_paired ty x y)
+        (trap_if_impl (mask_as_cond 3) tc)))
+(rule 7 (lower (has_type (fits_in_64 ty)
+                         (uadd_overflow_trap (u32_from_value x) y tc)))
+      (with_flags
+        (add_logical_zimm32_with_flags_paired ty y x)
+        (trap_if_impl (mask_as_cond 3) tc)))
+
+;; Add a register and memory (32/64-bit types).
+(rule 2 (lower (has_type (fits_in_64 ty)
+                         (uadd_overflow_trap x (sinkable_load_32_64 y) tc)))
+      (with_flags
+        (add_logical_mem_with_flags_paired ty x (sink_load y))
+        (trap_if_impl (mask_as_cond 3) tc)))
+(rule 6 (lower (has_type (fits_in_64 ty)
+                         (uadd_overflow_trap (sinkable_load_32_64 x) y tc)))
+      (with_flags
+        (add_logical_mem_with_flags_paired ty y (sink_load x))
+        (trap_if_impl (mask_as_cond 3) tc)))
+
+;; Add a register and zero-extended memory.
+(rule 1 (lower (has_type (fits_in_64 ty)
+                         (uadd_overflow_trap x (sinkable_uload32 y) tc)))
+      (with_flags
+        (add_logical_mem_zext32_with_flags_paired ty x (sink_uload32 y))
+        (trap_if_impl (mask_as_cond 3) tc)))
+(rule 5 (lower (has_type (fits_in_64 ty)
+                         (uadd_overflow_trap (sinkable_uload32 x) y tc)))
+      (with_flags
+        (add_logical_mem_zext32_with_flags_paired ty y (sink_uload32 x))
+        (trap_if_impl (mask_as_cond 3) tc)))
 
 ;;;; Rules for `return` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 

cranelift/codegen/src/isa/s390x/lower.rs

@@ -241,6 +241,7 @@ impl LowerBackend for S390xBackend {
             | Opcode::IaddCout
             | Opcode::IaddCarry
             | Opcode::IaddIfcarry
+            | Opcode::UaddOverflowTrap
             | Opcode::IsubBin
             | Opcode::IsubIfbin
             | Opcode::IsubBout

cranelift/codegen/src/isa/x64/lower.isle

@@ -1352,6 +1352,41 @@
 (rule (lower (trap code))
       (side_effect (x64_ud2 code)))
 
+;;;; Rules for `uadd_overflow_trap` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(rule (lower (has_type (fits_in_64 ty) (uadd_overflow_trap a b tc)))
+      (with_flags
+        (x64_add_with_flags_paired ty a b)
+        (trap_if (CC.B) tc)))
+
+;; Add a register and an immediate.
+
+(rule 1 (lower (has_type (fits_in_64 ty)
+                         (uadd_overflow_trap a (simm32_from_value b) tc)))
+      (with_flags
+        (x64_add_with_flags_paired ty a b)
+        (trap_if (CC.B) tc)))
+
+(rule 2 (lower (has_type (fits_in_64 ty)
+                         (uadd_overflow_trap (simm32_from_value a) b tc)))
+      (with_flags
+        (x64_add_with_flags_paired ty b a)
+        (trap_if (CC.B) tc)))
+
+;; Add a register and memory.
+
+(rule 3 (lower (has_type (fits_in_64 ty)
+                         (uadd_overflow_trap a (sinkable_load b) tc)))
+      (with_flags
+        (x64_add_with_flags_paired ty a (sink_load_to_gpr_mem_imm b))
+        (trap_if (CC.B) tc)))
+
+(rule 4 (lower (has_type (fits_in_64 ty)
+                         (uadd_overflow_trap (sinkable_load a) b tc)))
+      (with_flags
+        (x64_add_with_flags_paired ty b (sink_load_to_gpr_mem_imm a))
+        (trap_if (CC.B) tc)))
+
 ;;;; Rules for `trapif` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 ;; The flags must not have been clobbered by any other instruction between the

cranelift/codegen/src/isa/x64/lower.rs

@@ -527,6 +527,7 @@ fn lower_insn_to_regs(
         | Opcode::IsubIfbout
         | Opcode::IsubBorrow
         | Opcode::IsubIfborrow
+        | Opcode::UaddOverflowTrap
         | Opcode::BandImm
         | Opcode::BorImm
         | Opcode::BxorImm

cranelift/codegen/src/legalizer/heap.rs

@@ -74,12 +74,9 @@ fn dynamic_addr(
     } else {
         // We need an overflow check for the adjusted offset.
         let access_size_val = pos.ins().iconst(addr_ty, access_size as i64);
-        let (adj_offset, overflow) = pos.ins().iadd_ifcout(offset, access_size_val);
-        pos.ins().trapif(
-            isa.unsigned_add_overflow_condition(),
-            overflow,
-            ir::TrapCode::HeapOutOfBounds,
-        );
+        let adj_offset =
+            pos.ins()
+                .uadd_overflow_trap(offset, access_size_val, ir::TrapCode::HeapOutOfBounds);
         (IntCC::UnsignedGreaterThan, adj_offset, bound)
     };
     let oob = pos.ins().icmp(cc, lhs, bound);

cranelift/codegen/src/prelude_lower.isle

@@ -425,6 +425,14 @@
             (_y Unit (emit consumer_inst)))
         (value_regs producer_result consumer_result)))
 
+;; A flag-producer that also produces a result, paired with a consumer that has
+;; no results.
+(rule (with_flags (ProducesFlags.ProducesFlagsReturnsResultWithConsumer producer_inst producer_result)
+                  (ConsumesFlags.ConsumesFlagsSideEffect consumer_inst))
+      (let ((_ Unit (emit producer_inst))
+            (_ Unit (emit consumer_inst)))
+        (value_reg producer_result)))
+
 (rule (with_flags (ProducesFlags.ProducesFlagsSideEffect producer_inst)
                   (ConsumesFlags.ConsumesFlagsReturnsReg consumer_inst consumer_result))
       (let ((_x Unit (emit producer_inst))