Add iadd_overflow_trap

cranelift/codegen/meta/src/shared/formats.rs

@@ -20,6 +20,7 @@ pub(crate) struct Formats {
     pub(crate) int_compare: Rc<InstructionFormat>,
     pub(crate) int_compare_imm: Rc<InstructionFormat>,
     pub(crate) int_cond_trap: Rc<InstructionFormat>,
+    pub(crate) int_add_trap: Rc<InstructionFormat>,
     pub(crate) jump: Rc<InstructionFormat>,
     pub(crate) load: Rc<InstructionFormat>,
     pub(crate) load_no_offset: Rc<InstructionFormat>,
@@ -223,6 +224,12 @@ impl Formats {
                 .imm(&imm.trapcode)
                 .build(),
 
+            int_add_trap: Builder::new("IntAddTrap")
+                .value()
+                .value()
+                .imm(&imm.trapcode)
+                .build(),
+
             float_cond_trap: Builder::new("FloatCondTrap")
                 .imm(&imm.floatcc)
                 .value()

cranelift/codegen/meta/src/shared/instructions.rs

@@ -2174,6 +2174,37 @@ pub(crate) fn define(
         .operands_out(vec![a, c_if_out]),
     );
 
+    {
+        let code = &Operand::new("code", &imm.trapcode);
+
+        let i32_64 = &TypeVar::new(
+            "i32_64",
+            "A 32 or 64-bit scalar integer type",
+            TypeSetBuilder::new().ints(32..64).build(),
+        );
+
+        let a = &Operand::new("a", i32_64);
+        let x = &Operand::new("x", i32_64);
+        let y = &Operand::new("y", i32_64);
+        ig.push(
+            Inst::new(
+                "iadd_overflow_trap",
+                r#"
+            Add 32 or 64-bit integers, and trap if overflow occurs.
+
+            This is the same as `iadd_cout` but traps instead of returning a carry flag.
+
+            Polymorphic over all scalar integer types, but does not support vector
+            types.
+            "#,
+                &formats.int_add_trap,
+            )
+            .operands_in(vec![x, y, code])
+            .operands_out(vec![a])
+            .can_trap(true),
+        );
+    }
+
     ig.push(
         Inst::new(
             "isub_bin",

cranelift/codegen/src/isa/aarch64/inst.isle

@@ -2772,6 +2772,14 @@
         )
         x))
 
+;; Check for unsigned overflow.
+(decl trap_if_overflow (ProducesFlags TrapCode) Reg)
+(rule (trap_if_overflow producer tc)
+      (with_flags_reg
+        producer
+        (ConsumesFlags.ConsumesFlagsSideEffect
+          (MInst.TrapIf (cond_br_cond (Cond.Hs)) tc))))
+
 (decl sink_atomic_load (Inst) Reg)
 (rule (sink_atomic_load x @ (atomic_load _ addr))
       (let ((_ Unit (sink_inst x)))

cranelift/codegen/src/isa/aarch64/lower.isle

@@ -2378,6 +2378,9 @@
        (add_with_flags ty a b)
        (invalid_reg)))
 
+(rule (lower (has_type (fits_in_64 ty) (iadd_overflow_trap a b tc)))
+      (trap_if_overflow (add_with_flags_paired ty a b) tc))
+
 ;;; Rules for `tls_value` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 (rule (lower (tls_value (symbol_value_data name _ _)))

cranelift/codegen/src/isa/aarch64/lower_inst.rs

@@ -259,6 +259,8 @@ pub(crate) fn lower_insn_to_regs(
 
         Opcode::IaddIfcout => implemented_in_isle(ctx),
 
+        Opcode::IaddOverflowTrap => implemented_in_isle(ctx),
+
         Opcode::IaddImm
         | Opcode::ImulImm
         | Opcode::UdivImm

cranelift/codegen/src/isa/riscv64/inst.isle

@@ -1847,22 +1847,22 @@
   (gen_fabs x ty)
   (fpu_rrr (fabs_copy_sign ty) ty x x))
 
-;;; right now only return if overflow.
-(decl lower_uadd_overflow (Reg Reg Type) Reg)
+;;; Returns the sum in the first register, and the overflow test in the second.
+(decl lower_uadd_overflow (Reg Reg Type) ValueRegs)
 
 (rule 1
   (lower_uadd_overflow x y $I64)
-  (let
-    ((tmp Reg (alu_add x y)))
-    (gen_icmp (IntCC.UnsignedLessThan) tmp x $I64)))
+  (let ((tmp Reg (alu_add x y))
+        (test Reg (gen_icmp (IntCC.UnsignedLessThan) tmp x $I64)))
+    (value_regs tmp test)))
 
 (rule
   (lower_uadd_overflow x y (fits_in_32 ty))
-  (let
-    ((tmp_x Reg (ext_int_if_need $false x ty))
-      (tmp_y Reg (ext_int_if_need $false y ty))
-      (sum Reg (alu_add tmp_x tmp_y)))
-    (alu_srli sum (ty_bits ty))))
+  (let ((tmp_x Reg (ext_int_if_need $false x ty))
+        (tmp_y Reg (ext_int_if_need $false y ty))
+        (sum Reg (alu_add tmp_x tmp_y))
+        (test Reg (alu_srli sum (ty_bits ty))))
+    (value_regs sum test)))
 
 (decl inst_output_get (InstOutput u8) ValueRegs)
 (extern constructor inst_output_get inst_output_get)

cranelift/codegen/src/isa/riscv64/lower.isle

@@ -48,6 +48,13 @@
   (lower (has_type (fits_in_64 ty) (iadd_ifcout x y)))
   (output_ifcout (alu_add x y)))
 
+;;; Rules for `iadd_overflow_trap` ;;;;;;;;;;;;;
+(rule
+  (lower (has_type (fits_in_64 ty) (iadd_overflow_trap x y tc)))
+  (let ((res ValueRegs (lower_uadd_overflow x y ty))
+        (_ InstOutput (gen_trapif (value_regs_get res 1) tc)))
+    (value_regs_get res 0)))
+
 
 ;;;; Rules for `isub` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; Base case, simply subtracting things in registers.
@@ -857,8 +864,8 @@
 (rule
   (lower (trapif _ (iadd_ifcout a @ (value_type ty) b) trap_code))
   (let
-    ((test Reg (lower_uadd_overflow a b ty)))
-    (gen_trapif test trap_code)))
+    ((res ValueRegs (lower_uadd_overflow a b ty)))
+    (gen_trapif (value_regs_get res 1) trap_code)))
 
 
 ;;;;;  Rules for `trapff`;;;;;;;;;

cranelift/codegen/src/isa/s390x/inst.isle

@@ -3874,12 +3874,24 @@
 (decl add_logical_reg (Type Reg Reg) Reg)
 (rule (add_logical_reg ty x y) (alu_rrr ty (aluop_add_logical ty) x y))
 
+(decl add_logical_reg_with_flags_paired (Type Reg Reg) ProducesFlags)
+(rule (add_logical_reg_with_flags_paired ty x y)
+      (let ((dst WritableReg (temp_writable_reg ty)))
+        (ProducesFlags.ProducesFlagsReturnsResultWithConsumer
+          (MInst.AluRRR (aluop_add_logical ty) dst x y) dst)))
+
 (decl add_logical_reg_zext32 (Type Reg Reg) Reg)
 (rule (add_logical_reg_zext32 ty x y) (alu_rr ty (aluop_add_logical_zext32 ty) x y))
 
 (decl add_logical_zimm32 (Type Reg u32) Reg)
 (rule (add_logical_zimm32 ty x y) (alu_ruimm32 ty (aluop_add_logical ty) x y))
 
+(decl add_logical_zimm32_with_flags_paired (Type Reg u32) ProducesFlags)
+(rule (add_logical_zimm32_with_flags_paired ty x y)
+      (let ((dst WritableReg (temp_writable_reg ty)))
+        (ProducesFlags.ProducesFlagsReturnsResultWithConsumer
+          (MInst.AluRUImm32 (aluop_add_logical ty) dst x y) dst)))
+
 (decl add_logical_mem (Type Reg MemArg) Reg)
 (rule (add_logical_mem ty x y) (alu_rx ty (aluop_add_logical ty) x y))
 

cranelift/codegen/src/isa/s390x/lower.isle

@@ -3876,6 +3876,24 @@
         (trap_if_bool (bool (flags_to_producesflags flags) (mask_as_cond 3))
                       trap_code)))
 
+;;;; Rules for `iadd_overflow_trap` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(rule (lower (has_type (fits_in_64 ty) (iadd_overflow_trap x y tc)))
+      (with_flags
+        (add_logical_reg_with_flags_paired ty x y)
+        (trap_if_impl (mask_as_cond 3) tc)))
+
+(rule 1 (lower (has_type (fits_in_64 ty)
+                         (iadd_overflow_trap x (u32_from_value y) tc)))
+      (with_flags
+        (add_logical_zimm32_with_flags_paired ty x y)
+        (trap_if_impl (mask_as_cond 3) tc)))
+
+(rule 2 (lower (has_type (fits_in_64 ty)
+                         (iadd_overflow_trap (u32_from_value x) y tc)))
+      (with_flags
+        (add_logical_zimm32_with_flags_paired ty y x)
+        (trap_if_impl (mask_as_cond 3) tc)))
 
 ;;;; Rules for `return` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 

cranelift/codegen/src/isa/s390x/lower.rs

@@ -241,6 +241,7 @@ impl LowerBackend for S390xBackend {
             | Opcode::IaddCout
             | Opcode::IaddCarry
             | Opcode::IaddIfcarry
+            | Opcode::IaddOverflowTrap
             | Opcode::IsubBin
             | Opcode::IsubIfbin
             | Opcode::IsubBout

cranelift/codegen/src/isa/x64/lower.isle

@@ -1352,6 +1352,41 @@
 (rule (lower (trap code))
       (side_effect (x64_ud2 code)))
 
+;;;; Rules for `iadd_overflow_trap` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(rule (lower (has_type (fits_in_64 ty) (iadd_overflow_trap a b tc)))
+      (with_flags
+        (x64_add_with_flags_paired ty a b)
+        (trap_if (CC.B) tc)))
+
+;; Add a register and constant.
+
+(rule 1 (lower (has_type (fits_in_64 ty)
+                         (iadd_overflow_trap a (simm32_from_value b) tc)))
+      (with_flags
+        (x64_add_with_flags_paired ty a b)
+        (trap_if (CC.B) tc)))
+
+(rule 2 (lower (has_type (fits_in_64 ty)
+                         (iadd_overflow_trap (simm32_from_value a) b tc)))
+      (with_flags
+        (x64_add_with_flags_paired ty b a)
+        (trap_if (CC.B) tc)))
+
+;; Add a register and memory.
+
+(rule 3 (lower (has_type (fits_in_64 ty)
+                         (iadd_overflow_trap a (sinkable_load b) tc)))
+      (with_flags
+        (x64_add_with_flags_paired ty a (sink_load_to_gpr_mem_imm b))
+        (trap_if (CC.B) tc)))
+
+(rule 4 (lower (has_type (fits_in_64 ty)
+                         (iadd_overflow_trap (sinkable_load a) b tc)))
+      (with_flags
+        (x64_add_with_flags_paired ty b (sink_load_to_gpr_mem_imm a))
+        (trap_if (CC.B) tc)))
+
 ;;;; Rules for `trapif` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 ;; The flags must not have been clobbered by any other instruction between the

cranelift/codegen/src/isa/x64/lower.rs

@@ -521,6 +521,7 @@ fn lower_insn_to_regs(
         | Opcode::IaddCout
         | Opcode::IaddCarry
         | Opcode::IaddIfcarry
+        | Opcode::IaddOverflowTrap
         | Opcode::IsubBin
         | Opcode::IsubIfbin
         | Opcode::IsubBout

cranelift/codegen/src/legalizer/heap.rs

@@ -74,12 +74,9 @@ fn dynamic_addr(
     } else {
         // We need an overflow check for the adjusted offset.
         let access_size_val = pos.ins().iconst(addr_ty, access_size as i64);
-        let (adj_offset, overflow) = pos.ins().iadd_ifcout(offset, access_size_val);
-        pos.ins().trapif(
-            isa.unsigned_add_overflow_condition(),
-            overflow,
-            ir::TrapCode::HeapOutOfBounds,
-        );
+        let adj_offset =
+            pos.ins()
+                .iadd_overflow_trap(offset, access_size_val, ir::TrapCode::HeapOutOfBounds);
         (IntCC::UnsignedGreaterThan, adj_offset, bound)
     };
     let oob = pos.ins().icmp(cc, lhs, bound);

cranelift/codegen/src/prelude_lower.isle

@@ -425,6 +425,14 @@
             (_y Unit (emit consumer_inst)))
         (value_regs producer_result consumer_result)))
 
+;; A flag-producer that also produces a result, paired with a consumer that has
+;; no results.
+(rule (with_flags (ProducesFlags.ProducesFlagsReturnsResultWithConsumer producer_inst producer_result)
+                  (ConsumesFlags.ConsumesFlagsSideEffect consumer_inst))
+      (let ((_ Unit (emit producer_inst))
+            (_ Unit (emit consumer_inst)))
+        (value_reg producer_result)))
+
 (rule (with_flags (ProducesFlags.ProducesFlagsSideEffect producer_inst)
                   (ConsumesFlags.ConsumesFlagsReturnsReg consumer_inst consumer_result))
       (let ((_x Unit (emit producer_inst))

cranelift/codegen/src/verifier/mod.rs

@@ -759,6 +759,7 @@ impl<'a> Verifier<'a> {
             | Ternary { .. }
             | TernaryImm8 { .. }
             | Shuffle { .. }
+            | IntAddTrap { .. }
             | IntCompare { .. }
             | IntCompareImm { .. }
             | FloatCompare { .. }

cranelift/codegen/src/write.rs

@@ -418,6 +418,7 @@ pub fn write_operands(w: &mut dyn Write, dfg: &DataFlowGraph, inst: Inst) -> fmt
         }
         IntCompare { cond, args, .. } => write!(w, " {} {}, {}", cond, args[0], args[1]),
         IntCompareImm { cond, arg, imm, .. } => write!(w, " {} {}, {}", cond, arg, imm),
+        IntAddTrap { args, code, .. } => write!(w, " {}, {}, {}", args[0], args[1], code),
         FloatCompare { cond, args, .. } => write!(w, " {} {}, {}", cond, args[0], args[1]),
         Jump {
             destination,

cranelift/filetests/filetests/isa/aarch64/iadd_overflow_trap.clif

@@ -0,0 +1,77 @@
+test compile precise-output
+target aarch64
+
+function %f0(i32) -> i32 {
+block0(v0: i32):
+    v1 = iconst.i32 127
+    v2 = iadd_overflow_trap v0, v1, user0
+    return v2
+}
+
+; block0:
+;   movz x3, #127
+;   adds w0, w0, w3
+;   b.lo 8 ; udf
+;   ret
+
+function %f1(i32) -> i32 {
+block0(v0: i32):
+    v1 = iconst.i32 127
+    v2 = iadd_overflow_trap v1, v0, user0
+    return v2
+}
+
+; block0:
+;   movz x3, #127
+;   adds w0, w3, w0
+;   b.lo 8 ; udf
+;   ret
+
+function %f2(i32, i32) -> i32 {
+block0(v0: i32, v1: i32):
+    v2 = iadd_overflow_trap v0, v1, user0
+    return v2
+}
+
+; block0:
+;   adds w0, w0, w1
+;   b.lo 8 ; udf
+;   ret
+
+function %f3(i64) -> i64 {
+block0(v0: i64):
+    v1 = iconst.i64 127
+    v2 = iadd_overflow_trap v0, v1, user0
+    return v2
+}
+
+; block0:
+;   movz x3, #127
+;   adds x0, x0, x3
+;   b.lo 8 ; udf
+;   ret
+
+function %f3(i64) -> i64 {
+block0(v0: i64):
+    v1 = iconst.i64 127
+    v2 = iadd_overflow_trap v1, v0, user0
+    return v2
+}
+
+; block0:
+;   movz x3, #127
+;   adds x0, x3, x0
+;   b.lo 8 ; udf
+;   ret
+
+function %f4(i64, i64) -> i64 {
+block0(v0: i64, v1: i64):
+    v2 = iadd_overflow_trap v0, v1, user0
+    return v2
+}
+
+; block0:
+;   adds x0, x0, x1
+;   b.lo 8 ; udf
+;   ret
+