diff --git a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest index 1e38a6cc6359..f75e4e5adb6f 100644 --- a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest +++ b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest @@ -1,4 +1,4 @@ src/clif.isle f176ef3bba99365 -src/prelude.isle 7b911d3b894ae17 +src/prelude.isle 22dd5ff133398960 src/isa/aarch64/inst.isle 5fa80451697b084f src/isa/aarch64/lower.isle 2d2e1e076a0c8a23 diff --git a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs index ca4f68a59833..9764b5b8087d 100644 --- a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs +++ b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs @@ -24,6 +24,7 @@ pub trait Context { fn u8_and(&mut self, arg0: u8, arg1: u8) -> u8; fn value_reg(&mut self, arg0: Reg) -> ValueRegs; fn value_regs(&mut self, arg0: Reg, arg1: Reg) -> ValueRegs; + fn value_regs_invalid(&mut self) -> ValueRegs; fn temp_writable_reg(&mut self, arg0: Type) -> WritableReg; fn invalid_reg(&mut self) -> Reg; fn put_in_reg(&mut self, arg0: Value) -> Reg; @@ -92,13 +93,19 @@ pub trait Context { fn rotr_opposite_amount(&mut self, arg0: Type, arg1: ImmShift) -> ImmShift; } -/// Internal type ProducesFlags: defined at src/prelude.isle line 277. +/// Internal type SideEffectNoResult: defined at src/prelude.isle line 279. +#[derive(Clone, Debug)] +pub enum SideEffectNoResult { + Inst { inst: MInst }, +} + +/// Internal type ProducesFlags: defined at src/prelude.isle line 292. #[derive(Clone, Debug)] pub enum ProducesFlags { ProducesFlags { inst: MInst, result: Reg }, } -/// Internal type ConsumesFlags: defined at src/prelude.isle line 280. +/// Internal type ConsumesFlags: defined at src/prelude.isle line 295. #[derive(Clone, Debug)] pub enum ConsumesFlags { ConsumesFlags { inst: MInst, result: Reg }, @@ -978,7 +985,7 @@ pub enum AtomicRMWOp { // Generated as internal constructor for term temp_reg. pub fn constructor_temp_reg(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; - // Rule at src/prelude.isle line 66. + // Rule at src/prelude.isle line 70. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); return Some(expr1_0); @@ -987,13 +994,31 @@ pub fn constructor_temp_reg(ctx: &mut C, arg0: Type) -> Option // Generated as internal constructor for term lo_reg. pub fn constructor_lo_reg(ctx: &mut C, arg0: Value) -> Option { let pattern0_0 = arg0; - // Rule at src/prelude.isle line 101. + // Rule at src/prelude.isle line 105. let expr0_0 = C::put_in_regs(ctx, pattern0_0); let expr1_0: usize = 0; let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); return Some(expr2_0); } +// Generated as internal constructor for term value_regs_none. +pub fn constructor_value_regs_none( + ctx: &mut C, + arg0: &SideEffectNoResult, +) -> Option { + let pattern0_0 = arg0; + if let &SideEffectNoResult::Inst { + inst: ref pattern1_0, + } = pattern0_0 + { + // Rule at src/prelude.isle line 284. + let expr0_0 = C::emit(ctx, &pattern1_0); + let expr1_0 = C::value_regs_invalid(ctx); + return Some(expr1_0); + } + return None; +} + // Generated as internal constructor for term with_flags. pub fn constructor_with_flags( ctx: &mut C, @@ -1012,7 +1037,7 @@ pub fn constructor_with_flags( result: pattern3_1, } = pattern2_0 { - // Rule at src/prelude.isle line 290. + // Rule at src/prelude.isle line 305. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern3_0); let expr2_0 = C::value_regs(ctx, pattern1_1, pattern3_1); @@ -1040,7 +1065,7 @@ pub fn constructor_with_flags_1( result: pattern3_1, } = pattern2_0 { - // Rule at src/prelude.isle line 298. + // Rule at src/prelude.isle line 313. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern3_0); return Some(pattern3_1); @@ -1074,7 +1099,7 @@ pub fn constructor_with_flags_2( result: pattern5_1, } = pattern4_0 { - // Rule at src/prelude.isle line 308. + // Rule at src/prelude.isle line 323. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern5_0); let expr2_0 = C::emit(ctx, &pattern3_0); diff --git a/cranelift/codegen/src/isa/x64/inst.isle b/cranelift/codegen/src/isa/x64/inst.isle index 3a6f4089f54d..a126750872fa 100644 --- a/cranelift/codegen/src/isa/x64/inst.isle +++ b/cranelift/codegen/src/isa/x64/inst.isle @@ -4,6 +4,7 @@ (type MInst extern (enum (Nop (len u8)) + (Ud2 (trap_code TrapCode)) (AluRmiR (size OperandSize) (op AluRmiROpcode) (src1 Reg) @@ -72,9 +73,11 @@ (Not (size OperandSize) (src Reg) (dst WritableReg)) + (Neg (size OperandSize) + (src Reg) + (dst WritableReg)) (LoadEffectiveAddress (addr SyntheticAmode) - (dst WritableReg)) -)) + (dst WritableReg)))) (type OperandSize extern (enum Size8 @@ -534,9 +537,9 @@ (let ((wr WritableReg (temp_writable_reg ty)) (r Reg (writable_reg_to_reg wr)) (_ Unit (emit (MInst.XmmRmR (sse_cmp_op $I32X4) - r - (RegMem.Reg r) - wr)))) + r + (RegMem.Reg r) + wr)))) r)) ;; Helper for creating an SSE register holding an `i64x2` from two `i64` values. @@ -697,8 +700,8 @@ ;; ;; Use `m_` prefix (short for "mach inst") to disambiguate with the ISLE-builtin ;; `and` operator. -(decl m_and (Type Reg RegMemImm) Reg) -(rule (m_and ty src1 src2) +(decl x64_and (Type Reg RegMemImm) Reg) +(rule (x64_and ty src1 src2) (alu_rmi_r ty (AluRmiROpcode.And) src1 @@ -724,7 +727,7 @@ (decl imm (Type u64) Reg) ;; Integer immediates. -(rule (imm ty simm64) +(rule (imm (fits_in_64 ty) simm64) (let ((dst WritableReg (temp_writable_reg ty)) (size OperandSize (operand_size_of_type_32_64 ty)) (_ Unit (emit (MInst.Imm size simm64 dst)))) @@ -749,7 +752,7 @@ (writable_reg_to_reg dst))) ;; Special case for integer zero immediates: turn them into an `xor r, r`. -(rule (imm ty 0) +(rule (imm (fits_in_64 ty) 0) (let ((wr WritableReg (temp_writable_reg ty)) (r Reg (writable_reg_to_reg wr)) (size OperandSize (operand_size_of_type_32_64 ty)) @@ -766,9 +769,9 @@ (let ((wr WritableReg (temp_writable_reg ty)) (r Reg (writable_reg_to_reg wr)) (_ Unit (emit (MInst.XmmRmR (sse_xor_op ty) - r - (RegMem.Reg r) - wr)))) + r + (RegMem.Reg r) + wr)))) r)) ;; Special case for `f32` zero immediates to use `xorps`. @@ -807,10 +810,16 @@ ;; Helper for creating `rotl` instructions (prefixed with "m_", short for "mach ;; inst", to disambiguate this from clif's `rotl`). -(decl m_rotl (Type Reg Imm8Reg) Reg) -(rule (m_rotl ty src1 src2) +(decl x64_rotl (Type Reg Imm8Reg) Reg) +(rule (x64_rotl ty src1 src2) (shift_r ty (ShiftKind.RotateLeft) src1 src2)) +;; Helper for creating `rotr` instructions (prefixed with "m_", short for "mach +;; inst", to disambiguate this from clif's `rotr`). +(decl x64_rotr (Type Reg Imm8Reg) Reg) +(rule (x64_rotr ty src1 src2) + (shift_r ty (ShiftKind.RotateRight) src1 src2)) + ;; Helper for creating `shl` instructions. (decl shl (Type Reg Imm8Reg) Reg) (rule (shl ty src1 src2) @@ -1423,8 +1432,21 @@ (_ Unit (emit (MInst.Not size src dst)))) (writable_reg_to_reg dst))) +;; Helper for creating `neg` instructions. +(decl neg (Type Reg) Reg) +(rule (neg ty src) + (let ((dst WritableReg (temp_writable_reg ty)) + (size OperandSize (operand_size_of_type_32_64 ty)) + (_ Unit (emit (MInst.Neg size src dst)))) + (writable_reg_to_reg dst))) + (decl lea (SyntheticAmode) Reg) (rule (lea addr) (let ((dst WritableReg (temp_writable_reg $I64)) (_ Unit (emit (MInst.LoadEffectiveAddress addr dst)))) (writable_reg_to_reg dst))) + +;; Helper for creating `ud2` instructions. +(decl ud2 (TrapCode) SideEffectNoResult) +(rule (ud2 code) + (SideEffectNoResult.Inst (MInst.Ud2 code))) diff --git a/cranelift/codegen/src/isa/x64/inst/emit_tests.rs b/cranelift/codegen/src/isa/x64/inst/emit_tests.rs index e8f905bcc5ad..1c71187fdeb8 100644 --- a/cranelift/codegen/src/isa/x64/inst/emit_tests.rs +++ b/cranelift/codegen/src/isa/x64/inst/emit_tests.rs @@ -16,6 +16,17 @@ use super::*; use crate::isa::x64; use alloc::vec::Vec; +impl Inst { + fn neg(size: OperandSize, src: Writable) -> Inst { + debug_assert_eq!(src.to_reg().get_class(), RegClass::I64); + Inst::Neg { + size, + src: src.to_reg(), + dst: src, + } + } +} + #[test] fn test_x64_emit() { let rax = regs::rax(); diff --git a/cranelift/codegen/src/isa/x64/inst/mod.rs b/cranelift/codegen/src/isa/x64/inst/mod.rs index e8d958f7387a..6f00370ea09d 100644 --- a/cranelift/codegen/src/isa/x64/inst/mod.rs +++ b/cranelift/codegen/src/isa/x64/inst/mod.rs @@ -668,15 +668,6 @@ impl Inst { } } - pub(crate) fn neg(size: OperandSize, src: Writable) -> Inst { - debug_assert_eq!(src.to_reg().get_class(), RegClass::I64); - Inst::Neg { - size, - src: src.to_reg(), - dst: src, - } - } - pub(crate) fn div(size: OperandSize, signed: bool, divisor: RegMem) -> Inst { divisor.assert_regclass_is(RegClass::I64); Inst::Div { diff --git a/cranelift/codegen/src/isa/x64/lower.isle b/cranelift/codegen/src/isa/x64/lower.isle index 04513a1c5d13..9b881b5f691c 100644 --- a/cranelift/codegen/src/isa/x64/lower.isle +++ b/cranelift/codegen/src/isa/x64/lower.isle @@ -326,37 +326,37 @@ ;; And two registers. (rule (lower (has_type (fits_in_64 ty) (band x y))) - (value_reg (m_and ty - (put_in_reg x) - (RegMemImm.Reg (put_in_reg y))))) + (value_reg (x64_and ty + (put_in_reg x) + (RegMemImm.Reg (put_in_reg y))))) ;; And with a memory operand. (rule (lower (has_type (fits_in_64 ty) (band x (sinkable_load y)))) - (value_reg (m_and ty - (put_in_reg x) - (sink_load y)))) + (value_reg (x64_and ty + (put_in_reg x) + (sink_load y)))) (rule (lower (has_type (fits_in_64 ty) (band (sinkable_load x) y))) - (value_reg (m_and ty - (put_in_reg y) - (sink_load x)))) + (value_reg (x64_and ty + (put_in_reg y) + (sink_load x)))) ;; And with an immediate. (rule (lower (has_type (fits_in_64 ty) (band x (simm32_from_value y)))) - (value_reg (m_and ty - (put_in_reg x) - y))) + (value_reg (x64_and ty + (put_in_reg x) + y))) (rule (lower (has_type (fits_in_64 ty) (band (simm32_from_value x) y))) - (value_reg (m_and ty - (put_in_reg y) - x))) + (value_reg (x64_and ty + (put_in_reg y) + x))) ;; SSE. @@ -378,8 +378,8 @@ (y_regs ValueRegs (put_in_regs y)) (y_lo Reg (value_regs_get y_regs 0)) (y_hi Reg (value_regs_get y_regs 1))) - (value_regs (m_and $I64 x_lo (RegMemImm.Reg y_lo)) - (m_and $I64 x_hi (RegMemImm.Reg y_hi))))) + (value_regs (x64_and $I64 x_lo (RegMemImm.Reg y_lo)) + (x64_and $I64 x_hi (RegMemImm.Reg y_hi))))) (rule (lower (has_type $B128 (band x y))) ;; Booleans are always `0` or `1`, so we only need to do the `and` on the @@ -389,7 +389,7 @@ (x_lo Reg (value_regs_get x_regs 0)) (x_hi Reg (value_regs_get x_regs 1)) (y_lo Reg (lo_reg y))) - (value_regs (m_and $I64 x_lo (RegMemImm.Reg y_lo)) + (value_regs (x64_and $I64 x_lo (RegMemImm.Reg y_lo)) x_hi))) ;;;; Rules for `bor` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -832,13 +832,13 @@ (rule (lower (has_type (ty_8_or_16 ty) (rotl src amt))) (let ((amt_ Reg (extend_to_reg amt $I32 (ExtendKind.Zero)))) - (value_reg (m_rotl ty (put_in_reg src) (Imm8Reg.Reg amt_))))) + (value_reg (x64_rotl ty (put_in_reg src) (Imm8Reg.Reg amt_))))) (rule (lower (has_type (ty_8_or_16 ty) (rotl src (u64_from_iconst amt)))) - (value_reg (m_rotl ty - (put_in_reg src) - (const_to_type_masked_imm8 amt ty)))) + (value_reg (x64_rotl ty + (put_in_reg src) + (const_to_type_masked_imm8 amt ty)))) ;; `i64` and `i32`: we can rely on x86's rotate-amount masking since ;; we operate on the whole register. @@ -847,13 +847,13 @@ ;; NB: Only the low bits of `amt` matter since we logically mask the ;; shift amount to the value's bit width. (let ((amt_ Reg (lo_reg amt))) - (value_reg (m_rotl ty (put_in_reg src) (Imm8Reg.Reg amt_))))) + (value_reg (x64_rotl ty (put_in_reg src) (Imm8Reg.Reg amt_))))) (rule (lower (has_type (ty_32_or_64 ty) (rotl src (u64_from_iconst amt)))) - (value_reg (m_rotl ty - (put_in_reg src) - (const_to_type_masked_imm8 amt ty)))) + (value_reg (x64_rotl ty + (put_in_reg src) + (const_to_type_masked_imm8 amt ty)))) ;; `i128`. @@ -865,6 +865,71 @@ (or_i128 (shl_i128 src_ amt_) (shr_i128 src_ (sub $I64 (imm $I64 128) (RegMemImm.Reg amt_)))))) +;;;; Rules for `rotr` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; `i16` and `i8`: we need to extend the shift amount, or mask the +;; constant. + +(rule (lower (has_type (ty_8_or_16 ty) (rotr src amt))) + (let ((amt_ Reg (extend_to_reg amt $I32 (ExtendKind.Zero)))) + (value_reg (x64_rotr ty (put_in_reg src) (Imm8Reg.Reg amt_))))) + +(rule (lower (has_type (ty_8_or_16 ty) + (rotr src (u64_from_iconst amt)))) + (value_reg (x64_rotr ty + (put_in_reg src) + (const_to_type_masked_imm8 amt ty)))) + +;; `i64` and `i32`: we can rely on x86's rotate-amount masking since +;; we operate on the whole register. + +(rule (lower (has_type (ty_32_or_64 ty) (rotr src amt))) + ;; NB: Only the low bits of `amt` matter since we logically mask the + ;; shift amount to the value's bit width. + (let ((amt_ Reg (lo_reg amt))) + (value_reg (x64_rotr ty (put_in_reg src) (Imm8Reg.Reg amt_))))) + +(rule (lower (has_type (ty_32_or_64 ty) + (rotr src (u64_from_iconst amt)))) + (value_reg (x64_rotr ty + (put_in_reg src) + (const_to_type_masked_imm8 amt ty)))) + +;; `i128`. + +(rule (lower (has_type $I128 (rotr src amt))) + (let ((src_ ValueRegs (put_in_regs src)) + ;; NB: Only the low bits of `amt` matter since we logically mask the + ;; rotation amount to the value's bit width. + (amt_ Reg (lo_reg amt))) + (or_i128 (shr_i128 src_ amt_) + (shl_i128 src_ (sub $I64 (imm $I64 128) (RegMemImm.Reg amt_)))))) + +;;;; Rules for `ineg` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; `i64` and smaller. + +(rule (lower (has_type (fits_in_64 ty) (ineg x))) + (value_reg (neg ty (put_in_reg x)))) + +;; SSE. + +(rule (lower (has_type $I8X16 (ineg x))) + (value_reg (psubb (imm $I8X16 0) + (put_in_reg_mem x)))) + +(rule (lower (has_type $I16X8 (ineg x))) + (value_reg (psubw (imm $I16X8 0) + (put_in_reg_mem x)))) + +(rule (lower (has_type $I32X4 (ineg x))) + (value_reg (psubd (imm $I32X4 0) + (put_in_reg_mem x)))) + +(rule (lower (has_type $I64X2 (ineg x))) + (value_reg (psubq (imm $I64X2 0) + (put_in_reg_mem x)))) + ;;;; Rules for `avg_round` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (rule (lower (has_type (multi_lane 8 16) @@ -1367,3 +1432,13 @@ (rule (lower (has_type $I32X4 (umin x y))) (value_reg (pminud (put_in_reg x) (put_in_reg_mem y)))) + +;;;; Rules for `trap` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(rule (lower (trap code)) + (value_regs_none (ud2 code))) + +;;;; Rules for `resumable_trap` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(rule (lower (resumable_trap code)) + (value_regs_none (ud2 code))) diff --git a/cranelift/codegen/src/isa/x64/lower.rs b/cranelift/codegen/src/isa/x64/lower.rs index 606b208c17a8..a7765de6772b 100644 --- a/cranelift/codegen/src/isa/x64/lower.rs +++ b/cranelift/codegen/src/isa/x64/lower.rs @@ -877,316 +877,6 @@ fn emit_bitrev>(ctx: &mut C, src: Reg, dst: Writable, ctx.emit(Inst::gen_move(dst, tmp0.to_reg(), types::I64)); } -fn emit_shl_i128>( - ctx: &mut C, - src: ValueRegs, - dst: ValueRegs>, - amt_src: Reg, -) { - let src_lo = src.regs()[0]; - let src_hi = src.regs()[1]; - let dst_lo = dst.regs()[0]; - let dst_hi = dst.regs()[1]; - - // mov tmp1, src_lo - // shl tmp1, amt_src - // mov tmp2, src_hi - // shl tmp2, amt_src - // mov amt, 64 - // sub amt, amt_src - // mov tmp3, src_lo - // shr tmp3, amt - // xor dst_lo, dst_lo - // test amt_src, 127 - // cmovz tmp3, dst_lo - // or tmp3, tmp2 - // mov amt, amt_src - // and amt, 64 - // cmovz dst_hi, tmp3 - // cmovz dst_lo, tmp1 - // cmovnz dst_hi, tmp1 - - let tmp1 = ctx.alloc_tmp(types::I64).only_reg().unwrap(); - let tmp2 = ctx.alloc_tmp(types::I64).only_reg().unwrap(); - let tmp3 = ctx.alloc_tmp(types::I64).only_reg().unwrap(); - let amt = ctx.alloc_tmp(types::I64).only_reg().unwrap(); - - ctx.emit(Inst::gen_move(tmp1, src_lo, types::I64)); - ctx.emit(Inst::gen_move( - Writable::from_reg(regs::rcx()), - amt_src, - types::I64, - )); - ctx.emit(Inst::shift_r( - OperandSize::Size64, - ShiftKind::ShiftLeft, - None, - tmp1, - )); - - ctx.emit(Inst::gen_move(tmp2, src_hi, types::I64)); - ctx.emit(Inst::gen_move( - Writable::from_reg(regs::rcx()), - amt_src, - types::I64, - )); - ctx.emit(Inst::shift_r( - OperandSize::Size64, - ShiftKind::ShiftLeft, - None, - tmp2, - )); - - ctx.emit(Inst::imm(OperandSize::Size64, 64, amt)); - ctx.emit(Inst::alu_rmi_r( - OperandSize::Size64, - AluRmiROpcode::Sub, - RegMemImm::reg(amt_src), - amt, - )); - - ctx.emit(Inst::gen_move(tmp3, src_lo, types::I64)); - ctx.emit(Inst::gen_move( - Writable::from_reg(regs::rcx()), - amt.to_reg(), - types::I64, - )); - ctx.emit(Inst::shift_r( - OperandSize::Size64, - ShiftKind::ShiftRightLogical, - None, - tmp3, - )); - ctx.emit(Inst::alu_rmi_r( - OperandSize::Size64, - AluRmiROpcode::Xor, - RegMemImm::reg(dst_lo.to_reg()), - dst_lo, - )); - - ctx.emit(Inst::test_rmi_r( - OperandSize::Size64, - RegMemImm::imm(127), - amt_src, - )); - ctx.emit(Inst::cmove( - OperandSize::Size64, - CC::Z, - RegMem::reg(dst_lo.to_reg()), - tmp3, - )); - - ctx.emit(Inst::alu_rmi_r( - OperandSize::Size64, - AluRmiROpcode::Or, - RegMemImm::reg(tmp2.to_reg()), - tmp3, - )); - - // This isn't semantically necessary, but it keeps the - // register allocator happy, because it cannot otherwise - // infer that cmovz + cmovnz always defines dst_hi. - ctx.emit(Inst::alu_rmi_r( - OperandSize::Size64, - AluRmiROpcode::Xor, - RegMemImm::reg(dst_hi.to_reg()), - dst_hi, - )); - - ctx.emit(Inst::gen_move(amt, amt_src, types::I64)); - ctx.emit(Inst::alu_rmi_r( - OperandSize::Size64, - AluRmiROpcode::And, - RegMemImm::imm(64), - amt, - )); - ctx.emit(Inst::cmove( - OperandSize::Size64, - CC::Z, - RegMem::reg(tmp3.to_reg()), - dst_hi, - )); - ctx.emit(Inst::cmove( - OperandSize::Size64, - CC::Z, - RegMem::reg(tmp1.to_reg()), - dst_lo, - )); - ctx.emit(Inst::cmove( - OperandSize::Size64, - CC::NZ, - RegMem::reg(tmp1.to_reg()), - dst_hi, - )); -} - -fn emit_shr_i128>( - ctx: &mut C, - src: ValueRegs, - dst: ValueRegs>, - amt_src: Reg, - is_signed: bool, -) { - let src_lo = src.regs()[0]; - let src_hi = src.regs()[1]; - let dst_lo = dst.regs()[0]; - let dst_hi = dst.regs()[1]; - - // mov tmp1, src_hi - // {u,s}shr tmp1, amt_src - // mov tmp2, src_lo - // ushr tmp2, amt_src - // mov amt, 64 - // sub amt, amt_src - // mov tmp3, src_hi - // shl tmp3, amt - // xor dst_lo, dst_lo - // test amt_src, 127 - // cmovz tmp3, dst_lo - // or tmp3, tmp2 - // if is_signed: - // mov dst_hi, src_hi - // sshr dst_hi, 63 // get the sign bit - // else: - // xor dst_hi, dst_hi - // mov amt, amt_src - // and amt, 64 - // cmovz dst_hi, tmp1 - // cmovz dst_lo, tmp3 - // cmovnz dst_lo, tmp1 - - let tmp1 = ctx.alloc_tmp(types::I64).only_reg().unwrap(); - let tmp2 = ctx.alloc_tmp(types::I64).only_reg().unwrap(); - let tmp3 = ctx.alloc_tmp(types::I64).only_reg().unwrap(); - let amt = ctx.alloc_tmp(types::I64).only_reg().unwrap(); - - let shift_kind = if is_signed { - ShiftKind::ShiftRightArithmetic - } else { - ShiftKind::ShiftRightLogical - }; - - ctx.emit(Inst::gen_move(tmp1, src_hi, types::I64)); - ctx.emit(Inst::gen_move( - Writable::from_reg(regs::rcx()), - amt_src, - types::I64, - )); - ctx.emit(Inst::shift_r(OperandSize::Size64, shift_kind, None, tmp1)); - - ctx.emit(Inst::gen_move(tmp2, src_lo, types::I64)); - ctx.emit(Inst::gen_move( - Writable::from_reg(regs::rcx()), - amt_src, - types::I64, - )); - // N.B.: right-shift of *lower* half is *always* unsigned (its MSB is not a sign bit). - ctx.emit(Inst::shift_r( - OperandSize::Size64, - ShiftKind::ShiftRightLogical, - None, - tmp2, - )); - - ctx.emit(Inst::imm(OperandSize::Size64, 64, amt)); - ctx.emit(Inst::alu_rmi_r( - OperandSize::Size64, - AluRmiROpcode::Sub, - RegMemImm::reg(amt_src), - amt, - )); - - ctx.emit(Inst::gen_move(tmp3, src_hi, types::I64)); - ctx.emit(Inst::gen_move( - Writable::from_reg(regs::rcx()), - amt.to_reg(), - types::I64, - )); - ctx.emit(Inst::shift_r( - OperandSize::Size64, - ShiftKind::ShiftLeft, - None, - tmp3, - )); - - ctx.emit(Inst::alu_rmi_r( - OperandSize::Size64, - AluRmiROpcode::Xor, - RegMemImm::reg(dst_lo.to_reg()), - dst_lo, - )); - ctx.emit(Inst::test_rmi_r( - OperandSize::Size64, - RegMemImm::imm(127), - amt_src, - )); - ctx.emit(Inst::cmove( - OperandSize::Size64, - CC::Z, - RegMem::reg(dst_lo.to_reg()), - tmp3, - )); - - ctx.emit(Inst::alu_rmi_r( - OperandSize::Size64, - AluRmiROpcode::Or, - RegMemImm::reg(tmp2.to_reg()), - tmp3, - )); - - if is_signed { - ctx.emit(Inst::gen_move(dst_hi, src_hi, types::I64)); - ctx.emit(Inst::shift_r( - OperandSize::Size64, - ShiftKind::ShiftRightArithmetic, - Some(63), - dst_hi, - )); - } else { - ctx.emit(Inst::alu_rmi_r( - OperandSize::Size64, - AluRmiROpcode::Xor, - RegMemImm::reg(dst_hi.to_reg()), - dst_hi, - )); - } - // This isn't semantically necessary, but it keeps the - // register allocator happy, because it cannot otherwise - // infer that cmovz + cmovnz always defines dst_lo. - ctx.emit(Inst::alu_rmi_r( - OperandSize::Size64, - AluRmiROpcode::Xor, - RegMemImm::reg(dst_lo.to_reg()), - dst_lo, - )); - - ctx.emit(Inst::gen_move(amt, amt_src, types::I64)); - ctx.emit(Inst::alu_rmi_r( - OperandSize::Size64, - AluRmiROpcode::And, - RegMemImm::imm(64), - amt, - )); - ctx.emit(Inst::cmove( - OperandSize::Size64, - CC::Z, - RegMem::reg(tmp1.to_reg()), - dst_hi, - )); - ctx.emit(Inst::cmove( - OperandSize::Size64, - CC::Z, - RegMem::reg(tmp3.to_reg()), - dst_lo, - )); - ctx.emit(Inst::cmove( - OperandSize::Size64, - CC::NZ, - RegMem::reg(tmp1.to_reg()), - dst_lo, - )); -} - fn make_libcall_sig>( ctx: &mut C, insn: IRInst, @@ -1541,142 +1231,12 @@ fn lower_insn_to_regs>( | Opcode::Vselect | Opcode::Ushr | Opcode::Sshr - | Opcode::Ishl => implemented_in_isle(ctx), - - Opcode::Rotl | Opcode::Rotr => { - let dst_ty = ctx.output_ty(insn, 0); - debug_assert_eq!(ctx.input_ty(insn, 0), dst_ty); - - if !dst_ty.is_vector() && dst_ty.bits() <= 64 { - if op != Opcode::Rotr { - implemented_in_isle(ctx); - } - - // Scalar shifts on x86 have various encodings: - // - shift by one bit, e.g. `SAL r/m8, 1` (not used here) - // - shift by an immediate amount, e.g. `SAL r/m8, imm8` - // - shift by a dynamic amount but only from the CL register, e.g. `SAL r/m8, CL`. - // This implementation uses the last two encoding methods. - let (size, lhs) = match dst_ty { - types::I8 | types::I16 => match op { - Opcode::Rotr => ( - OperandSize::from_ty(dst_ty), - put_input_in_reg(ctx, inputs[0]), - ), - _ => unreachable!(), - }, - types::I32 | types::I64 => ( - OperandSize::from_ty(dst_ty), - put_input_in_reg(ctx, inputs[0]), - ), - _ => unreachable!("unhandled output type for shift/rotates: {}", dst_ty), - }; - - let (count, rhs) = - if let Some(cst) = ctx.get_input_as_source_or_const(insn, 1).constant { - // Mask count, according to Cranelift's semantics. - let cst = (cst as u8) & (dst_ty.bits() as u8 - 1); - (Some(cst), None) - } else { - // We can ignore upper registers if shift amount is multi-reg, because we - // are taking the shift amount mod 2^(lhs_width) anyway. - (None, Some(put_input_in_regs(ctx, inputs[1]).regs()[0])) - }; - - let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); - - let shift_kind = match op { - Opcode::Rotr => ShiftKind::RotateRight, - _ => unreachable!(), - }; - - let w_rcx = Writable::from_reg(regs::rcx()); - ctx.emit(Inst::mov_r_r(OperandSize::Size64, lhs, dst)); - if count.is_none() { - ctx.emit(Inst::mov_r_r(OperandSize::Size64, rhs.unwrap(), w_rcx)); - } - ctx.emit(Inst::shift_r(size, shift_kind, count, dst)); - } else if dst_ty == types::I128 { - let amt_src = put_input_in_regs(ctx, inputs[1]).regs()[0]; - let src = put_input_in_regs(ctx, inputs[0]); - let dst = get_output_reg(ctx, outputs[0]); - - match op { - Opcode::Rotr => { - // (mov tmp, src) - // (ushr.i128 tmp, amt) - // (mov dst, src) - // (shl.i128 dst, 128-amt) - // (or dst, tmp) - let tmp = ctx.alloc_tmp(types::I128); - emit_shr_i128(ctx, src, tmp, amt_src, /* is_signed = */ false); - let inv_amt = ctx.alloc_tmp(types::I64).only_reg().unwrap(); - ctx.emit(Inst::imm(OperandSize::Size64, 128, inv_amt)); - ctx.emit(Inst::alu_rmi_r( - OperandSize::Size64, - AluRmiROpcode::Sub, - RegMemImm::reg(amt_src), - inv_amt, - )); - emit_shl_i128(ctx, src, dst, inv_amt.to_reg()); - ctx.emit(Inst::alu_rmi_r( - OperandSize::Size64, - AluRmiROpcode::Or, - RegMemImm::reg(tmp.regs()[0].to_reg()), - dst.regs()[0], - )); - ctx.emit(Inst::alu_rmi_r( - OperandSize::Size64, - AluRmiROpcode::Or, - RegMemImm::reg(tmp.regs()[1].to_reg()), - dst.regs()[1], - )); - } - _ => unreachable!(), - } - } else { - implemented_in_isle(ctx); - } - } - - Opcode::Ineg => { - let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); - let ty = ty.unwrap(); - - if ty.is_vector() { - // Zero's out a register and then does a packed subtraction - // of the input from the register. - - let src = input_to_reg_mem(ctx, inputs[0]); - let tmp = ctx.alloc_tmp(types::I32X4).only_reg().unwrap(); - - let subtract_opcode = match ty { - types::I8X16 => SseOpcode::Psubb, - types::I16X8 => SseOpcode::Psubw, - types::I32X4 => SseOpcode::Psubd, - types::I64X2 => SseOpcode::Psubq, - _ => panic!("Unsupported type for Ineg instruction, found {}", ty), - }; - - // Note we must zero out a tmp instead of using the destination register since - // the desitnation could be an alias for the source input register - ctx.emit(Inst::xmm_rm_r( - SseOpcode::Pxor, - RegMem::reg(tmp.to_reg()), - tmp, - )); - ctx.emit(Inst::xmm_rm_r(subtract_opcode, src, tmp)); - ctx.emit(Inst::xmm_unary_rm_r( - SseOpcode::Movapd, - RegMem::reg(tmp.to_reg()), - dst, - )); - } else { - let src = put_input_in_reg(ctx, inputs[0]); - ctx.emit(Inst::gen_move(dst, src, ty)); - ctx.emit(Inst::neg(OperandSize::from_ty(ty), dst)); - } - } + | Opcode::Ishl + | Opcode::Rotl + | Opcode::Rotr + | Opcode::Ineg + | Opcode::Trap + | Opcode::ResumableTrap => implemented_in_isle(ctx), Opcode::Clz => { let orig_ty = ty.unwrap(); @@ -2811,11 +2371,6 @@ fn lower_insn_to_regs>( ctx.emit(Inst::Hlt); } - Opcode::Trap | Opcode::ResumableTrap => { - let trap_code = ctx.data(insn).trap_code().unwrap(); - ctx.emit_safepoint(Inst::Ud2 { trap_code }); - } - Opcode::Trapif | Opcode::Trapff => { let trap_code = ctx.data(insn).trap_code().unwrap(); diff --git a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest index 2ed8c1a2e325..5a897a60088a 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest +++ b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest @@ -1,4 +1,4 @@ src/clif.isle f176ef3bba99365 -src/prelude.isle 7b911d3b894ae17 -src/isa/x64/inst.isle 41304d8ef6f7d816 -src/isa/x64/lower.isle 4689585f55f41438 +src/prelude.isle 22dd5ff133398960 +src/isa/x64/inst.isle 61004acbb1289816 +src/isa/x64/lower.isle 82db7f7d47ac7809 diff --git a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs index 354ca5719279..a31e0583c1f7 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs +++ b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs @@ -24,6 +24,7 @@ pub trait Context { fn u8_and(&mut self, arg0: u8, arg1: u8) -> u8; fn value_reg(&mut self, arg0: Reg) -> ValueRegs; fn value_regs(&mut self, arg0: Reg, arg1: Reg) -> ValueRegs; + fn value_regs_invalid(&mut self) -> ValueRegs; fn temp_writable_reg(&mut self, arg0: Type) -> WritableReg; fn invalid_reg(&mut self) -> Reg; fn put_in_reg(&mut self, arg0: Value) -> Reg; @@ -89,19 +90,25 @@ pub trait Context { fn sse_insertps_lane_imm(&mut self, arg0: u8) -> u8; } -/// Internal type ProducesFlags: defined at src/prelude.isle line 277. +/// Internal type SideEffectNoResult: defined at src/prelude.isle line 279. +#[derive(Clone, Debug)] +pub enum SideEffectNoResult { + Inst { inst: MInst }, +} + +/// Internal type ProducesFlags: defined at src/prelude.isle line 292. #[derive(Clone, Debug)] pub enum ProducesFlags { ProducesFlags { inst: MInst, result: Reg }, } -/// Internal type ConsumesFlags: defined at src/prelude.isle line 280. +/// Internal type ConsumesFlags: defined at src/prelude.isle line 295. #[derive(Clone, Debug)] pub enum ConsumesFlags { ConsumesFlags { inst: MInst, result: Reg }, } -/// Internal type ExtendKind: defined at src/isa/x64/inst.isle line 461. +/// Internal type ExtendKind: defined at src/isa/x64/inst.isle line 464. #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub enum ExtendKind { Sign, @@ -111,7 +118,7 @@ pub enum ExtendKind { // Generated as internal constructor for term temp_reg. pub fn constructor_temp_reg(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; - // Rule at src/prelude.isle line 66. + // Rule at src/prelude.isle line 70. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); return Some(expr1_0); @@ -120,13 +127,31 @@ pub fn constructor_temp_reg(ctx: &mut C, arg0: Type) -> Option // Generated as internal constructor for term lo_reg. pub fn constructor_lo_reg(ctx: &mut C, arg0: Value) -> Option { let pattern0_0 = arg0; - // Rule at src/prelude.isle line 101. + // Rule at src/prelude.isle line 105. let expr0_0 = C::put_in_regs(ctx, pattern0_0); let expr1_0: usize = 0; let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); return Some(expr2_0); } +// Generated as internal constructor for term value_regs_none. +pub fn constructor_value_regs_none( + ctx: &mut C, + arg0: &SideEffectNoResult, +) -> Option { + let pattern0_0 = arg0; + if let &SideEffectNoResult::Inst { + inst: ref pattern1_0, + } = pattern0_0 + { + // Rule at src/prelude.isle line 284. + let expr0_0 = C::emit(ctx, &pattern1_0); + let expr1_0 = C::value_regs_invalid(ctx); + return Some(expr1_0); + } + return None; +} + // Generated as internal constructor for term with_flags. pub fn constructor_with_flags( ctx: &mut C, @@ -145,7 +170,7 @@ pub fn constructor_with_flags( result: pattern3_1, } = pattern2_0 { - // Rule at src/prelude.isle line 290. + // Rule at src/prelude.isle line 305. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern3_0); let expr2_0 = C::value_regs(ctx, pattern1_1, pattern3_1); @@ -173,7 +198,7 @@ pub fn constructor_with_flags_1( result: pattern3_1, } = pattern2_0 { - // Rule at src/prelude.isle line 298. + // Rule at src/prelude.isle line 313. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern3_0); return Some(pattern3_1); @@ -207,7 +232,7 @@ pub fn constructor_with_flags_2( result: pattern5_1, } = pattern4_0 { - // Rule at src/prelude.isle line 308. + // Rule at src/prelude.isle line 323. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern5_0); let expr2_0 = C::emit(ctx, &pattern3_0); @@ -224,22 +249,22 @@ pub fn constructor_operand_size_bits(ctx: &mut C, arg0: &OperandSize let pattern0_0 = arg0; match pattern0_0 { &OperandSize::Size8 => { - // Rule at src/isa/x64/inst.isle line 95. + // Rule at src/isa/x64/inst.isle line 98. let expr0_0: u16 = 8; return Some(expr0_0); } &OperandSize::Size16 => { - // Rule at src/isa/x64/inst.isle line 96. + // Rule at src/isa/x64/inst.isle line 99. let expr0_0: u16 = 16; return Some(expr0_0); } &OperandSize::Size32 => { - // Rule at src/isa/x64/inst.isle line 97. + // Rule at src/isa/x64/inst.isle line 100. let expr0_0: u16 = 32; return Some(expr0_0); } &OperandSize::Size64 => { - // Rule at src/isa/x64/inst.isle line 98. + // Rule at src/isa/x64/inst.isle line 101. let expr0_0: u16 = 64; return Some(expr0_0); } @@ -260,12 +285,12 @@ pub fn constructor_extend_to_reg( let pattern2_0 = arg1; if pattern2_0 == pattern1_0 { let pattern4_0 = arg2; - // Rule at src/isa/x64/inst.isle line 473. + // Rule at src/isa/x64/inst.isle line 476. let expr0_0 = C::put_in_reg(ctx, pattern0_0); return Some(expr0_0); } let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 476. + // Rule at src/isa/x64/inst.isle line 479. let expr0_0 = C::ty_bits_u16(ctx, pattern1_0); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern2_0); let expr2_0 = constructor_operand_size_bits(ctx, &expr1_0)?; @@ -289,7 +314,7 @@ pub fn constructor_extend( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/inst.isle line 496. + // Rule at src/isa/x64/inst.isle line 499. let expr0_0 = constructor_movsx(ctx, pattern2_0, pattern3_0, pattern4_0)?; return Some(expr0_0); } @@ -297,7 +322,7 @@ pub fn constructor_extend( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/inst.isle line 492. + // Rule at src/isa/x64/inst.isle line 495. let expr0_0 = constructor_movzx(ctx, pattern2_0, pattern3_0, pattern4_0)?; return Some(expr0_0); } @@ -310,17 +335,17 @@ pub fn constructor_extend( pub fn constructor_sse_xor_op(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32X4 { - // Rule at src/isa/x64/inst.isle line 503. + // Rule at src/isa/x64/inst.isle line 506. let expr0_0 = SseOpcode::Xorps; return Some(expr0_0); } if pattern0_0 == F64X2 { - // Rule at src/isa/x64/inst.isle line 504. + // Rule at src/isa/x64/inst.isle line 507. let expr0_0 = SseOpcode::Xorpd; return Some(expr0_0); } if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { - // Rule at src/isa/x64/inst.isle line 505. + // Rule at src/isa/x64/inst.isle line 508. let expr0_0 = SseOpcode::Pxor; return Some(expr0_0); } @@ -337,7 +362,7 @@ pub fn constructor_sse_xor( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 509. + // Rule at src/isa/x64/inst.isle line 512. let expr0_0 = constructor_sse_xor_op(ctx, pattern0_0)?; let expr1_0 = constructor_xmm_rm_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -347,40 +372,40 @@ pub fn constructor_sse_xor( pub fn constructor_sse_cmp_op(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32X4 { - // Rule at src/isa/x64/inst.isle line 518. + // Rule at src/isa/x64/inst.isle line 521. let expr0_0 = SseOpcode::Cmpps; return Some(expr0_0); } if pattern0_0 == F64X2 { - // Rule at src/isa/x64/inst.isle line 519. + // Rule at src/isa/x64/inst.isle line 522. let expr0_0 = SseOpcode::Cmppd; return Some(expr0_0); } if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { if pattern1_0 == 8 { if pattern1_1 == 16 { - // Rule at src/isa/x64/inst.isle line 514. + // Rule at src/isa/x64/inst.isle line 517. let expr0_0 = SseOpcode::Pcmpeqb; return Some(expr0_0); } } if pattern1_0 == 16 { if pattern1_1 == 8 { - // Rule at src/isa/x64/inst.isle line 515. + // Rule at src/isa/x64/inst.isle line 518. let expr0_0 = SseOpcode::Pcmpeqw; return Some(expr0_0); } } if pattern1_0 == 32 { if pattern1_1 == 4 { - // Rule at src/isa/x64/inst.isle line 516. + // Rule at src/isa/x64/inst.isle line 519. let expr0_0 = SseOpcode::Pcmpeqd; return Some(expr0_0); } } if pattern1_0 == 64 { if pattern1_1 == 2 { - // Rule at src/isa/x64/inst.isle line 517. + // Rule at src/isa/x64/inst.isle line 520. let expr0_0 = SseOpcode::Pcmpeqq; return Some(expr0_0); } @@ -392,7 +417,7 @@ pub fn constructor_sse_cmp_op(ctx: &mut C, arg0: Type) -> Option(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 533. + // Rule at src/isa/x64/inst.isle line 536. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); let expr2_0: Type = I32X4; @@ -416,7 +441,7 @@ pub fn constructor_make_i64x2_from_lanes( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 544. + // Rule at src/isa/x64/inst.isle line 547. let expr0_0: Type = I64X2; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = C::writable_reg_to_reg(ctx, expr1_0); @@ -457,11 +482,11 @@ pub fn constructor_reg_mem_imm_to_xmm( let pattern0_0 = arg0; match pattern0_0 { &RegMemImm::Imm { simm32: pattern1_0 } => { - // Rule at src/isa/x64/inst.isle line 565. + // Rule at src/isa/x64/inst.isle line 568. return Some(pattern0_0.clone()); } &RegMemImm::Reg { reg: pattern1_0 } => { - // Rule at src/isa/x64/inst.isle line 566. + // Rule at src/isa/x64/inst.isle line 569. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Movd; let expr2_0 = RegMem::Reg { reg: pattern1_0 }; @@ -473,7 +498,7 @@ pub fn constructor_reg_mem_imm_to_xmm( &RegMemImm::Mem { addr: ref pattern1_0, } => { - // Rule at src/isa/x64/inst.isle line 564. + // Rule at src/isa/x64/inst.isle line 567. return Some(pattern0_0.clone()); } _ => {} @@ -492,7 +517,7 @@ pub fn constructor_x64_load( if pattern0_0 == I64 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 582. + // Rule at src/isa/x64/inst.isle line 585. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::Mov64MR { @@ -506,7 +531,7 @@ pub fn constructor_x64_load( if pattern0_0 == F32 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 587. + // Rule at src/isa/x64/inst.isle line 590. let expr0_0 = SseOpcode::Movss; let expr1_0 = C::synthetic_amode_to_reg_mem(ctx, pattern2_0); let expr2_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, &expr1_0)?; @@ -515,7 +540,7 @@ pub fn constructor_x64_load( if pattern0_0 == F64 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 591. + // Rule at src/isa/x64/inst.isle line 594. let expr0_0 = SseOpcode::Movsd; let expr1_0 = C::synthetic_amode_to_reg_mem(ctx, pattern2_0); let expr2_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, &expr1_0)?; @@ -524,7 +549,7 @@ pub fn constructor_x64_load( if pattern0_0 == F32X4 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 595. + // Rule at src/isa/x64/inst.isle line 598. let expr0_0 = SseOpcode::Movups; let expr1_0 = C::synthetic_amode_to_reg_mem(ctx, pattern2_0); let expr2_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, &expr1_0)?; @@ -533,7 +558,7 @@ pub fn constructor_x64_load( if pattern0_0 == F64X2 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 599. + // Rule at src/isa/x64/inst.isle line 602. let expr0_0 = SseOpcode::Movupd; let expr1_0 = C::synthetic_amode_to_reg_mem(ctx, pattern2_0); let expr2_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, &expr1_0)?; @@ -542,7 +567,7 @@ pub fn constructor_x64_load( if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 603. + // Rule at src/isa/x64/inst.isle line 606. let expr0_0 = SseOpcode::Movdqu; let expr1_0 = C::synthetic_amode_to_reg_mem(ctx, pattern2_0); let expr2_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, &expr1_0)?; @@ -552,7 +577,7 @@ pub fn constructor_x64_load( let pattern2_0 = arg1; let pattern3_0 = arg2; if let &ExtKind::SignExtend = pattern3_0 { - // Rule at src/isa/x64/inst.isle line 577. + // Rule at src/isa/x64/inst.isle line 580. let expr0_0 = C::ty_bytes(ctx, pattern1_0); let expr1_0: u16 = 8; let expr2_0 = C::ext_mode(ctx, expr0_0, expr1_0); @@ -576,7 +601,7 @@ pub fn constructor_alu_rmi_r( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 622. + // Rule at src/isa/x64/inst.isle line 625. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = MInst::AluRmiR { @@ -601,7 +626,7 @@ pub fn constructor_add( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 630. + // Rule at src/isa/x64/inst.isle line 633. let expr0_0 = AluRmiROpcode::Add; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -617,7 +642,7 @@ pub fn constructor_add_with_flags( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 638. + // Rule at src/isa/x64/inst.isle line 641. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = AluRmiROpcode::Add; @@ -646,7 +671,7 @@ pub fn constructor_adc( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 649. + // Rule at src/isa/x64/inst.isle line 652. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = AluRmiROpcode::Adc; @@ -675,7 +700,7 @@ pub fn constructor_sub( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 660. + // Rule at src/isa/x64/inst.isle line 663. let expr0_0 = AluRmiROpcode::Sub; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -691,7 +716,7 @@ pub fn constructor_sub_with_flags( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 668. + // Rule at src/isa/x64/inst.isle line 671. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = AluRmiROpcode::Sub; @@ -720,7 +745,7 @@ pub fn constructor_sbb( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 679. + // Rule at src/isa/x64/inst.isle line 682. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = AluRmiROpcode::Sbb; @@ -749,14 +774,14 @@ pub fn constructor_mul( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 690. + // Rule at src/isa/x64/inst.isle line 693. let expr0_0 = AluRmiROpcode::Mul; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); } -// Generated as internal constructor for term m_and. -pub fn constructor_m_and( +// Generated as internal constructor for term x64_and. +pub fn constructor_x64_and( ctx: &mut C, arg0: Type, arg1: Reg, @@ -765,7 +790,7 @@ pub fn constructor_m_and( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 701. + // Rule at src/isa/x64/inst.isle line 704. let expr0_0 = AluRmiROpcode::And; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -781,7 +806,7 @@ pub fn constructor_or( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 709. + // Rule at src/isa/x64/inst.isle line 712. let expr0_0 = AluRmiROpcode::Or; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -797,7 +822,7 @@ pub fn constructor_xor( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 717. + // Rule at src/isa/x64/inst.isle line 720. let expr0_0 = AluRmiROpcode::Xor; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -809,7 +834,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option if pattern0_0 == I64 { let pattern2_0 = arg1; if let Some(pattern3_0) = C::nonzero_u64_fits_in_u32(ctx, pattern2_0) { - // Rule at src/isa/x64/inst.isle line 746. + // Rule at src/isa/x64/inst.isle line 749. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = OperandSize::Size32; @@ -826,7 +851,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option if pattern0_0 == F32 { let pattern2_0 = arg1; if pattern2_0 == 0 { - // Rule at src/isa/x64/inst.isle line 775. + // Rule at src/isa/x64/inst.isle line 778. let expr0_0: Type = F32; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = C::writable_reg_to_reg(ctx, expr1_0); @@ -841,7 +866,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option let expr6_0 = C::emit(ctx, &expr5_0); return Some(expr2_0); } - // Rule at src/isa/x64/inst.isle line 734. + // Rule at src/isa/x64/inst.isle line 737. let expr0_0: Type = F32; let expr1_0 = SseOpcode::Movd; let expr2_0: Type = I32; @@ -854,7 +879,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option if pattern0_0 == F64 { let pattern2_0 = arg1; if pattern2_0 == 0 { - // Rule at src/isa/x64/inst.isle line 787. + // Rule at src/isa/x64/inst.isle line 790. let expr0_0: Type = F64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = C::writable_reg_to_reg(ctx, expr1_0); @@ -869,7 +894,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option let expr6_0 = C::emit(ctx, &expr5_0); return Some(expr2_0); } - // Rule at src/isa/x64/inst.isle line 738. + // Rule at src/isa/x64/inst.isle line 741. let expr0_0: Type = F64; let expr1_0 = SseOpcode::Movq; let expr2_0: Type = I64; @@ -882,7 +907,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { let pattern2_0 = arg1; if pattern2_0 == 0 { - // Rule at src/isa/x64/inst.isle line 765. + // Rule at src/isa/x64/inst.isle line 768. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); let expr2_0 = constructor_sse_xor_op(ctx, pattern0_0)?; @@ -897,35 +922,38 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option return Some(expr1_0); } } - let pattern1_0 = arg1; - if pattern1_0 == 0 { - // Rule at src/isa/x64/inst.isle line 752. - let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); - let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); - let expr2_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); - let expr3_0 = AluRmiROpcode::Xor; - let expr4_0 = RegMemImm::Reg { reg: expr1_0 }; - let expr5_0 = MInst::AluRmiR { - size: expr2_0, - op: expr3_0, - src1: expr1_0, - src2: expr4_0, + if let Some(pattern1_0) = C::fits_in_64(ctx, pattern0_0) { + let pattern2_0 = arg1; + if pattern2_0 == 0 { + // Rule at src/isa/x64/inst.isle line 755. + let expr0_0 = C::temp_writable_reg(ctx, pattern1_0); + let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); + let expr2_0 = C::operand_size_of_type_32_64(ctx, pattern1_0); + let expr3_0 = AluRmiROpcode::Xor; + let expr4_0 = RegMemImm::Reg { reg: expr1_0 }; + let expr5_0 = MInst::AluRmiR { + size: expr2_0, + op: expr3_0, + src1: expr1_0, + src2: expr4_0, + dst: expr0_0, + }; + let expr6_0 = C::emit(ctx, &expr5_0); + return Some(expr1_0); + } + // Rule at src/isa/x64/inst.isle line 730. + let expr0_0 = C::temp_writable_reg(ctx, pattern1_0); + let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern1_0); + let expr2_0 = MInst::Imm { + dst_size: expr1_0, + simm64: pattern2_0, dst: expr0_0, }; - let expr6_0 = C::emit(ctx, &expr5_0); - return Some(expr1_0); + let expr3_0 = C::emit(ctx, &expr2_0); + let expr4_0 = C::writable_reg_to_reg(ctx, expr0_0); + return Some(expr4_0); } - // Rule at src/isa/x64/inst.isle line 727. - let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); - let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); - let expr2_0 = MInst::Imm { - dst_size: expr1_0, - simm64: pattern1_0, - dst: expr0_0, - }; - let expr3_0 = C::emit(ctx, &expr2_0); - let expr4_0 = C::writable_reg_to_reg(ctx, expr0_0); - return Some(expr4_0); + return None; } // Generated as internal constructor for term shift_r. @@ -940,7 +968,7 @@ pub fn constructor_shift_r( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 800. + // Rule at src/isa/x64/inst.isle line 803. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::raw_operand_size_of_type(ctx, pattern0_0); let expr2_0 = MInst::ShiftR { @@ -955,8 +983,8 @@ pub fn constructor_shift_r( return Some(expr4_0); } -// Generated as internal constructor for term m_rotl. -pub fn constructor_m_rotl( +// Generated as internal constructor for term x64_rotl. +pub fn constructor_x64_rotl( ctx: &mut C, arg0: Type, arg1: Reg, @@ -965,12 +993,28 @@ pub fn constructor_m_rotl( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 811. + // Rule at src/isa/x64/inst.isle line 814. let expr0_0 = ShiftKind::RotateLeft; let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); } +// Generated as internal constructor for term x64_rotr. +pub fn constructor_x64_rotr( + ctx: &mut C, + arg0: Type, + arg1: Reg, + arg2: &Imm8Reg, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + // Rule at src/isa/x64/inst.isle line 820. + let expr0_0 = ShiftKind::RotateRight; + let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; + return Some(expr1_0); +} + // Generated as internal constructor for term shl. pub fn constructor_shl( ctx: &mut C, @@ -981,7 +1025,7 @@ pub fn constructor_shl( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 816. + // Rule at src/isa/x64/inst.isle line 825. let expr0_0 = ShiftKind::ShiftLeft; let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -997,7 +1041,7 @@ pub fn constructor_shr( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 821. + // Rule at src/isa/x64/inst.isle line 830. let expr0_0 = ShiftKind::ShiftRightLogical; let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -1013,7 +1057,7 @@ pub fn constructor_sar( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 826. + // Rule at src/isa/x64/inst.isle line 835. let expr0_0 = ShiftKind::ShiftRightArithmetic; let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -1031,7 +1075,7 @@ pub fn constructor_cmp_rmi_r( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 831. + // Rule at src/isa/x64/inst.isle line 840. let expr0_0 = MInst::CmpRmiR { size: pattern0_0.clone(), opcode: pattern1_0.clone(), @@ -1056,7 +1100,7 @@ pub fn constructor_cmp( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 840. + // Rule at src/isa/x64/inst.isle line 849. let expr0_0 = CmpOpcode::Cmp; let expr1_0 = constructor_cmp_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -1072,7 +1116,7 @@ pub fn constructor_test( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 845. + // Rule at src/isa/x64/inst.isle line 854. let expr0_0 = CmpOpcode::Test; let expr1_0 = constructor_cmp_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -1090,7 +1134,7 @@ pub fn constructor_cmove( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 850. + // Rule at src/isa/x64/inst.isle line 859. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = MInst::Cmove { @@ -1118,7 +1162,7 @@ pub fn constructor_movzx( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 858. + // Rule at src/isa/x64/inst.isle line 867. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::MovzxRmR { ext_mode: pattern1_0.clone(), @@ -1140,7 +1184,7 @@ pub fn constructor_movsx( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 865. + // Rule at src/isa/x64/inst.isle line 874. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::MovsxRmR { ext_mode: pattern1_0.clone(), @@ -1164,7 +1208,7 @@ pub fn constructor_xmm_rm_r( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 872. + // Rule at src/isa/x64/inst.isle line 881. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::XmmRmR { op: pattern1_0.clone(), @@ -1181,7 +1225,7 @@ pub fn constructor_xmm_rm_r( pub fn constructor_paddb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 879. + // Rule at src/isa/x64/inst.isle line 888. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Paddb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1192,7 +1236,7 @@ pub fn constructor_paddb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_paddw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 884. + // Rule at src/isa/x64/inst.isle line 893. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Paddw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1203,7 +1247,7 @@ pub fn constructor_paddw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_paddd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 889. + // Rule at src/isa/x64/inst.isle line 898. let expr0_0: Type = I32X4; let expr1_0 = SseOpcode::Paddd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1214,7 +1258,7 @@ pub fn constructor_paddd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_paddq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 894. + // Rule at src/isa/x64/inst.isle line 903. let expr0_0: Type = I64X2; let expr1_0 = SseOpcode::Paddq; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1225,7 +1269,7 @@ pub fn constructor_paddq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_paddsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 899. + // Rule at src/isa/x64/inst.isle line 908. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Paddsb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1236,7 +1280,7 @@ pub fn constructor_paddsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_paddsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 904. + // Rule at src/isa/x64/inst.isle line 913. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Paddsw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1247,7 +1291,7 @@ pub fn constructor_paddsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_paddusb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 909. + // Rule at src/isa/x64/inst.isle line 918. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Paddusb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1258,7 +1302,7 @@ pub fn constructor_paddusb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_paddusw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 914. + // Rule at src/isa/x64/inst.isle line 923. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Paddusw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1269,7 +1313,7 @@ pub fn constructor_paddusw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_psubb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 919. + // Rule at src/isa/x64/inst.isle line 928. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Psubb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1280,7 +1324,7 @@ pub fn constructor_psubb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_psubw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 924. + // Rule at src/isa/x64/inst.isle line 933. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Psubw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1291,7 +1335,7 @@ pub fn constructor_psubw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_psubd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 929. + // Rule at src/isa/x64/inst.isle line 938. let expr0_0: Type = I32X4; let expr1_0 = SseOpcode::Psubd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1302,7 +1346,7 @@ pub fn constructor_psubd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_psubq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 934. + // Rule at src/isa/x64/inst.isle line 943. let expr0_0: Type = I64X2; let expr1_0 = SseOpcode::Psubq; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1313,7 +1357,7 @@ pub fn constructor_psubq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_psubsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 939. + // Rule at src/isa/x64/inst.isle line 948. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Psubsb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1324,7 +1368,7 @@ pub fn constructor_psubsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_psubsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 944. + // Rule at src/isa/x64/inst.isle line 953. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Psubsw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1335,7 +1379,7 @@ pub fn constructor_psubsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_psubusb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 949. + // Rule at src/isa/x64/inst.isle line 958. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Psubusb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1346,7 +1390,7 @@ pub fn constructor_psubusb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_psubusw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 954. + // Rule at src/isa/x64/inst.isle line 963. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Psubusw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1357,7 +1401,7 @@ pub fn constructor_psubusw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pavgb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 959. + // Rule at src/isa/x64/inst.isle line 968. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pavgb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1368,7 +1412,7 @@ pub fn constructor_pavgb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_pavgw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 964. + // Rule at src/isa/x64/inst.isle line 973. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pavgw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1379,7 +1423,7 @@ pub fn constructor_pavgw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_pand(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 969. + // Rule at src/isa/x64/inst.isle line 978. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Pand; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1390,7 +1434,7 @@ pub fn constructor_pand(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Op pub fn constructor_andps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 974. + // Rule at src/isa/x64/inst.isle line 983. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Andps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1401,7 +1445,7 @@ pub fn constructor_andps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_andpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 979. + // Rule at src/isa/x64/inst.isle line 988. let expr0_0: Type = F64X2; let expr1_0 = SseOpcode::Andpd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1412,7 +1456,7 @@ pub fn constructor_andpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_por(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 984. + // Rule at src/isa/x64/inst.isle line 993. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Por; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1423,7 +1467,7 @@ pub fn constructor_por(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Opt pub fn constructor_orps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 989. + // Rule at src/isa/x64/inst.isle line 998. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Orps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1434,7 +1478,7 @@ pub fn constructor_orps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Op pub fn constructor_orpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 994. + // Rule at src/isa/x64/inst.isle line 1003. let expr0_0: Type = F64X2; let expr1_0 = SseOpcode::Orpd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1445,7 +1489,7 @@ pub fn constructor_orpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Op pub fn constructor_pxor(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 999. + // Rule at src/isa/x64/inst.isle line 1008. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pxor; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1456,7 +1500,7 @@ pub fn constructor_pxor(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Op pub fn constructor_xorps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1004. + // Rule at src/isa/x64/inst.isle line 1013. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Xorps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1467,7 +1511,7 @@ pub fn constructor_xorps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_xorpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1009. + // Rule at src/isa/x64/inst.isle line 1018. let expr0_0: Type = F64X2; let expr1_0 = SseOpcode::Xorpd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1478,7 +1522,7 @@ pub fn constructor_xorpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_pmullw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1014. + // Rule at src/isa/x64/inst.isle line 1023. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pmullw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1489,7 +1533,7 @@ pub fn constructor_pmullw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmulld(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1019. + // Rule at src/isa/x64/inst.isle line 1028. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pmulld; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1500,7 +1544,7 @@ pub fn constructor_pmulld(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmulhw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1024. + // Rule at src/isa/x64/inst.isle line 1033. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pmulhw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1511,7 +1555,7 @@ pub fn constructor_pmulhw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmulhuw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1029. + // Rule at src/isa/x64/inst.isle line 1038. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pmulhuw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1522,7 +1566,7 @@ pub fn constructor_pmulhuw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmuldq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1034. + // Rule at src/isa/x64/inst.isle line 1043. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pmuldq; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1533,7 +1577,7 @@ pub fn constructor_pmuldq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmuludq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1039. + // Rule at src/isa/x64/inst.isle line 1048. let expr0_0: Type = I64X2; let expr1_0 = SseOpcode::Pmuludq; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1544,7 +1588,7 @@ pub fn constructor_pmuludq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_punpckhwd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1044. + // Rule at src/isa/x64/inst.isle line 1053. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Punpckhwd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1555,7 +1599,7 @@ pub fn constructor_punpckhwd(ctx: &mut C, arg0: Reg, arg1: &RegMem) pub fn constructor_punpcklwd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1049. + // Rule at src/isa/x64/inst.isle line 1058. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Punpcklwd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1566,7 +1610,7 @@ pub fn constructor_punpcklwd(ctx: &mut C, arg0: Reg, arg1: &RegMem) pub fn constructor_andnps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1054. + // Rule at src/isa/x64/inst.isle line 1063. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Andnps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1577,7 +1621,7 @@ pub fn constructor_andnps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_andnpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1059. + // Rule at src/isa/x64/inst.isle line 1068. let expr0_0: Type = F64X2; let expr1_0 = SseOpcode::Andnpd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1588,7 +1632,7 @@ pub fn constructor_andnpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pandn(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1064. + // Rule at src/isa/x64/inst.isle line 1073. let expr0_0: Type = F64X2; let expr1_0 = SseOpcode::Pandn; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1599,17 +1643,17 @@ pub fn constructor_pandn(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_sse_blend_op(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32X4 { - // Rule at src/isa/x64/inst.isle line 1068. + // Rule at src/isa/x64/inst.isle line 1077. let expr0_0 = SseOpcode::Blendvps; return Some(expr0_0); } if pattern0_0 == F64X2 { - // Rule at src/isa/x64/inst.isle line 1069. + // Rule at src/isa/x64/inst.isle line 1078. let expr0_0 = SseOpcode::Blendvpd; return Some(expr0_0); } if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { - // Rule at src/isa/x64/inst.isle line 1070. + // Rule at src/isa/x64/inst.isle line 1079. let expr0_0 = SseOpcode::Pblendvb; return Some(expr0_0); } @@ -1620,17 +1664,17 @@ pub fn constructor_sse_blend_op(ctx: &mut C, arg0: Type) -> Option(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32X4 { - // Rule at src/isa/x64/inst.isle line 1073. + // Rule at src/isa/x64/inst.isle line 1082. let expr0_0 = SseOpcode::Movaps; return Some(expr0_0); } if pattern0_0 == F64X2 { - // Rule at src/isa/x64/inst.isle line 1074. + // Rule at src/isa/x64/inst.isle line 1083. let expr0_0 = SseOpcode::Movapd; return Some(expr0_0); } if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { - // Rule at src/isa/x64/inst.isle line 1075. + // Rule at src/isa/x64/inst.isle line 1084. let expr0_0 = SseOpcode::Movdqa; return Some(expr0_0); } @@ -1649,7 +1693,7 @@ pub fn constructor_sse_blend( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1079. + // Rule at src/isa/x64/inst.isle line 1088. let expr0_0 = C::xmm0(ctx); let expr1_0 = constructor_sse_mov_op(ctx, pattern0_0)?; let expr2_0 = MInst::XmmUnaryRmR { @@ -1673,7 +1717,7 @@ pub fn constructor_blendvpd( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1091. + // Rule at src/isa/x64/inst.isle line 1100. let expr0_0 = C::xmm0(ctx); let expr1_0 = SseOpcode::Movapd; let expr2_0 = RegMem::Reg { reg: pattern2_0 }; @@ -1693,7 +1737,7 @@ pub fn constructor_blendvpd( pub fn constructor_movsd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1103. + // Rule at src/isa/x64/inst.isle line 1112. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Movsd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1704,7 +1748,7 @@ pub fn constructor_movsd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_movlhps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1108. + // Rule at src/isa/x64/inst.isle line 1117. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Movlhps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1715,7 +1759,7 @@ pub fn constructor_movlhps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmaxsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1113. + // Rule at src/isa/x64/inst.isle line 1122. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxsb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1726,7 +1770,7 @@ pub fn constructor_pmaxsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmaxsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1118. + // Rule at src/isa/x64/inst.isle line 1127. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxsw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1737,7 +1781,7 @@ pub fn constructor_pmaxsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmaxsd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1123. + // Rule at src/isa/x64/inst.isle line 1132. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxsd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1748,7 +1792,7 @@ pub fn constructor_pmaxsd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pminsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1128. + // Rule at src/isa/x64/inst.isle line 1137. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pminsb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1759,7 +1803,7 @@ pub fn constructor_pminsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pminsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1133. + // Rule at src/isa/x64/inst.isle line 1142. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pminsw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1770,7 +1814,7 @@ pub fn constructor_pminsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pminsd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1138. + // Rule at src/isa/x64/inst.isle line 1147. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pminsd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1781,7 +1825,7 @@ pub fn constructor_pminsd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmaxub(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1143. + // Rule at src/isa/x64/inst.isle line 1152. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxub; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1792,7 +1836,7 @@ pub fn constructor_pmaxub(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmaxuw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1148. + // Rule at src/isa/x64/inst.isle line 1157. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxuw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1803,7 +1847,7 @@ pub fn constructor_pmaxuw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmaxud(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1153. + // Rule at src/isa/x64/inst.isle line 1162. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxud; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1814,7 +1858,7 @@ pub fn constructor_pmaxud(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pminub(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1158. + // Rule at src/isa/x64/inst.isle line 1167. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pminub; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1825,7 +1869,7 @@ pub fn constructor_pminub(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pminuw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1163. + // Rule at src/isa/x64/inst.isle line 1172. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pminuw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1836,7 +1880,7 @@ pub fn constructor_pminuw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pminud(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1168. + // Rule at src/isa/x64/inst.isle line 1177. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pminud; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1847,7 +1891,7 @@ pub fn constructor_pminud(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_punpcklbw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1173. + // Rule at src/isa/x64/inst.isle line 1182. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Punpcklbw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1858,7 +1902,7 @@ pub fn constructor_punpcklbw(ctx: &mut C, arg0: Reg, arg1: &RegMem) pub fn constructor_punpckhbw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1178. + // Rule at src/isa/x64/inst.isle line 1187. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Punpckhbw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1869,7 +1913,7 @@ pub fn constructor_punpckhbw(ctx: &mut C, arg0: Reg, arg1: &RegMem) pub fn constructor_packsswb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1183. + // Rule at src/isa/x64/inst.isle line 1192. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Packsswb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1890,7 +1934,7 @@ pub fn constructor_xmm_rm_r_imm( let pattern2_0 = arg2; let pattern3_0 = arg3; let pattern4_0 = arg4; - // Rule at src/isa/x64/inst.isle line 1188. + // Rule at src/isa/x64/inst.isle line 1197. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::XmmRmRImm { @@ -1918,7 +1962,7 @@ pub fn constructor_palignr( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1200. + // Rule at src/isa/x64/inst.isle line 1209. let expr0_0 = SseOpcode::Palignr; let expr1_0 = constructor_xmm_rm_r_imm( ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0, pattern3_0, @@ -1936,7 +1980,7 @@ pub fn constructor_pshufd( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1209. + // Rule at src/isa/x64/inst.isle line 1218. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = C::writable_reg_to_reg(ctx, expr1_0); @@ -1961,7 +2005,7 @@ pub fn constructor_xmm_unary_rm_r( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1222. + // Rule at src/isa/x64/inst.isle line 1231. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::XmmUnaryRmR { @@ -1977,7 +2021,7 @@ pub fn constructor_xmm_unary_rm_r( // Generated as internal constructor for term pmovsxbw. pub fn constructor_pmovsxbw(ctx: &mut C, arg0: &RegMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1229. + // Rule at src/isa/x64/inst.isle line 1238. let expr0_0 = SseOpcode::Pmovsxbw; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -1986,7 +2030,7 @@ pub fn constructor_pmovsxbw(ctx: &mut C, arg0: &RegMem) -> Option(ctx: &mut C, arg0: &RegMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1234. + // Rule at src/isa/x64/inst.isle line 1243. let expr0_0 = SseOpcode::Pmovzxbw; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -1995,7 +2039,7 @@ pub fn constructor_pmovzxbw(ctx: &mut C, arg0: &RegMem) -> Option(ctx: &mut C, arg0: &RegMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1239. + // Rule at src/isa/x64/inst.isle line 1248. let expr0_0 = SseOpcode::Pabsb; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -2004,7 +2048,7 @@ pub fn constructor_pabsb(ctx: &mut C, arg0: &RegMem) -> Option // Generated as internal constructor for term pabsw. pub fn constructor_pabsw(ctx: &mut C, arg0: &RegMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1244. + // Rule at src/isa/x64/inst.isle line 1253. let expr0_0 = SseOpcode::Pabsw; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -2013,7 +2057,7 @@ pub fn constructor_pabsw(ctx: &mut C, arg0: &RegMem) -> Option // Generated as internal constructor for term pabsd. pub fn constructor_pabsd(ctx: &mut C, arg0: &RegMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1249. + // Rule at src/isa/x64/inst.isle line 1258. let expr0_0 = SseOpcode::Pabsd; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -2027,7 +2071,7 @@ pub fn constructor_xmm_unary_rm_r_evex( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1254. + // Rule at src/isa/x64/inst.isle line 1263. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::XmmUnaryRmREvex { @@ -2043,7 +2087,7 @@ pub fn constructor_xmm_unary_rm_r_evex( // Generated as internal constructor for term vpabsq. pub fn constructor_vpabsq(ctx: &mut C, arg0: &RegMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1261. + // Rule at src/isa/x64/inst.isle line 1270. let expr0_0 = Avx512Opcode::Vpabsq; let expr1_0 = constructor_xmm_unary_rm_r_evex(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -2059,7 +2103,7 @@ pub fn constructor_xmm_rm_r_evex( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1266. + // Rule at src/isa/x64/inst.isle line 1275. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::XmmRmREvex { @@ -2077,7 +2121,7 @@ pub fn constructor_xmm_rm_r_evex( pub fn constructor_vpmullq(ctx: &mut C, arg0: &RegMem, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1278. + // Rule at src/isa/x64/inst.isle line 1287. let expr0_0 = Avx512Opcode::Vpmullq; let expr1_0 = constructor_xmm_rm_r_evex(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2093,7 +2137,7 @@ pub fn constructor_xmm_rmi_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1285. + // Rule at src/isa/x64/inst.isle line 1294. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::XmmRmiReg { @@ -2111,7 +2155,7 @@ pub fn constructor_xmm_rmi_reg( pub fn constructor_psllw(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1295. + // Rule at src/isa/x64/inst.isle line 1304. let expr0_0 = SseOpcode::Psllw; let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2121,7 +2165,7 @@ pub fn constructor_psllw(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) - pub fn constructor_pslld(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1300. + // Rule at src/isa/x64/inst.isle line 1309. let expr0_0 = SseOpcode::Pslld; let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2131,7 +2175,7 @@ pub fn constructor_pslld(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) - pub fn constructor_psllq(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1305. + // Rule at src/isa/x64/inst.isle line 1314. let expr0_0 = SseOpcode::Psllq; let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2141,7 +2185,7 @@ pub fn constructor_psllq(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) - pub fn constructor_psrlw(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1310. + // Rule at src/isa/x64/inst.isle line 1319. let expr0_0 = SseOpcode::Psrlw; let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2151,7 +2195,7 @@ pub fn constructor_psrlw(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) - pub fn constructor_psrld(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1315. + // Rule at src/isa/x64/inst.isle line 1324. let expr0_0 = SseOpcode::Psrld; let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2161,7 +2205,7 @@ pub fn constructor_psrld(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) - pub fn constructor_psrlq(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1320. + // Rule at src/isa/x64/inst.isle line 1329. let expr0_0 = SseOpcode::Psrlq; let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2171,7 +2215,7 @@ pub fn constructor_psrlq(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) - pub fn constructor_psraw(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1325. + // Rule at src/isa/x64/inst.isle line 1334. let expr0_0 = SseOpcode::Psraw; let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2181,7 +2225,7 @@ pub fn constructor_psraw(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) - pub fn constructor_psrad(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1330. + // Rule at src/isa/x64/inst.isle line 1339. let expr0_0 = SseOpcode::Psrad; let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2199,7 +2243,7 @@ pub fn constructor_mul_hi( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1337. + // Rule at src/isa/x64/inst.isle line 1346. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::temp_writable_reg(ctx, pattern0_0); let expr2_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); @@ -2228,7 +2272,7 @@ pub fn constructor_mulhi_u( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1353. + // Rule at src/isa/x64/inst.isle line 1362. let expr0_0: bool = false; let expr1_0 = constructor_mul_hi(ctx, pattern0_0, expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2244,7 +2288,7 @@ pub fn constructor_cmpps( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1358. + // Rule at src/isa/x64/inst.isle line 1367. let expr0_0 = SseOpcode::Cmpps; let expr1_0 = C::encode_fcmp_imm(ctx, pattern2_0); let expr2_0 = OperandSize::Size32; @@ -2263,7 +2307,7 @@ pub fn constructor_cmppd( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1371. + // Rule at src/isa/x64/inst.isle line 1380. let expr0_0 = SseOpcode::Cmppd; let expr1_0 = C::encode_fcmp_imm(ctx, pattern2_0); let expr2_0 = OperandSize::Size32; @@ -2284,7 +2328,7 @@ pub fn constructor_gpr_to_xmm( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1380. + // Rule at src/isa/x64/inst.isle line 1389. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::GprToXmm { op: pattern1_0.clone(), @@ -2307,7 +2351,7 @@ pub fn constructor_pinsrb( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1387. + // Rule at src/isa/x64/inst.isle line 1396. let expr0_0 = SseOpcode::Pinsrb; let expr1_0 = OperandSize::Size32; let expr2_0 = @@ -2325,7 +2369,7 @@ pub fn constructor_pinsrw( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1392. + // Rule at src/isa/x64/inst.isle line 1401. let expr0_0 = SseOpcode::Pinsrw; let expr1_0 = OperandSize::Size32; let expr2_0 = @@ -2345,7 +2389,7 @@ pub fn constructor_pinsrd( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1397. + // Rule at src/isa/x64/inst.isle line 1406. let expr0_0 = SseOpcode::Pinsrd; let expr1_0 = constructor_xmm_rm_r_imm( ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0, pattern3_0, @@ -2363,7 +2407,7 @@ pub fn constructor_insertps( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1402. + // Rule at src/isa/x64/inst.isle line 1411. let expr0_0 = SseOpcode::Insertps; let expr1_0 = OperandSize::Size32; let expr2_0 = @@ -2376,7 +2420,7 @@ pub fn constructor_pextrd(ctx: &mut C, arg0: Type, arg1: Reg, arg2: let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1407. + // Rule at src/isa/x64/inst.isle line 1416. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); let expr2_0 = SseOpcode::Pextrd; @@ -2399,7 +2443,7 @@ pub fn constructor_pextrd(ctx: &mut C, arg0: Type, arg1: Reg, arg2: pub fn constructor_not(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1420. + // Rule at src/isa/x64/inst.isle line 1429. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = MInst::Not { @@ -2412,10 +2456,27 @@ pub fn constructor_not(ctx: &mut C, arg0: Type, arg1: Reg) -> Option return Some(expr4_0); } +// Generated as internal constructor for term neg. +pub fn constructor_neg(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 1437. + let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); + let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); + let expr2_0 = MInst::Neg { + size: expr1_0, + src: pattern1_0, + dst: expr0_0, + }; + let expr3_0 = C::emit(ctx, &expr2_0); + let expr4_0 = C::writable_reg_to_reg(ctx, expr0_0); + return Some(expr4_0); +} + // Generated as internal constructor for term lea. pub fn constructor_lea(ctx: &mut C, arg0: &SyntheticAmode) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1427. + // Rule at src/isa/x64/inst.isle line 1444. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::LoadEffectiveAddress { @@ -2427,6 +2488,17 @@ pub fn constructor_lea(ctx: &mut C, arg0: &SyntheticAmode) -> Option return Some(expr4_0); } +// Generated as internal constructor for term ud2. +pub fn constructor_ud2(ctx: &mut C, arg0: &TrapCode) -> Option { + let pattern0_0 = arg0; + // Rule at src/isa/x64/inst.isle line 1451. + let expr0_0 = MInst::Ud2 { + trap_code: pattern0_0.clone(), + }; + let expr1_0 = SideEffectNoResult::Inst { inst: expr0_0 }; + return Some(expr1_0); +} + // Generated as internal constructor for term lower. pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let pattern0_0 = arg0; @@ -2458,6 +2530,26 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { + match &pattern2_0 { + &Opcode::Trap => { + // Rule at src/isa/x64/lower.isle line 1438. + let expr0_0 = constructor_ud2(ctx, &pattern2_1)?; + let expr1_0 = constructor_value_regs_none(ctx, &expr0_0)?; + return Some(expr1_0); + } + &Opcode::ResumableTrap => { + // Rule at src/isa/x64/lower.isle line 1443. + let expr0_0 = constructor_ud2(ctx, &pattern2_1)?; + let expr1_0 = constructor_value_regs_none(ctx, &expr0_0)?; + return Some(expr1_0); + } + _ => {} + } + } &InstructionData::TernaryImm8 { opcode: ref pattern2_0, args: ref pattern2_1, @@ -2467,7 +2559,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::Bnot = &pattern5_0 { - // Rule at src/isa/x64/lower.isle line 1239. + // Rule at src/isa/x64/lower.isle line 1304. let expr0_0 = constructor_i128_not(ctx, pattern5_1)?; return Some(expr0_0); } @@ -2656,7 +2748,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 927. + // Rule at src/isa/x64/lower.isle line 992. let expr0_0 = C::put_in_regs(ctx, pattern7_0); let expr1_0: usize = 0; let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); @@ -2705,10 +2797,10 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { + let (pattern7_0, pattern7_1) = + C::unpack_value_array_2(ctx, &pattern5_1); + // Rule at src/isa/x64/lower.isle line 900. + let expr0_0 = C::put_in_regs(ctx, pattern7_0); + let expr1_0 = constructor_lo_reg(ctx, pattern7_1)?; + let expr2_0 = constructor_shr_i128(ctx, expr0_0, expr1_0)?; + let expr3_0: Type = I64; + let expr4_0: Type = I64; + let expr5_0: u64 = 128; + let expr6_0 = constructor_imm(ctx, expr4_0, expr5_0)?; + let expr7_0 = RegMemImm::Reg { reg: expr1_0 }; + let expr8_0 = constructor_sub(ctx, expr3_0, expr6_0, &expr7_0)?; + let expr9_0 = constructor_shl_i128(ctx, expr0_0, expr8_0)?; + let expr10_0 = constructor_or_i128(ctx, expr2_0, expr9_0)?; + return Some(expr10_0); + } &Opcode::Ishl => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); @@ -2796,7 +2905,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::Bnot = &pattern5_0 { - // Rule at src/isa/x64/lower.isle line 1236. + // Rule at src/isa/x64/lower.isle line 1301. let expr0_0 = constructor_i128_not(ctx, pattern5_1)?; return Some(expr0_0); } @@ -2841,7 +2950,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1340. + // Rule at src/isa/x64/lower.isle line 1405. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pminsb(ctx, expr0_0, &expr1_0)?; @@ -2851,7 +2960,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1362. + // Rule at src/isa/x64/lower.isle line 1427. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pminub(ctx, expr0_0, &expr1_0)?; @@ -2861,7 +2970,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1329. + // Rule at src/isa/x64/lower.isle line 1394. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pmaxsb(ctx, expr0_0, &expr1_0)?; @@ -2871,7 +2980,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1351. + // Rule at src/isa/x64/lower.isle line 1416. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pmaxub(ctx, expr0_0, &expr1_0)?; @@ -2941,12 +3050,25 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - if let &Opcode::Iabs = &pattern5_0 { - // Rule at src/isa/x64/lower.isle line 1182. - let expr0_0 = C::put_in_reg_mem(ctx, pattern5_1); - let expr1_0 = constructor_pabsb(ctx, &expr0_0)?; - let expr2_0 = C::value_reg(ctx, expr1_0); - return Some(expr2_0); + match &pattern5_0 { + &Opcode::Ineg => { + // Rule at src/isa/x64/lower.isle line 917. + let expr0_0: Type = I8X16; + let expr1_0: u64 = 0; + let expr2_0 = constructor_imm(ctx, expr0_0, expr1_0)?; + let expr3_0 = C::put_in_reg_mem(ctx, pattern5_1); + let expr4_0 = constructor_psubb(ctx, expr2_0, &expr3_0)?; + let expr5_0 = C::value_reg(ctx, expr4_0); + return Some(expr5_0); + } + &Opcode::Iabs => { + // Rule at src/isa/x64/lower.isle line 1247. + let expr0_0 = C::put_in_reg_mem(ctx, pattern5_1); + let expr1_0 = constructor_pabsb(ctx, &expr0_0)?; + let expr2_0 = C::value_reg(ctx, expr1_0); + return Some(expr2_0); + } + _ => {} } } _ => {} @@ -2963,7 +3085,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1343. + // Rule at src/isa/x64/lower.isle line 1408. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pminsw(ctx, expr0_0, &expr1_0)?; @@ -2973,7 +3095,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1365. + // Rule at src/isa/x64/lower.isle line 1430. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pminuw(ctx, expr0_0, &expr1_0)?; @@ -2983,7 +3105,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1332. + // Rule at src/isa/x64/lower.isle line 1397. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pmaxsw(ctx, expr0_0, &expr1_0)?; @@ -2993,7 +3115,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1354. + // Rule at src/isa/x64/lower.isle line 1419. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pmaxuw(ctx, expr0_0, &expr1_0)?; @@ -3040,12 +3162,25 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - if let &Opcode::Iabs = &pattern5_0 { - // Rule at src/isa/x64/lower.isle line 1185. - let expr0_0 = C::put_in_reg_mem(ctx, pattern5_1); - let expr1_0 = constructor_pabsw(ctx, &expr0_0)?; - let expr2_0 = C::value_reg(ctx, expr1_0); - return Some(expr2_0); + match &pattern5_0 { + &Opcode::Ineg => { + // Rule at src/isa/x64/lower.isle line 921. + let expr0_0: Type = I16X8; + let expr1_0: u64 = 0; + let expr2_0 = constructor_imm(ctx, expr0_0, expr1_0)?; + let expr3_0 = C::put_in_reg_mem(ctx, pattern5_1); + let expr4_0 = constructor_psubw(ctx, expr2_0, &expr3_0)?; + let expr5_0 = C::value_reg(ctx, expr4_0); + return Some(expr5_0); + } + &Opcode::Iabs => { + // Rule at src/isa/x64/lower.isle line 1250. + let expr0_0 = C::put_in_reg_mem(ctx, pattern5_1); + let expr1_0 = constructor_pabsw(ctx, &expr0_0)?; + let expr2_0 = C::value_reg(ctx, expr1_0); + return Some(expr2_0); + } + _ => {} } } _ => {} @@ -3062,7 +3197,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1346. + // Rule at src/isa/x64/lower.isle line 1411. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pminsd(ctx, expr0_0, &expr1_0)?; @@ -3072,7 +3207,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1368. + // Rule at src/isa/x64/lower.isle line 1433. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pminud(ctx, expr0_0, &expr1_0)?; @@ -3082,7 +3217,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1335. + // Rule at src/isa/x64/lower.isle line 1400. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pmaxsd(ctx, expr0_0, &expr1_0)?; @@ -3092,7 +3227,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1357. + // Rule at src/isa/x64/lower.isle line 1422. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pmaxud(ctx, expr0_0, &expr1_0)?; @@ -3139,12 +3274,25 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - if let &Opcode::Iabs = &pattern5_0 { - // Rule at src/isa/x64/lower.isle line 1188. - let expr0_0 = C::put_in_reg_mem(ctx, pattern5_1); - let expr1_0 = constructor_pabsd(ctx, &expr0_0)?; - let expr2_0 = C::value_reg(ctx, expr1_0); - return Some(expr2_0); + match &pattern5_0 { + &Opcode::Ineg => { + // Rule at src/isa/x64/lower.isle line 925. + let expr0_0: Type = I32X4; + let expr1_0: u64 = 0; + let expr2_0 = constructor_imm(ctx, expr0_0, expr1_0)?; + let expr3_0 = C::put_in_reg_mem(ctx, pattern5_1); + let expr4_0 = constructor_psubd(ctx, expr2_0, &expr3_0)?; + let expr5_0 = C::value_reg(ctx, expr4_0); + return Some(expr5_0); + } + &Opcode::Iabs => { + // Rule at src/isa/x64/lower.isle line 1253. + let expr0_0 = C::put_in_reg_mem(ctx, pattern5_1); + let expr1_0 = constructor_pabsd(ctx, &expr0_0)?; + let expr2_0 = C::value_reg(ctx, expr1_0); + return Some(expr2_0); + } + _ => {} } } _ => {} @@ -3211,18 +3359,31 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - if let &Opcode::Iabs = &pattern5_0 { - // Rule at src/isa/x64/lower.isle line 1202. - let expr0_0 = C::put_in_reg(ctx, pattern5_1); - let expr1_0: Type = I64X2; - let expr2_0: u64 = 0; - let expr3_0 = constructor_imm(ctx, expr1_0, expr2_0)?; - let expr4_0 = RegMem::Reg { reg: expr0_0 }; - let expr5_0 = constructor_psubq(ctx, expr3_0, &expr4_0)?; - let expr6_0 = RegMem::Reg { reg: expr0_0 }; - let expr7_0 = constructor_blendvpd(ctx, expr5_0, &expr6_0, expr5_0)?; - let expr8_0 = C::value_reg(ctx, expr7_0); - return Some(expr8_0); + match &pattern5_0 { + &Opcode::Ineg => { + // Rule at src/isa/x64/lower.isle line 929. + let expr0_0: Type = I64X2; + let expr1_0: u64 = 0; + let expr2_0 = constructor_imm(ctx, expr0_0, expr1_0)?; + let expr3_0 = C::put_in_reg_mem(ctx, pattern5_1); + let expr4_0 = constructor_psubq(ctx, expr2_0, &expr3_0)?; + let expr5_0 = C::value_reg(ctx, expr4_0); + return Some(expr5_0); + } + &Opcode::Iabs => { + // Rule at src/isa/x64/lower.isle line 1267. + let expr0_0 = C::put_in_reg(ctx, pattern5_1); + let expr1_0: Type = I64X2; + let expr2_0: u64 = 0; + let expr3_0 = constructor_imm(ctx, expr1_0, expr2_0)?; + let expr4_0 = RegMem::Reg { reg: expr0_0 }; + let expr5_0 = constructor_psubq(ctx, expr3_0, &expr4_0)?; + let expr6_0 = RegMem::Reg { reg: expr0_0 }; + let expr7_0 = constructor_blendvpd(ctx, expr5_0, &expr6_0, expr5_0)?; + let expr8_0 = C::value_reg(ctx, expr7_0); + return Some(expr8_0); + } + _ => {} } } _ => {} @@ -3236,7 +3397,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::BandNot = &pattern4_0 { let (pattern6_0, pattern6_1) = C::unpack_value_array_2(ctx, &pattern4_1); - // Rule at src/isa/x64/lower.isle line 1175. + // Rule at src/isa/x64/lower.isle line 1240. let expr0_0 = C::put_in_reg(ctx, pattern6_1); let expr1_0 = C::put_in_reg_mem(ctx, pattern6_0); let expr2_0 = constructor_sse_and_not(ctx, pattern2_0, expr0_0, &expr1_0)?; @@ -3315,7 +3476,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { let (pattern9_0, pattern9_1) = C::unpack_value_array_2(ctx, &pattern7_1); - // Rule at src/isa/x64/lower.isle line 870. + // Rule at src/isa/x64/lower.isle line 935. let expr0_0 = C::put_in_reg(ctx, pattern9_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); let expr2_0 = constructor_pavgb(ctx, expr0_0, &expr1_0)?; @@ -3443,7 +3604,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern9_0, pattern9_1) = C::unpack_value_array_2(ctx, &pattern7_1); - // Rule at src/isa/x64/lower.isle line 874. + // Rule at src/isa/x64/lower.isle line 939. let expr0_0 = C::put_in_reg(ctx, pattern9_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); let expr2_0 = constructor_pavgw(ctx, expr0_0, &expr1_0)?; @@ -3556,7 +3717,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1, pattern7_2) = C::unpack_value_array_3(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1249. + // Rule at src/isa/x64/lower.isle line 1314. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg(ctx, pattern7_1); let expr2_0 = RegMem::Reg { reg: expr0_0 }; @@ -4503,7 +4664,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1, pattern7_2) = C::unpack_value_array_3(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1263. + // Rule at src/isa/x64/lower.isle line 1328. let expr0_0 = C::put_in_reg_mem(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = C::put_in_reg(ctx, pattern7_2); @@ -4521,7 +4682,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::Bnot = &pattern5_0 { - // Rule at src/isa/x64/lower.isle line 1244. + // Rule at src/isa/x64/lower.isle line 1309. let expr0_0 = C::put_in_reg(ctx, pattern5_1); let expr1_0 = constructor_vector_all_ones(ctx, pattern2_0)?; let expr2_0 = RegMem::Reg { reg: expr1_0 }; @@ -4648,7 +4809,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { - if let &Opcode::Bnot = &pattern5_0 { - // Rule at src/isa/x64/lower.isle line 1223. - let expr0_0 = C::put_in_reg(ctx, pattern5_1); - let expr1_0 = constructor_not(ctx, pattern3_0, expr0_0)?; - let expr2_0 = C::value_reg(ctx, expr1_0); - return Some(expr2_0); + match &pattern5_0 { + &Opcode::Ineg => { + // Rule at src/isa/x64/lower.isle line 912. + let expr0_0 = C::put_in_reg(ctx, pattern5_1); + let expr1_0 = constructor_neg(ctx, pattern3_0, expr0_0)?; + let expr2_0 = C::value_reg(ctx, expr1_0); + return Some(expr2_0); + } + &Opcode::Bnot => { + // Rule at src/isa/x64/lower.isle line 1288. + let expr0_0 = C::put_in_reg(ctx, pattern5_1); + let expr1_0 = constructor_not(ctx, pattern3_0, expr0_0)?; + let expr2_0 = C::value_reg(ctx, expr1_0); + return Some(expr2_0); + } + _ => {} } } &InstructionData::BinaryImm64 { @@ -4951,35 +5122,68 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { + let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); + if let Some(pattern8_0) = C::def_inst(ctx, pattern7_1) { + let pattern9_0 = C::inst_data(ctx, pattern8_0); + if let &InstructionData::UnaryImm { + opcode: ref pattern10_0, + imm: pattern10_1, + } = &pattern9_0 + { + if let &Opcode::Iconst = &pattern10_0 { + let pattern12_0 = C::u64_from_imm64(ctx, pattern10_1); + // Rule at src/isa/x64/lower.isle line 852. + let expr0_0 = C::put_in_reg(ctx, pattern7_0); + let expr1_0 = + C::const_to_type_masked_imm8(ctx, pattern12_0, pattern3_0); + let expr2_0 = + constructor_x64_rotl(ctx, pattern3_0, expr0_0, &expr1_0)?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } + } + } + // Rule at src/isa/x64/lower.isle line 846. + let expr0_0 = constructor_lo_reg(ctx, pattern7_1)?; + let expr1_0 = C::put_in_reg(ctx, pattern7_0); + let expr2_0 = Imm8Reg::Reg { reg: expr0_0 }; + let expr3_0 = constructor_x64_rotl(ctx, pattern3_0, expr1_0, &expr2_0)?; + let expr4_0 = C::value_reg(ctx, expr3_0); + return Some(expr4_0); + } + &Opcode::Rotr => { + let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); + if let Some(pattern8_0) = C::def_inst(ctx, pattern7_1) { + let pattern9_0 = C::inst_data(ctx, pattern8_0); + if let &InstructionData::UnaryImm { + opcode: ref pattern10_0, + imm: pattern10_1, + } = &pattern9_0 + { + if let &Opcode::Iconst = &pattern10_0 { + let pattern12_0 = C::u64_from_imm64(ctx, pattern10_1); + // Rule at src/isa/x64/lower.isle line 892. + let expr0_0 = C::put_in_reg(ctx, pattern7_0); + let expr1_0 = + C::const_to_type_masked_imm8(ctx, pattern12_0, pattern3_0); + let expr2_0 = + constructor_x64_rotr(ctx, pattern3_0, expr0_0, &expr1_0)?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } } } + // Rule at src/isa/x64/lower.isle line 886. + let expr0_0 = constructor_lo_reg(ctx, pattern7_1)?; + let expr1_0 = C::put_in_reg(ctx, pattern7_0); + let expr2_0 = Imm8Reg::Reg { reg: expr0_0 }; + let expr3_0 = constructor_x64_rotr(ctx, pattern3_0, expr1_0, &expr2_0)?; + let expr4_0 = C::value_reg(ctx, expr3_0); + return Some(expr4_0); } - // Rule at src/isa/x64/lower.isle line 846. - let expr0_0 = constructor_lo_reg(ctx, pattern7_1)?; - let expr1_0 = C::put_in_reg(ctx, pattern7_0); - let expr2_0 = Imm8Reg::Reg { reg: expr0_0 }; - let expr3_0 = constructor_m_rotl(ctx, pattern3_0, expr1_0, &expr2_0)?; - let expr4_0 = C::value_reg(ctx, expr3_0); - return Some(expr4_0); + _ => {} } } } @@ -4990,37 +5194,74 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { + let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); + if let Some(pattern8_0) = C::def_inst(ctx, pattern7_1) { + let pattern9_0 = C::inst_data(ctx, pattern8_0); + if let &InstructionData::UnaryImm { + opcode: ref pattern10_0, + imm: pattern10_1, + } = &pattern9_0 + { + if let &Opcode::Iconst = &pattern10_0 { + let pattern12_0 = C::u64_from_imm64(ctx, pattern10_1); + // Rule at src/isa/x64/lower.isle line 837. + let expr0_0 = C::put_in_reg(ctx, pattern7_0); + let expr1_0 = + C::const_to_type_masked_imm8(ctx, pattern12_0, pattern3_0); + let expr2_0 = + constructor_x64_rotl(ctx, pattern3_0, expr0_0, &expr1_0)?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } + } + } + // Rule at src/isa/x64/lower.isle line 833. + let expr0_0: Type = I32; + let expr1_0 = ExtendKind::Zero; + let expr2_0 = + constructor_extend_to_reg(ctx, pattern7_1, expr0_0, &expr1_0)?; + let expr3_0 = C::put_in_reg(ctx, pattern7_0); + let expr4_0 = Imm8Reg::Reg { reg: expr2_0 }; + let expr5_0 = constructor_x64_rotl(ctx, pattern3_0, expr3_0, &expr4_0)?; + let expr6_0 = C::value_reg(ctx, expr5_0); + return Some(expr6_0); + } + &Opcode::Rotr => { + let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); + if let Some(pattern8_0) = C::def_inst(ctx, pattern7_1) { + let pattern9_0 = C::inst_data(ctx, pattern8_0); + if let &InstructionData::UnaryImm { + opcode: ref pattern10_0, + imm: pattern10_1, + } = &pattern9_0 + { + if let &Opcode::Iconst = &pattern10_0 { + let pattern12_0 = C::u64_from_imm64(ctx, pattern10_1); + // Rule at src/isa/x64/lower.isle line 877. + let expr0_0 = C::put_in_reg(ctx, pattern7_0); + let expr1_0 = + C::const_to_type_masked_imm8(ctx, pattern12_0, pattern3_0); + let expr2_0 = + constructor_x64_rotr(ctx, pattern3_0, expr0_0, &expr1_0)?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } } } + // Rule at src/isa/x64/lower.isle line 873. + let expr0_0: Type = I32; + let expr1_0 = ExtendKind::Zero; + let expr2_0 = + constructor_extend_to_reg(ctx, pattern7_1, expr0_0, &expr1_0)?; + let expr3_0 = C::put_in_reg(ctx, pattern7_0); + let expr4_0 = Imm8Reg::Reg { reg: expr2_0 }; + let expr5_0 = constructor_x64_rotr(ctx, pattern3_0, expr3_0, &expr4_0)?; + let expr6_0 = C::value_reg(ctx, expr5_0); + return Some(expr6_0); } - // Rule at src/isa/x64/lower.isle line 833. - let expr0_0: Type = I32; - let expr1_0 = ExtendKind::Zero; - let expr2_0 = constructor_extend_to_reg(ctx, pattern7_1, expr0_0, &expr1_0)?; - let expr3_0 = C::put_in_reg(ctx, pattern7_0); - let expr4_0 = Imm8Reg::Reg { reg: expr2_0 }; - let expr5_0 = constructor_m_rotl(ctx, pattern3_0, expr3_0, &expr4_0)?; - let expr6_0 = C::value_reg(ctx, expr5_0); - return Some(expr6_0); + _ => {} } } } @@ -5439,21 +5680,21 @@ pub fn constructor_sse_and_not( if pattern0_0 == F32X4 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/lower.isle line 1164. + // Rule at src/isa/x64/lower.isle line 1229. let expr0_0 = constructor_andnps(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if pattern0_0 == F64X2 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/lower.isle line 1165. + // Rule at src/isa/x64/lower.isle line 1230. let expr0_0 = constructor_andnpd(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/lower.isle line 1166. + // Rule at src/isa/x64/lower.isle line 1231. let expr0_0 = constructor_pandn(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } @@ -5463,7 +5704,7 @@ pub fn constructor_sse_and_not( // Generated as internal constructor for term i128_not. pub fn constructor_i128_not(ctx: &mut C, arg0: Value) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/lower.isle line 1229. + // Rule at src/isa/x64/lower.isle line 1294. let expr0_0 = C::put_in_regs(ctx, pattern0_0); let expr1_0: usize = 0; let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); @@ -5490,7 +5731,7 @@ pub fn constructor_vec_insert_lane( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/lower.isle line 1283. + // Rule at src/isa/x64/lower.isle line 1348. let expr0_0 = constructor_pinsrb(ctx, pattern2_0, pattern3_0, pattern4_0)?; return Some(expr0_0); } @@ -5498,7 +5739,7 @@ pub fn constructor_vec_insert_lane( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/lower.isle line 1286. + // Rule at src/isa/x64/lower.isle line 1351. let expr0_0 = constructor_pinsrw(ctx, pattern2_0, pattern3_0, pattern4_0)?; return Some(expr0_0); } @@ -5506,7 +5747,7 @@ pub fn constructor_vec_insert_lane( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/lower.isle line 1289. + // Rule at src/isa/x64/lower.isle line 1354. let expr0_0 = OperandSize::Size32; let expr1_0 = constructor_pinsrd(ctx, pattern2_0, pattern3_0, pattern4_0, &expr0_0)?; return Some(expr1_0); @@ -5515,7 +5756,7 @@ pub fn constructor_vec_insert_lane( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/lower.isle line 1292. + // Rule at src/isa/x64/lower.isle line 1357. let expr0_0 = OperandSize::Size64; let expr1_0 = constructor_pinsrd(ctx, pattern2_0, pattern3_0, pattern4_0, &expr0_0)?; return Some(expr1_0); @@ -5524,7 +5765,7 @@ pub fn constructor_vec_insert_lane( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/lower.isle line 1295. + // Rule at src/isa/x64/lower.isle line 1360. let expr0_0 = C::sse_insertps_lane_imm(ctx, pattern4_0); let expr1_0 = constructor_insertps(ctx, pattern2_0, pattern3_0, expr0_0)?; return Some(expr1_0); @@ -5535,7 +5776,7 @@ pub fn constructor_vec_insert_lane( if let &RegMem::Reg { reg: pattern4_0 } = pattern3_0 { let pattern5_0 = arg3; if pattern5_0 == 0 { - // Rule at src/isa/x64/lower.isle line 1316. + // Rule at src/isa/x64/lower.isle line 1381. let expr0_0 = RegMem::Reg { reg: pattern4_0 }; let expr1_0 = constructor_movsd(ctx, pattern2_0, &expr0_0)?; return Some(expr1_0); @@ -5543,7 +5784,7 @@ pub fn constructor_vec_insert_lane( } let pattern4_0 = arg3; if pattern4_0 == 0 { - // Rule at src/isa/x64/lower.isle line 1317. + // Rule at src/isa/x64/lower.isle line 1382. let expr0_0 = SseOpcode::Movsd; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern3_0)?; let expr2_0 = RegMem::Reg { reg: expr1_0 }; @@ -5551,7 +5792,7 @@ pub fn constructor_vec_insert_lane( return Some(expr3_0); } if pattern4_0 == 1 { - // Rule at src/isa/x64/lower.isle line 1325. + // Rule at src/isa/x64/lower.isle line 1390. let expr0_0 = constructor_movlhps(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } diff --git a/cranelift/codegen/src/machinst/isle.rs b/cranelift/codegen/src/machinst/isle.rs index 95541b057d57..808954dea529 100644 --- a/cranelift/codegen/src/machinst/isle.rs +++ b/cranelift/codegen/src/machinst/isle.rs @@ -48,6 +48,11 @@ macro_rules! isle_prelude_methods { ValueRegs::two(r1, r2) } + #[inline] + fn value_regs_invalid(&mut self) -> ValueRegs { + ValueRegs::invalid() + } + #[inline] fn temp_writable_reg(&mut self, ty: Type) -> WritableReg { let value_regs = self.lower_ctx.alloc_tmp(ty); diff --git a/cranelift/codegen/src/prelude.isle b/cranelift/codegen/src/prelude.isle index 96c5740b5d84..8ce3d787a52a 100644 --- a/cranelift/codegen/src/prelude.isle +++ b/cranelift/codegen/src/prelude.isle @@ -57,6 +57,10 @@ (decl value_regs (Reg Reg) ValueRegs) (extern constructor value_regs value_regs) +;; Construct an empty `ValueRegs` containing only invalid register sentinels. +(decl value_regs_invalid () ValueRegs) +(extern constructor value_regs_invalid value_regs_invalid) + ;; Get a temporary register for writing. (decl temp_writable_reg (Type) WritableReg) (extern constructor temp_writable_reg temp_writable_reg) @@ -270,6 +274,17 @@ (extractor (u64_from_iconst x) (def_inst (iconst (u64_from_imm64 x)))) +;;;; Helpers for Side-Effectful Instructions Without Results ;;;;;;;;;;;;;;;;;;; + +(type SideEffectNoResult (enum (Inst (inst MInst)))) + +;; Create an empty `ValueRegs`, but do emit the given side-effectful +;; instruction. +(decl value_regs_none (SideEffectNoResult) ValueRegs) +(rule (value_regs_none (SideEffectNoResult.Inst inst)) + (let ((_ Unit (emit inst))) + (value_regs_invalid))) + ;;;; Helpers for Working with Flags ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; Newtype wrapper around `MInst` for instructions that are used for their diff --git a/cranelift/filetests/filetests/isa/x64/i128.clif b/cranelift/filetests/filetests/isa/x64/i128.clif index f1d90210b1e7..20064dac82e8 100644 --- a/cranelift/filetests/filetests/isa/x64/i128.clif +++ b/cranelift/filetests/filetests/isa/x64/i128.clif @@ -1270,56 +1270,52 @@ block0(v0: i128, v1: i128): ; Entry block: 0 ; Block 0: ; (original IR block: block0) -; (instruction range: 0 .. 50) +; (instruction range: 0 .. 46) ; Inst 0: pushq %rbp ; Inst 1: movq %rsp, %rbp -; Inst 2: movq %rsi, %rax -; Inst 3: movq %rax, %r9 -; Inst 4: movq %rdx, %rcx -; Inst 5: shrq %cl, %r9 -; Inst 6: movq %rdi, %rsi -; Inst 7: movq %rdx, %rcx -; Inst 8: shrq %cl, %rsi -; Inst 9: movl $64, %ecx -; Inst 10: subq %rdx, %rcx -; Inst 11: movq %rax, %r10 -; Inst 12: shlq %cl, %r10 -; Inst 13: xorq %rcx, %rcx -; Inst 14: testq $127, %rdx -; Inst 15: cmovzq %rcx, %r10 -; Inst 16: orq %rsi, %r10 -; Inst 17: xorq %rsi, %rsi -; Inst 18: xorq %r8, %r8 -; Inst 19: movq %rdx, %rcx -; Inst 20: andq $64, %rcx -; Inst 21: cmovzq %r9, %rsi -; Inst 22: cmovzq %r10, %r8 -; Inst 23: cmovnzq %r9, %r8 -; Inst 24: movl $128, %r9d -; Inst 25: subq %rdx, %r9 -; Inst 26: movq %rdi, %rdx -; Inst 27: movq %r9, %rcx -; Inst 28: shlq %cl, %rdx -; Inst 29: movq %r9, %rcx -; Inst 30: shlq %cl, %rax -; Inst 31: movl $64, %ecx -; Inst 32: subq %r9, %rcx -; Inst 33: shrq %cl, %rdi -; Inst 34: xorq %rcx, %rcx -; Inst 35: testq $127, %r9 -; Inst 36: cmovzq %rcx, %rdi -; Inst 37: orq %rax, %rdi -; Inst 38: xorq %rax, %rax -; Inst 39: andq $64, %r9 -; Inst 40: cmovzq %rdi, %rax -; Inst 41: cmovzq %rdx, %rcx -; Inst 42: cmovnzq %rdx, %rax -; Inst 43: orq %r8, %rcx -; Inst 44: orq %rsi, %rax -; Inst 45: movq %rax, %rdx -; Inst 46: movq %rcx, %rax -; Inst 47: movq %rbp, %rsp -; Inst 48: popq %rbp -; Inst 49: ret +; Inst 2: movq %rdi, %rax +; Inst 3: movq %rdx, %rcx +; Inst 4: shrq %cl, %rax +; Inst 5: movq %rsi, %r8 +; Inst 6: movq %rdx, %rcx +; Inst 7: shrq %cl, %r8 +; Inst 8: movl $64, %ecx +; Inst 9: subq %rdx, %rcx +; Inst 10: movq %rsi, %r9 +; Inst 11: shlq %cl, %r9 +; Inst 12: xorq %rcx, %rcx +; Inst 13: testq $127, %rdx +; Inst 14: cmovzq %rcx, %r9 +; Inst 15: movq %r9, %rcx +; Inst 16: orq %rax, %rcx +; Inst 17: xorq %rax, %rax +; Inst 18: testq $64, %rdx +; Inst 19: cmovzq %r8, %rax +; Inst 20: cmovzq %rcx, %r8 +; Inst 21: movl $128, %r9d +; Inst 22: subq %rdx, %r9 +; Inst 23: movq %rdi, %rdx +; Inst 24: movq %r9, %rcx +; Inst 25: shlq %cl, %rdx +; Inst 26: movq %r9, %rcx +; Inst 27: shlq %cl, %rsi +; Inst 28: movl $64, %ecx +; Inst 29: subq %r9, %rcx +; Inst 30: shrq %cl, %rdi +; Inst 31: xorq %rcx, %rcx +; Inst 32: testq $127, %r9 +; Inst 33: cmovzq %rcx, %rdi +; Inst 34: orq %rsi, %rdi +; Inst 35: testq $64, %r9 +; Inst 36: movq %rdx, %rsi +; Inst 37: cmovzq %rdi, %rsi +; Inst 38: cmovzq %rdx, %rcx +; Inst 39: orq %rcx, %r8 +; Inst 40: orq %rsi, %rax +; Inst 41: movq %rax, %rdx +; Inst 42: movq %r8, %rax +; Inst 43: movq %rbp, %rsp +; Inst 44: popq %rbp +; Inst 45: ret ; }}