diff --git a/Cargo.lock b/Cargo.lock index d95912d78449..5a01aaacc735 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2381,9 +2381,9 @@ dependencies = [ [[package]] name = "regalloc" -version = "0.0.33" +version = "0.0.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d808cff91dfca7b239d40b972ba628add94892b1d9e19a842aedc5cfae8ab1a" +checksum = "62446b1d3ebf980bdc68837700af1d77b37bc430e524bf95319c6eada2a4cc02" dependencies = [ "log", "rustc-hash", diff --git a/cranelift/codegen/Cargo.toml b/cranelift/codegen/Cargo.toml index 3e20edcc8d56..27979797bf9d 100644 --- a/cranelift/codegen/Cargo.toml +++ b/cranelift/codegen/Cargo.toml @@ -23,7 +23,7 @@ serde = { version = "1.0.94", features = ["derive"], optional = true } bincode = { version = "1.2.1", optional = true } gimli = { version = "0.26.0", default-features = false, features = ["write"], optional = true } smallvec = { version = "1.6.1" } -regalloc = { version = "0.0.33" } +regalloc = "0.0.34" souper-ir = { version = "2.1.0", optional = true } # It is a goal of the cranelift-codegen crate to have minimal external dependencies. # Please don't add any unless they are essential to the task of creating binary diff --git a/cranelift/codegen/build.rs b/cranelift/codegen/build.rs index cb704cb0abbb..1b95bf02a834 100644 --- a/cranelift/codegen/build.rs +++ b/cranelift/codegen/build.rs @@ -406,6 +406,8 @@ fn rebuild_isle( ) -> Result<(), Box> { use cranelift_isle as isle; + println!("Rebuilding {}", compilation.output.display()); + // First, remove the manifest, if any; we will recreate it // below if the compilation is successful. Ignore error if no // manifest was present. diff --git a/cranelift/codegen/src/isa/aarch64/inst.isle b/cranelift/codegen/src/isa/aarch64/inst.isle index 4b6e305ba3d1..778098d81160 100644 --- a/cranelift/codegen/src/isa/aarch64/inst.isle +++ b/cranelift/codegen/src/isa/aarch64/inst.isle @@ -871,15 +871,10 @@ (type BoxCallInfo (primitive BoxCallInfo)) (type BoxCallIndInfo (primitive BoxCallIndInfo)) -(type VecMachLabel (primitive VecMachLabel)) (type CondBrKind (primitive CondBrKind)) (type BranchTarget (primitive BranchTarget)) (type BoxJTSequenceInfo (primitive BoxJTSequenceInfo)) -(type BoxExternalName (primitive BoxExternalName)) (type CodeOffset (primitive CodeOffset)) -(type ExternalName (primitive ExternalName)) -(type ValueLabel (primitive ValueLabel)) -(type UnwindInst (primitive UnwindInst)) (type ExtendOp extern (enum diff --git a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest index db1f12e7e61b..209a864f892b 100644 --- a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest +++ b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest @@ -1,4 +1,4 @@ src/clif.isle 9ea75a6f790b5c03 -src/prelude.isle 2bfcafbef6b29358 -src/isa/aarch64/inst.isle 944323ff7d6db098 +src/prelude.isle 6aaf8ce0f5a5c2ec +src/isa/aarch64/inst.isle dafd813ba278ce19 src/isa/aarch64/lower.isle 2d2e1e076a0c8a23 diff --git a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs index a5a717b448fc..54b56919b72d 100644 --- a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs +++ b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs @@ -96,19 +96,19 @@ pub trait Context { fn rotr_opposite_amount(&mut self, arg0: Type, arg1: ImmShift) -> ImmShift; } -/// Internal type SideEffectNoResult: defined at src/prelude.isle line 295. +/// Internal type SideEffectNoResult: defined at src/prelude.isle line 307. #[derive(Clone, Debug)] pub enum SideEffectNoResult { Inst { inst: MInst }, } -/// Internal type ProducesFlags: defined at src/prelude.isle line 314. +/// Internal type ProducesFlags: defined at src/prelude.isle line 326. #[derive(Clone, Debug)] pub enum ProducesFlags { ProducesFlags { inst: MInst, result: Reg }, } -/// Internal type ConsumesFlags: defined at src/prelude.isle line 317. +/// Internal type ConsumesFlags: defined at src/prelude.isle line 329. #[derive(Clone, Debug)] pub enum ConsumesFlags { ConsumesFlags { inst: MInst, result: Reg }, @@ -683,7 +683,7 @@ pub enum MInst { } /// Internal type ALUOp: defined at src/isa/aarch64/inst.isle line 789. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum ALUOp { Add32, Add64, @@ -730,7 +730,7 @@ pub enum ALUOp { } /// Internal type ALUOp3: defined at src/isa/aarch64/inst.isle line 850. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum ALUOp3 { MAdd32, MAdd64, @@ -738,8 +738,8 @@ pub enum ALUOp3 { MSub64, } -/// Internal type BitOp: defined at src/isa/aarch64/inst.isle line 898. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type BitOp: defined at src/isa/aarch64/inst.isle line 893. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum BitOp { RBit32, RBit64, @@ -749,8 +749,8 @@ pub enum BitOp { Cls64, } -/// Internal type FPUOp1: defined at src/isa/aarch64/inst.isle line 964. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type FPUOp1: defined at src/isa/aarch64/inst.isle line 959. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum FPUOp1 { Abs32, Abs64, @@ -762,8 +762,8 @@ pub enum FPUOp1 { Cvt64To32, } -/// Internal type FPUOp2: defined at src/isa/aarch64/inst.isle line 977. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type FPUOp2: defined at src/isa/aarch64/inst.isle line 972. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum FPUOp2 { Add32, Add64, @@ -783,15 +783,15 @@ pub enum FPUOp2 { Uqsub64, } -/// Internal type FPUOp3: defined at src/isa/aarch64/inst.isle line 1002. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type FPUOp3: defined at src/isa/aarch64/inst.isle line 997. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum FPUOp3 { MAdd32, MAdd64, } -/// Internal type FpuToIntOp: defined at src/isa/aarch64/inst.isle line 1009. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type FpuToIntOp: defined at src/isa/aarch64/inst.isle line 1004. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum FpuToIntOp { F32ToU32, F32ToI32, @@ -803,8 +803,8 @@ pub enum FpuToIntOp { F64ToI64, } -/// Internal type IntToFpuOp: defined at src/isa/aarch64/inst.isle line 1022. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type IntToFpuOp: defined at src/isa/aarch64/inst.isle line 1017. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum IntToFpuOp { U32ToF32, I32ToF32, @@ -816,8 +816,8 @@ pub enum IntToFpuOp { I64ToF64, } -/// Internal type FpuRoundMode: defined at src/isa/aarch64/inst.isle line 1036. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type FpuRoundMode: defined at src/isa/aarch64/inst.isle line 1031. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum FpuRoundMode { Minus32, Minus64, @@ -829,8 +829,8 @@ pub enum FpuRoundMode { Nearest64, } -/// Internal type VecExtendOp: defined at src/isa/aarch64/inst.isle line 1049. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type VecExtendOp: defined at src/isa/aarch64/inst.isle line 1044. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum VecExtendOp { Sxtl8, Sxtl16, @@ -840,8 +840,8 @@ pub enum VecExtendOp { Uxtl32, } -/// Internal type VecALUOp: defined at src/isa/aarch64/inst.isle line 1066. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type VecALUOp: defined at src/isa/aarch64/inst.isle line 1061. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum VecALUOp { Sqadd, Uqadd, @@ -882,8 +882,8 @@ pub enum VecALUOp { Sqrdmulh, } -/// Internal type VecMisc2: defined at src/isa/aarch64/inst.isle line 1145. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type VecMisc2: defined at src/isa/aarch64/inst.isle line 1140. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum VecMisc2 { Not, Neg, @@ -904,8 +904,8 @@ pub enum VecMisc2 { Cmeq0, } -/// Internal type VecRRLongOp: defined at src/isa/aarch64/inst.isle line 1184. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type VecRRLongOp: defined at src/isa/aarch64/inst.isle line 1179. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum VecRRLongOp { Fcvtl16, Fcvtl32, @@ -914,8 +914,8 @@ pub enum VecRRLongOp { Shll32, } -/// Internal type VecRRNarrowOp: defined at src/isa/aarch64/inst.isle line 1199. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type VecRRNarrowOp: defined at src/isa/aarch64/inst.isle line 1194. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum VecRRNarrowOp { Xtn16, Xtn32, @@ -933,8 +933,8 @@ pub enum VecRRNarrowOp { Fcvtn64, } -/// Internal type VecRRRLongOp: defined at src/isa/aarch64/inst.isle line 1231. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type VecRRRLongOp: defined at src/isa/aarch64/inst.isle line 1226. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum VecRRRLongOp { Smull8, Smull16, @@ -947,14 +947,14 @@ pub enum VecRRRLongOp { Umlal32, } -/// Internal type VecPairOp: defined at src/isa/aarch64/inst.isle line 1248. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type VecPairOp: defined at src/isa/aarch64/inst.isle line 1243. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum VecPairOp { Addp, } -/// Internal type VecRRPairLongOp: defined at src/isa/aarch64/inst.isle line 1256. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type VecRRPairLongOp: defined at src/isa/aarch64/inst.isle line 1251. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum VecRRPairLongOp { Saddlp8, Saddlp16, @@ -962,23 +962,23 @@ pub enum VecRRPairLongOp { Uaddlp16, } -/// Internal type VecLanesOp: defined at src/isa/aarch64/inst.isle line 1267. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type VecLanesOp: defined at src/isa/aarch64/inst.isle line 1262. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum VecLanesOp { Addv, Uminv, } -/// Internal type VecShiftImmOp: defined at src/isa/aarch64/inst.isle line 1276. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type VecShiftImmOp: defined at src/isa/aarch64/inst.isle line 1271. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum VecShiftImmOp { Shl, Ushr, Sshr, } -/// Internal type AtomicRMWOp: defined at src/isa/aarch64/inst.isle line 1287. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type AtomicRMWOp: defined at src/isa/aarch64/inst.isle line 1282. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum AtomicRMWOp { Add, Clr, @@ -993,7 +993,7 @@ pub enum AtomicRMWOp { // Generated as internal constructor for term temp_reg. pub fn constructor_temp_reg(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; - // Rule at src/prelude.isle line 70. + // Rule at src/prelude.isle line 73. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); return Some(expr1_0); @@ -1002,7 +1002,7 @@ pub fn constructor_temp_reg(ctx: &mut C, arg0: Type) -> Option // Generated as internal constructor for term lo_reg. pub fn constructor_lo_reg(ctx: &mut C, arg0: Value) -> Option { let pattern0_0 = arg0; - // Rule at src/prelude.isle line 105. + // Rule at src/prelude.isle line 108. let expr0_0 = C::put_in_regs(ctx, pattern0_0); let expr1_0: usize = 0; let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); @@ -1019,7 +1019,7 @@ pub fn constructor_value_regs_none( inst: ref pattern1_0, } = pattern0_0 { - // Rule at src/prelude.isle line 300. + // Rule at src/prelude.isle line 312. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::value_regs_invalid(ctx); return Some(expr1_0); @@ -1037,7 +1037,7 @@ pub fn constructor_safepoint( inst: ref pattern1_0, } = pattern0_0 { - // Rule at src/prelude.isle line 306. + // Rule at src/prelude.isle line 318. let expr0_0 = C::emit_safepoint(ctx, &pattern1_0); let expr1_0 = C::value_regs_invalid(ctx); return Some(expr1_0); @@ -1063,7 +1063,7 @@ pub fn constructor_with_flags( result: pattern3_1, } = pattern2_0 { - // Rule at src/prelude.isle line 327. + // Rule at src/prelude.isle line 339. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern3_0); let expr2_0 = C::value_regs(ctx, pattern1_1, pattern3_1); @@ -1091,7 +1091,7 @@ pub fn constructor_with_flags_1( result: pattern3_1, } = pattern2_0 { - // Rule at src/prelude.isle line 335. + // Rule at src/prelude.isle line 347. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern3_0); return Some(pattern3_1); @@ -1125,7 +1125,7 @@ pub fn constructor_with_flags_2( result: pattern5_1, } = pattern4_0 { - // Rule at src/prelude.isle line 345. + // Rule at src/prelude.isle line 357. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern5_0); let expr2_0 = C::emit(ctx, &pattern3_0); @@ -1143,28 +1143,28 @@ pub fn constructor_vector_size(ctx: &mut C, arg0: Type) -> Option( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1383. + // Rule at src/isa/aarch64/inst.isle line 1378. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::MovZ { @@ -1202,7 +1202,7 @@ pub fn constructor_movn( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1390. + // Rule at src/isa/aarch64/inst.isle line 1385. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::MovN { @@ -1225,7 +1225,7 @@ pub fn constructor_alu_rr_imm_logic( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1397. + // Rule at src/isa/aarch64/inst.isle line 1392. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::AluRRImmLogic { @@ -1249,7 +1249,7 @@ pub fn constructor_alu_rr_imm_shift( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1404. + // Rule at src/isa/aarch64/inst.isle line 1399. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::AluRRImmShift { @@ -1273,7 +1273,7 @@ pub fn constructor_alu_rrr( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1411. + // Rule at src/isa/aarch64/inst.isle line 1406. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::AluRRR { @@ -1299,7 +1299,7 @@ pub fn constructor_vec_rrr( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1418. + // Rule at src/isa/aarch64/inst.isle line 1413. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::VecRRR { @@ -1324,7 +1324,7 @@ pub fn constructor_vec_lanes( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1425. + // Rule at src/isa/aarch64/inst.isle line 1420. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::VecLanes { @@ -1342,7 +1342,7 @@ pub fn constructor_vec_lanes( pub fn constructor_vec_dup(ctx: &mut C, arg0: Reg, arg1: &VectorSize) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1432. + // Rule at src/isa/aarch64/inst.isle line 1427. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::VecDup { @@ -1365,7 +1365,7 @@ pub fn constructor_alu_rr_imm12( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1439. + // Rule at src/isa/aarch64/inst.isle line 1434. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::AluRRImm12 { @@ -1391,7 +1391,7 @@ pub fn constructor_alu_rrr_shift( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1446. + // Rule at src/isa/aarch64/inst.isle line 1441. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::AluRRRShift { @@ -1418,7 +1418,7 @@ pub fn constructor_alu_rrr_extend( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1453. + // Rule at src/isa/aarch64/inst.isle line 1448. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::AluRRRExtend { @@ -1443,7 +1443,7 @@ pub fn constructor_alu_rr_extend_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1461. + // Rule at src/isa/aarch64/inst.isle line 1456. let expr0_0 = C::put_extended_in_reg(ctx, pattern2_0); let expr1_0 = C::get_extended_op(ctx, pattern2_0); let expr2_0 = constructor_alu_rrr_extend(ctx, pattern0_0, pattern1_0, expr0_0, &expr1_0)?; @@ -1462,7 +1462,7 @@ pub fn constructor_alu_rrrr( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1468. + // Rule at src/isa/aarch64/inst.isle line 1463. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::AluRRRR { @@ -1481,7 +1481,7 @@ pub fn constructor_alu_rrrr( pub fn constructor_bit_rr(ctx: &mut C, arg0: &BitOp, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1475. + // Rule at src/isa/aarch64/inst.isle line 1470. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::BitRR { @@ -1502,7 +1502,7 @@ pub fn constructor_add64_with_flags( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1482. + // Rule at src/isa/aarch64/inst.isle line 1477. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = ALUOp::AddS64; @@ -1524,7 +1524,7 @@ pub fn constructor_add64_with_flags( pub fn constructor_adc64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1489. + // Rule at src/isa/aarch64/inst.isle line 1484. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = ALUOp::Adc64; @@ -1550,7 +1550,7 @@ pub fn constructor_sub64_with_flags( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1496. + // Rule at src/isa/aarch64/inst.isle line 1491. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = ALUOp::SubS64; @@ -1576,7 +1576,7 @@ pub fn constructor_cmp64_imm( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1502. + // Rule at src/isa/aarch64/inst.isle line 1497. let expr0_0 = ALUOp::SubS64; let expr1_0 = C::writable_zero_reg(ctx); let expr2_0 = MInst::AluRRImm12 { @@ -1597,7 +1597,7 @@ pub fn constructor_cmp64_imm( pub fn constructor_sbc64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1508. + // Rule at src/isa/aarch64/inst.isle line 1503. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = ALUOp::Sbc64; @@ -1625,7 +1625,7 @@ pub fn constructor_vec_misc( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1515. + // Rule at src/isa/aarch64/inst.isle line 1510. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::VecMisc { @@ -1651,7 +1651,7 @@ pub fn constructor_vec_rrr_long( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1522. + // Rule at src/isa/aarch64/inst.isle line 1517. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::VecRRRLong { @@ -1680,7 +1680,7 @@ pub fn constructor_vec_rrrr_long( let pattern2_0 = arg2; let pattern3_0 = arg3; let pattern4_0 = arg4; - // Rule at src/isa/aarch64/inst.isle line 1532. + // Rule at src/isa/aarch64/inst.isle line 1527. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::FpuMove128 { @@ -1710,7 +1710,7 @@ pub fn constructor_vec_rr_narrow( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1540. + // Rule at src/isa/aarch64/inst.isle line 1535. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::VecRRNarrow { @@ -1734,7 +1734,7 @@ pub fn constructor_vec_rr_long( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1547. + // Rule at src/isa/aarch64/inst.isle line 1542. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::VecRRLong { @@ -1756,7 +1756,7 @@ pub fn constructor_mov_to_fpu( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1554. + // Rule at src/isa/aarch64/inst.isle line 1549. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::MovToFpu { @@ -1781,7 +1781,7 @@ pub fn constructor_mov_to_vec( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1561. + // Rule at src/isa/aarch64/inst.isle line 1556. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::FpuMove128 { @@ -1810,7 +1810,7 @@ pub fn constructor_mov_from_vec( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1569. + // Rule at src/isa/aarch64/inst.isle line 1564. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::MovFromVec { @@ -1836,7 +1836,7 @@ pub fn constructor_mov_from_vec_signed( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1576. + // Rule at src/isa/aarch64/inst.isle line 1571. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::MovFromVecSigned { @@ -1863,7 +1863,7 @@ pub fn constructor_extend( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1583. + // Rule at src/isa/aarch64/inst.isle line 1578. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::Extend { @@ -1882,7 +1882,7 @@ pub fn constructor_extend( pub fn constructor_load_acquire(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1590. + // Rule at src/isa/aarch64/inst.isle line 1585. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::LoadAcquire { @@ -1903,7 +1903,7 @@ pub fn constructor_tst64_imm( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1600. + // Rule at src/isa/aarch64/inst.isle line 1595. let expr0_0 = ALUOp::AndS64; let expr1_0 = C::writable_zero_reg(ctx); let expr2_0 = MInst::AluRRImmLogic { @@ -1930,7 +1930,7 @@ pub fn constructor_csel( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1613. + // Rule at src/isa/aarch64/inst.isle line 1608. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::CSel { @@ -1953,14 +1953,14 @@ pub fn constructor_add(ctx: &mut C, arg0: Type, arg1: Reg, arg2: Reg if pattern0_0 == I64 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1622. + // Rule at src/isa/aarch64/inst.isle line 1617. let expr0_0 = constructor_add64(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if let Some(pattern1_0) = C::fits_in_32(ctx, pattern0_0) { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1621. + // Rule at src/isa/aarch64/inst.isle line 1616. let expr0_0 = constructor_add32(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } @@ -1971,7 +1971,7 @@ pub fn constructor_add(ctx: &mut C, arg0: Type, arg1: Reg, arg2: Reg pub fn constructor_add32(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1625. + // Rule at src/isa/aarch64/inst.isle line 1620. let expr0_0 = ALUOp::Add32; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -1981,7 +1981,7 @@ pub fn constructor_add32(ctx: &mut C, arg0: Reg, arg1: Reg) -> Optio pub fn constructor_add64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1628. + // Rule at src/isa/aarch64/inst.isle line 1623. let expr0_0 = ALUOp::Add64; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -1998,14 +1998,14 @@ pub fn constructor_add_imm( if pattern0_0 == I64 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1632. + // Rule at src/isa/aarch64/inst.isle line 1627. let expr0_0 = constructor_add64_imm(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if let Some(pattern1_0) = C::fits_in_32(ctx, pattern0_0) { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1631. + // Rule at src/isa/aarch64/inst.isle line 1626. let expr0_0 = constructor_add32_imm(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } @@ -2016,7 +2016,7 @@ pub fn constructor_add_imm( pub fn constructor_add32_imm(ctx: &mut C, arg0: Reg, arg1: Imm12) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1635. + // Rule at src/isa/aarch64/inst.isle line 1630. let expr0_0 = ALUOp::Add32; let expr1_0 = constructor_alu_rr_imm12(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2026,7 +2026,7 @@ pub fn constructor_add32_imm(ctx: &mut C, arg0: Reg, arg1: Imm12) -> pub fn constructor_add64_imm(ctx: &mut C, arg0: Reg, arg1: Imm12) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1638. + // Rule at src/isa/aarch64/inst.isle line 1633. let expr0_0 = ALUOp::Add64; let expr1_0 = constructor_alu_rr_imm12(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2043,14 +2043,14 @@ pub fn constructor_add_extend( if pattern0_0 == I64 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1642. + // Rule at src/isa/aarch64/inst.isle line 1637. let expr0_0 = constructor_add64_extend(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if let Some(pattern1_0) = C::fits_in_32(ctx, pattern0_0) { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1641. + // Rule at src/isa/aarch64/inst.isle line 1636. let expr0_0 = constructor_add32_extend(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } @@ -2065,7 +2065,7 @@ pub fn constructor_add32_extend( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1645. + // Rule at src/isa/aarch64/inst.isle line 1640. let expr0_0 = ALUOp::Add32; let expr1_0 = constructor_alu_rr_extend_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2079,7 +2079,7 @@ pub fn constructor_add64_extend( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1648. + // Rule at src/isa/aarch64/inst.isle line 1643. let expr0_0 = ALUOp::Add64; let expr1_0 = constructor_alu_rr_extend_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2098,7 +2098,7 @@ pub fn constructor_add_shift( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1652. + // Rule at src/isa/aarch64/inst.isle line 1647. let expr0_0 = constructor_add64_shift(ctx, pattern2_0, pattern3_0, pattern4_0)?; return Some(expr0_0); } @@ -2106,7 +2106,7 @@ pub fn constructor_add_shift( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1651. + // Rule at src/isa/aarch64/inst.isle line 1646. let expr0_0 = constructor_add32_shift(ctx, pattern2_0, pattern3_0, pattern4_0)?; return Some(expr0_0); } @@ -2123,7 +2123,7 @@ pub fn constructor_add32_shift( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1655. + // Rule at src/isa/aarch64/inst.isle line 1650. let expr0_0 = ALUOp::Add32; let expr1_0 = constructor_alu_rrr_shift(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2139,7 +2139,7 @@ pub fn constructor_add64_shift( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1658. + // Rule at src/isa/aarch64/inst.isle line 1653. let expr0_0 = ALUOp::Add64; let expr1_0 = constructor_alu_rrr_shift(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2155,7 +2155,7 @@ pub fn constructor_add_vec( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1661. + // Rule at src/isa/aarch64/inst.isle line 1656. let expr0_0 = VecALUOp::Add; let expr1_0 = constructor_vec_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2167,14 +2167,14 @@ pub fn constructor_sub(ctx: &mut C, arg0: Type, arg1: Reg, arg2: Reg if pattern0_0 == I64 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1667. + // Rule at src/isa/aarch64/inst.isle line 1662. let expr0_0 = constructor_sub64(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if let Some(pattern1_0) = C::fits_in_32(ctx, pattern0_0) { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1666. + // Rule at src/isa/aarch64/inst.isle line 1661. let expr0_0 = constructor_sub32(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } @@ -2185,7 +2185,7 @@ pub fn constructor_sub(ctx: &mut C, arg0: Type, arg1: Reg, arg2: Reg pub fn constructor_sub32(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1670. + // Rule at src/isa/aarch64/inst.isle line 1665. let expr0_0 = ALUOp::Sub32; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2195,7 +2195,7 @@ pub fn constructor_sub32(ctx: &mut C, arg0: Reg, arg1: Reg) -> Optio pub fn constructor_sub64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1673. + // Rule at src/isa/aarch64/inst.isle line 1668. let expr0_0 = ALUOp::Sub64; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2212,14 +2212,14 @@ pub fn constructor_sub_imm( if pattern0_0 == I64 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1677. + // Rule at src/isa/aarch64/inst.isle line 1672. let expr0_0 = constructor_sub64_imm(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if let Some(pattern1_0) = C::fits_in_32(ctx, pattern0_0) { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1676. + // Rule at src/isa/aarch64/inst.isle line 1671. let expr0_0 = constructor_sub32_imm(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } @@ -2230,7 +2230,7 @@ pub fn constructor_sub_imm( pub fn constructor_sub32_imm(ctx: &mut C, arg0: Reg, arg1: Imm12) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1680. + // Rule at src/isa/aarch64/inst.isle line 1675. let expr0_0 = ALUOp::Sub32; let expr1_0 = constructor_alu_rr_imm12(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2240,7 +2240,7 @@ pub fn constructor_sub32_imm(ctx: &mut C, arg0: Reg, arg1: Imm12) -> pub fn constructor_sub64_imm(ctx: &mut C, arg0: Reg, arg1: Imm12) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1683. + // Rule at src/isa/aarch64/inst.isle line 1678. let expr0_0 = ALUOp::Sub64; let expr1_0 = constructor_alu_rr_imm12(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2257,14 +2257,14 @@ pub fn constructor_sub_extend( if pattern0_0 == I64 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1687. + // Rule at src/isa/aarch64/inst.isle line 1682. let expr0_0 = constructor_sub64_extend(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if let Some(pattern1_0) = C::fits_in_32(ctx, pattern0_0) { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1686. + // Rule at src/isa/aarch64/inst.isle line 1681. let expr0_0 = constructor_sub32_extend(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } @@ -2279,7 +2279,7 @@ pub fn constructor_sub32_extend( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1690. + // Rule at src/isa/aarch64/inst.isle line 1685. let expr0_0 = ALUOp::Sub32; let expr1_0 = constructor_alu_rr_extend_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2293,7 +2293,7 @@ pub fn constructor_sub64_extend( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1693. + // Rule at src/isa/aarch64/inst.isle line 1688. let expr0_0 = ALUOp::Sub64; let expr1_0 = constructor_alu_rr_extend_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2312,7 +2312,7 @@ pub fn constructor_sub_shift( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1697. + // Rule at src/isa/aarch64/inst.isle line 1692. let expr0_0 = constructor_sub64_shift(ctx, pattern2_0, pattern3_0, pattern4_0)?; return Some(expr0_0); } @@ -2320,7 +2320,7 @@ pub fn constructor_sub_shift( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1696. + // Rule at src/isa/aarch64/inst.isle line 1691. let expr0_0 = constructor_sub32_shift(ctx, pattern2_0, pattern3_0, pattern4_0)?; return Some(expr0_0); } @@ -2337,7 +2337,7 @@ pub fn constructor_sub32_shift( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1700. + // Rule at src/isa/aarch64/inst.isle line 1695. let expr0_0 = ALUOp::Sub32; let expr1_0 = constructor_alu_rrr_shift(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2353,7 +2353,7 @@ pub fn constructor_sub64_shift( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1703. + // Rule at src/isa/aarch64/inst.isle line 1698. let expr0_0 = ALUOp::Sub64; let expr1_0 = constructor_alu_rrr_shift(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2369,7 +2369,7 @@ pub fn constructor_sub_vec( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1706. + // Rule at src/isa/aarch64/inst.isle line 1701. let expr0_0 = VecALUOp::Sub; let expr1_0 = constructor_vec_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2388,7 +2388,7 @@ pub fn constructor_madd( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1712. + // Rule at src/isa/aarch64/inst.isle line 1707. let expr0_0 = constructor_madd64(ctx, pattern2_0, pattern3_0, pattern4_0)?; return Some(expr0_0); } @@ -2396,7 +2396,7 @@ pub fn constructor_madd( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1711. + // Rule at src/isa/aarch64/inst.isle line 1706. let expr0_0 = constructor_madd32(ctx, pattern2_0, pattern3_0, pattern4_0)?; return Some(expr0_0); } @@ -2408,7 +2408,7 @@ pub fn constructor_madd32(ctx: &mut C, arg0: Reg, arg1: Reg, arg2: R let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1715. + // Rule at src/isa/aarch64/inst.isle line 1710. let expr0_0 = ALUOp3::MAdd32; let expr1_0 = constructor_alu_rrrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2419,7 +2419,7 @@ pub fn constructor_madd64(ctx: &mut C, arg0: Reg, arg1: Reg, arg2: R let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1718. + // Rule at src/isa/aarch64/inst.isle line 1713. let expr0_0 = ALUOp3::MAdd64; let expr1_0 = constructor_alu_rrrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2430,7 +2430,7 @@ pub fn constructor_msub64(ctx: &mut C, arg0: Reg, arg1: Reg, arg2: R let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1723. + // Rule at src/isa/aarch64/inst.isle line 1718. let expr0_0 = ALUOp3::MSub64; let expr1_0 = constructor_alu_rrrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2446,7 +2446,7 @@ pub fn constructor_uqadd( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1727. + // Rule at src/isa/aarch64/inst.isle line 1722. let expr0_0 = VecALUOp::Uqadd; let expr1_0 = constructor_vec_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2462,7 +2462,7 @@ pub fn constructor_sqadd( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1731. + // Rule at src/isa/aarch64/inst.isle line 1726. let expr0_0 = VecALUOp::Sqadd; let expr1_0 = constructor_vec_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2478,7 +2478,7 @@ pub fn constructor_uqsub( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1735. + // Rule at src/isa/aarch64/inst.isle line 1730. let expr0_0 = VecALUOp::Uqsub; let expr1_0 = constructor_vec_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2494,7 +2494,7 @@ pub fn constructor_sqsub( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1739. + // Rule at src/isa/aarch64/inst.isle line 1734. let expr0_0 = VecALUOp::Sqsub; let expr1_0 = constructor_vec_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2504,7 +2504,7 @@ pub fn constructor_sqsub( pub fn constructor_umulh(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1743. + // Rule at src/isa/aarch64/inst.isle line 1738. let expr0_0 = ALUOp::UMulH; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2514,7 +2514,7 @@ pub fn constructor_umulh(ctx: &mut C, arg0: Reg, arg1: Reg) -> Optio pub fn constructor_smulh(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1747. + // Rule at src/isa/aarch64/inst.isle line 1742. let expr0_0 = ALUOp::SMulH; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2530,7 +2530,7 @@ pub fn constructor_mul( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1751. + // Rule at src/isa/aarch64/inst.isle line 1746. let expr0_0 = VecALUOp::Mul; let expr1_0 = constructor_vec_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2540,7 +2540,7 @@ pub fn constructor_mul( pub fn constructor_neg(ctx: &mut C, arg0: Reg, arg1: &VectorSize) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1755. + // Rule at src/isa/aarch64/inst.isle line 1750. let expr0_0 = VecMisc2::Neg; let expr1_0 = constructor_vec_misc(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2550,7 +2550,7 @@ pub fn constructor_neg(ctx: &mut C, arg0: Reg, arg1: &VectorSize) -> pub fn constructor_rev64(ctx: &mut C, arg0: Reg, arg1: &VectorSize) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1759. + // Rule at src/isa/aarch64/inst.isle line 1754. let expr0_0 = VecMisc2::Rev64; let expr1_0 = constructor_vec_misc(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2560,7 +2560,7 @@ pub fn constructor_rev64(ctx: &mut C, arg0: Reg, arg1: &VectorSize) pub fn constructor_xtn64(ctx: &mut C, arg0: Reg, arg1: bool) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1763. + // Rule at src/isa/aarch64/inst.isle line 1758. let expr0_0 = VecRRNarrowOp::Xtn64; let expr1_0 = constructor_vec_rr_narrow(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2576,7 +2576,7 @@ pub fn constructor_addp( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1767. + // Rule at src/isa/aarch64/inst.isle line 1762. let expr0_0 = VecALUOp::Addp; let expr1_0 = constructor_vec_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2586,7 +2586,7 @@ pub fn constructor_addp( pub fn constructor_addv(ctx: &mut C, arg0: Reg, arg1: &VectorSize) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1771. + // Rule at src/isa/aarch64/inst.isle line 1766. let expr0_0 = VecLanesOp::Addv; let expr1_0 = constructor_vec_lanes(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2596,7 +2596,7 @@ pub fn constructor_addv(ctx: &mut C, arg0: Reg, arg1: &VectorSize) - pub fn constructor_shll32(ctx: &mut C, arg0: Reg, arg1: bool) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1775. + // Rule at src/isa/aarch64/inst.isle line 1770. let expr0_0 = VecRRLongOp::Shll32; let expr1_0 = constructor_vec_rr_long(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2614,7 +2614,7 @@ pub fn constructor_umlal32( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1779. + // Rule at src/isa/aarch64/inst.isle line 1774. let expr0_0 = VecRRRLongOp::Umlal32; let expr1_0 = constructor_vec_rrrr_long( ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0, pattern3_0, @@ -2632,7 +2632,7 @@ pub fn constructor_smull8( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1783. + // Rule at src/isa/aarch64/inst.isle line 1778. let expr0_0 = VecRRRLongOp::Smull8; let expr1_0 = constructor_vec_rrr_long(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2648,7 +2648,7 @@ pub fn constructor_umull8( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1787. + // Rule at src/isa/aarch64/inst.isle line 1782. let expr0_0 = VecRRRLongOp::Umull8; let expr1_0 = constructor_vec_rrr_long(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2664,7 +2664,7 @@ pub fn constructor_smull16( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1791. + // Rule at src/isa/aarch64/inst.isle line 1786. let expr0_0 = VecRRRLongOp::Smull16; let expr1_0 = constructor_vec_rrr_long(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2680,7 +2680,7 @@ pub fn constructor_umull16( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1795. + // Rule at src/isa/aarch64/inst.isle line 1790. let expr0_0 = VecRRRLongOp::Umull16; let expr1_0 = constructor_vec_rrr_long(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2696,7 +2696,7 @@ pub fn constructor_smull32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1799. + // Rule at src/isa/aarch64/inst.isle line 1794. let expr0_0 = VecRRRLongOp::Smull32; let expr1_0 = constructor_vec_rrr_long(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2712,7 +2712,7 @@ pub fn constructor_umull32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1803. + // Rule at src/isa/aarch64/inst.isle line 1798. let expr0_0 = VecRRRLongOp::Umull32; let expr1_0 = constructor_vec_rrr_long(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2722,7 +2722,7 @@ pub fn constructor_umull32( pub fn constructor_asr64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1807. + // Rule at src/isa/aarch64/inst.isle line 1802. let expr0_0 = ALUOp::Asr64; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2732,7 +2732,7 @@ pub fn constructor_asr64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Optio pub fn constructor_asr64_imm(ctx: &mut C, arg0: Reg, arg1: ImmShift) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1810. + // Rule at src/isa/aarch64/inst.isle line 1805. let expr0_0 = ALUOp::Asr64; let expr1_0 = constructor_alu_rr_imm_shift(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2742,7 +2742,7 @@ pub fn constructor_asr64_imm(ctx: &mut C, arg0: Reg, arg1: ImmShift) pub fn constructor_lsr32(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1814. + // Rule at src/isa/aarch64/inst.isle line 1809. let expr0_0 = ALUOp::Lsr32; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2752,7 +2752,7 @@ pub fn constructor_lsr32(ctx: &mut C, arg0: Reg, arg1: Reg) -> Optio pub fn constructor_lsr32_imm(ctx: &mut C, arg0: Reg, arg1: ImmShift) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1817. + // Rule at src/isa/aarch64/inst.isle line 1812. let expr0_0 = ALUOp::Lsr32; let expr1_0 = constructor_alu_rr_imm_shift(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2762,7 +2762,7 @@ pub fn constructor_lsr32_imm(ctx: &mut C, arg0: Reg, arg1: ImmShift) pub fn constructor_lsr64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1820. + // Rule at src/isa/aarch64/inst.isle line 1815. let expr0_0 = ALUOp::Lsr64; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2772,7 +2772,7 @@ pub fn constructor_lsr64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Optio pub fn constructor_lsr64_imm(ctx: &mut C, arg0: Reg, arg1: ImmShift) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1823. + // Rule at src/isa/aarch64/inst.isle line 1818. let expr0_0 = ALUOp::Lsr64; let expr1_0 = constructor_alu_rr_imm_shift(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2782,7 +2782,7 @@ pub fn constructor_lsr64_imm(ctx: &mut C, arg0: Reg, arg1: ImmShift) pub fn constructor_lsl32(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1827. + // Rule at src/isa/aarch64/inst.isle line 1822. let expr0_0 = ALUOp::Lsl32; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2792,7 +2792,7 @@ pub fn constructor_lsl32(ctx: &mut C, arg0: Reg, arg1: Reg) -> Optio pub fn constructor_lsl32_imm(ctx: &mut C, arg0: Reg, arg1: ImmShift) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1830. + // Rule at src/isa/aarch64/inst.isle line 1825. let expr0_0 = ALUOp::Lsl32; let expr1_0 = constructor_alu_rr_imm_shift(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2802,7 +2802,7 @@ pub fn constructor_lsl32_imm(ctx: &mut C, arg0: Reg, arg1: ImmShift) pub fn constructor_lsl64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1833. + // Rule at src/isa/aarch64/inst.isle line 1828. let expr0_0 = ALUOp::Lsl64; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2812,7 +2812,7 @@ pub fn constructor_lsl64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Optio pub fn constructor_lsl64_imm(ctx: &mut C, arg0: Reg, arg1: ImmShift) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1836. + // Rule at src/isa/aarch64/inst.isle line 1831. let expr0_0 = ALUOp::Lsl64; let expr1_0 = constructor_alu_rr_imm_shift(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2822,7 +2822,7 @@ pub fn constructor_lsl64_imm(ctx: &mut C, arg0: Reg, arg1: ImmShift) pub fn constructor_udiv64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1840. + // Rule at src/isa/aarch64/inst.isle line 1835. let expr0_0 = ALUOp::UDiv64; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2832,7 +2832,7 @@ pub fn constructor_udiv64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Opti pub fn constructor_sdiv64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1844. + // Rule at src/isa/aarch64/inst.isle line 1839. let expr0_0 = ALUOp::SDiv64; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2842,7 +2842,7 @@ pub fn constructor_sdiv64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Opti pub fn constructor_not(ctx: &mut C, arg0: Reg, arg1: &VectorSize) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1848. + // Rule at src/isa/aarch64/inst.isle line 1843. let expr0_0 = VecMisc2::Not; let expr1_0 = constructor_vec_misc(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2859,14 +2859,14 @@ pub fn constructor_orr_not( if pattern0_0 == I64 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1854. + // Rule at src/isa/aarch64/inst.isle line 1849. let expr0_0 = constructor_orr_not64(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if let Some(pattern1_0) = C::fits_in_32(ctx, pattern0_0) { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1853. + // Rule at src/isa/aarch64/inst.isle line 1848. let expr0_0 = constructor_orr_not32(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } @@ -2877,7 +2877,7 @@ pub fn constructor_orr_not( pub fn constructor_orr_not32(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1857. + // Rule at src/isa/aarch64/inst.isle line 1852. let expr0_0 = ALUOp::OrrNot32; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2887,7 +2887,7 @@ pub fn constructor_orr_not32(ctx: &mut C, arg0: Reg, arg1: Reg) -> O pub fn constructor_orr_not64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1860. + // Rule at src/isa/aarch64/inst.isle line 1855. let expr0_0 = ALUOp::OrrNot64; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2906,7 +2906,7 @@ pub fn constructor_orr_not_shift( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1864. + // Rule at src/isa/aarch64/inst.isle line 1859. let expr0_0 = constructor_orr_not_shift64(ctx, pattern2_0, pattern3_0, pattern4_0)?; return Some(expr0_0); } @@ -2914,7 +2914,7 @@ pub fn constructor_orr_not_shift( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1863. + // Rule at src/isa/aarch64/inst.isle line 1858. let expr0_0 = constructor_orr_not_shift32(ctx, pattern2_0, pattern3_0, pattern4_0)?; return Some(expr0_0); } @@ -2931,7 +2931,7 @@ pub fn constructor_orr_not_shift32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1867. + // Rule at src/isa/aarch64/inst.isle line 1862. let expr0_0 = ALUOp::OrrNot32; let expr1_0 = constructor_alu_rrr_shift(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2947,7 +2947,7 @@ pub fn constructor_orr_not_shift64( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1870. + // Rule at src/isa/aarch64/inst.isle line 1865. let expr0_0 = ALUOp::OrrNot64; let expr1_0 = constructor_alu_rrr_shift(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2957,7 +2957,7 @@ pub fn constructor_orr_not_shift64( pub fn constructor_orr32(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1875. + // Rule at src/isa/aarch64/inst.isle line 1870. let expr0_0 = ALUOp::Orr32; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2967,7 +2967,7 @@ pub fn constructor_orr32(ctx: &mut C, arg0: Reg, arg1: Reg) -> Optio pub fn constructor_orr32_imm(ctx: &mut C, arg0: Reg, arg1: ImmLogic) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1878. + // Rule at src/isa/aarch64/inst.isle line 1873. let expr0_0 = ALUOp::Orr32; let expr1_0 = constructor_alu_rr_imm_logic(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2977,7 +2977,7 @@ pub fn constructor_orr32_imm(ctx: &mut C, arg0: Reg, arg1: ImmLogic) pub fn constructor_orr64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1881. + // Rule at src/isa/aarch64/inst.isle line 1876. let expr0_0 = ALUOp::Orr64; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2987,7 +2987,7 @@ pub fn constructor_orr64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Optio pub fn constructor_orr64_imm(ctx: &mut C, arg0: Reg, arg1: ImmLogic) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1884. + // Rule at src/isa/aarch64/inst.isle line 1879. let expr0_0 = ALUOp::Orr64; let expr1_0 = constructor_alu_rr_imm_logic(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -3003,7 +3003,7 @@ pub fn constructor_orr_vec( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1887. + // Rule at src/isa/aarch64/inst.isle line 1882. let expr0_0 = VecALUOp::Orr; let expr1_0 = constructor_vec_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -3013,7 +3013,7 @@ pub fn constructor_orr_vec( pub fn constructor_and32_imm(ctx: &mut C, arg0: Reg, arg1: ImmLogic) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1892. + // Rule at src/isa/aarch64/inst.isle line 1887. let expr0_0 = ALUOp::And32; let expr1_0 = constructor_alu_rr_imm_logic(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -3029,7 +3029,7 @@ pub fn constructor_and_vec( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1895. + // Rule at src/isa/aarch64/inst.isle line 1890. let expr0_0 = VecALUOp::And; let expr1_0 = constructor_vec_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -3045,7 +3045,7 @@ pub fn constructor_eor_vec( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1899. + // Rule at src/isa/aarch64/inst.isle line 1894. let expr0_0 = VecALUOp::Eor; let expr1_0 = constructor_vec_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -3061,7 +3061,7 @@ pub fn constructor_bic_vec( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1903. + // Rule at src/isa/aarch64/inst.isle line 1898. let expr0_0 = VecALUOp::Bic; let expr1_0 = constructor_vec_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -3077,7 +3077,7 @@ pub fn constructor_sshl( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1907. + // Rule at src/isa/aarch64/inst.isle line 1902. let expr0_0 = VecALUOp::Sshl; let expr1_0 = constructor_vec_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -3093,7 +3093,7 @@ pub fn constructor_ushl( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1911. + // Rule at src/isa/aarch64/inst.isle line 1906. let expr0_0 = VecALUOp::Ushl; let expr1_0 = constructor_vec_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -3103,7 +3103,7 @@ pub fn constructor_ushl( pub fn constructor_rotr32(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1916. + // Rule at src/isa/aarch64/inst.isle line 1911. let expr0_0 = ALUOp::RotR32; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -3113,7 +3113,7 @@ pub fn constructor_rotr32(ctx: &mut C, arg0: Reg, arg1: Reg) -> Opti pub fn constructor_rotr32_imm(ctx: &mut C, arg0: Reg, arg1: ImmShift) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1919. + // Rule at src/isa/aarch64/inst.isle line 1914. let expr0_0 = ALUOp::RotR32; let expr1_0 = constructor_alu_rr_imm_shift(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -3123,7 +3123,7 @@ pub fn constructor_rotr32_imm(ctx: &mut C, arg0: Reg, arg1: ImmShift pub fn constructor_rotr64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1922. + // Rule at src/isa/aarch64/inst.isle line 1917. let expr0_0 = ALUOp::RotR64; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -3133,7 +3133,7 @@ pub fn constructor_rotr64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Opti pub fn constructor_rotr64_imm(ctx: &mut C, arg0: Reg, arg1: ImmShift) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1925. + // Rule at src/isa/aarch64/inst.isle line 1920. let expr0_0 = ALUOp::RotR64; let expr1_0 = constructor_alu_rr_imm_shift(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -3142,7 +3142,7 @@ pub fn constructor_rotr64_imm(ctx: &mut C, arg0: Reg, arg1: ImmShift // Generated as internal constructor for term rbit32. pub fn constructor_rbit32(ctx: &mut C, arg0: Reg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/aarch64/inst.isle line 1930. + // Rule at src/isa/aarch64/inst.isle line 1925. let expr0_0 = BitOp::RBit32; let expr1_0 = constructor_bit_rr(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -3151,7 +3151,7 @@ pub fn constructor_rbit32(ctx: &mut C, arg0: Reg) -> Option { // Generated as internal constructor for term rbit64. pub fn constructor_rbit64(ctx: &mut C, arg0: Reg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/aarch64/inst.isle line 1933. + // Rule at src/isa/aarch64/inst.isle line 1928. let expr0_0 = BitOp::RBit64; let expr1_0 = constructor_bit_rr(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -3160,7 +3160,7 @@ pub fn constructor_rbit64(ctx: &mut C, arg0: Reg) -> Option { // Generated as internal constructor for term clz32. pub fn constructor_clz32(ctx: &mut C, arg0: Reg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/aarch64/inst.isle line 1938. + // Rule at src/isa/aarch64/inst.isle line 1933. let expr0_0 = BitOp::Clz32; let expr1_0 = constructor_bit_rr(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -3169,7 +3169,7 @@ pub fn constructor_clz32(ctx: &mut C, arg0: Reg) -> Option { // Generated as internal constructor for term clz64. pub fn constructor_clz64(ctx: &mut C, arg0: Reg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/aarch64/inst.isle line 1941. + // Rule at src/isa/aarch64/inst.isle line 1936. let expr0_0 = BitOp::Clz64; let expr1_0 = constructor_bit_rr(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -3178,7 +3178,7 @@ pub fn constructor_clz64(ctx: &mut C, arg0: Reg) -> Option { // Generated as internal constructor for term cls32. pub fn constructor_cls32(ctx: &mut C, arg0: Reg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/aarch64/inst.isle line 1946. + // Rule at src/isa/aarch64/inst.isle line 1941. let expr0_0 = BitOp::Cls32; let expr1_0 = constructor_bit_rr(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -3187,7 +3187,7 @@ pub fn constructor_cls32(ctx: &mut C, arg0: Reg) -> Option { // Generated as internal constructor for term cls64. pub fn constructor_cls64(ctx: &mut C, arg0: Reg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/aarch64/inst.isle line 1949. + // Rule at src/isa/aarch64/inst.isle line 1944. let expr0_0 = BitOp::Cls64; let expr1_0 = constructor_bit_rr(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -3197,7 +3197,7 @@ pub fn constructor_cls64(ctx: &mut C, arg0: Reg) -> Option { pub fn constructor_eon32(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1954. + // Rule at src/isa/aarch64/inst.isle line 1949. let expr0_0 = ALUOp::EorNot32; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -3207,7 +3207,7 @@ pub fn constructor_eon32(ctx: &mut C, arg0: Reg, arg1: Reg) -> Optio pub fn constructor_eon64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1957. + // Rule at src/isa/aarch64/inst.isle line 1952. let expr0_0 = ALUOp::EorNot64; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -3217,7 +3217,7 @@ pub fn constructor_eon64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Optio pub fn constructor_vec_cnt(ctx: &mut C, arg0: Reg, arg1: &VectorSize) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1962. + // Rule at src/isa/aarch64/inst.isle line 1957. let expr0_0 = VecMisc2::Cnt; let expr1_0 = constructor_vec_misc(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -3234,25 +3234,25 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option }; if let Some(pattern3_0) = closure3() { if let Some(pattern4_0) = C::imm_logic_from_u64(ctx, pattern2_0, pattern3_0) { - // Rule at src/isa/aarch64/inst.isle line 1977. + // Rule at src/isa/aarch64/inst.isle line 1972. let expr0_0 = C::zero_reg(ctx); let expr1_0 = constructor_orr64_imm(ctx, expr0_0, pattern4_0)?; return Some(expr1_0); } } if let Some(pattern3_0) = C::move_wide_const_from_u64(ctx, pattern2_0) { - // Rule at src/isa/aarch64/inst.isle line 1969. + // Rule at src/isa/aarch64/inst.isle line 1964. let expr0_0 = OperandSize::Size64; let expr1_0 = constructor_movz(ctx, pattern3_0, &expr0_0)?; return Some(expr1_0); } if let Some(pattern3_0) = C::move_wide_const_from_negated_u64(ctx, pattern2_0) { - // Rule at src/isa/aarch64/inst.isle line 1973. + // Rule at src/isa/aarch64/inst.isle line 1968. let expr0_0 = OperandSize::Size64; let expr1_0 = constructor_movn(ctx, pattern3_0, &expr0_0)?; return Some(expr1_0); } - // Rule at src/isa/aarch64/inst.isle line 1984. + // Rule at src/isa/aarch64/inst.isle line 1979. let expr0_0 = C::load_constant64_full(ctx, pattern2_0); return Some(expr0_0); } @@ -3264,17 +3264,17 @@ pub fn constructor_put_in_reg_sext32(ctx: &mut C, arg0: Value) -> Op let pattern0_0 = arg0; let pattern1_0 = C::value_type(ctx, pattern0_0); if pattern1_0 == I32 { - // Rule at src/isa/aarch64/inst.isle line 1995. + // Rule at src/isa/aarch64/inst.isle line 1990. let expr0_0 = C::put_in_reg(ctx, pattern0_0); return Some(expr0_0); } if pattern1_0 == I64 { - // Rule at src/isa/aarch64/inst.isle line 1996. + // Rule at src/isa/aarch64/inst.isle line 1991. let expr0_0 = C::put_in_reg(ctx, pattern0_0); return Some(expr0_0); } if let Some(pattern2_0) = C::fits_in_32(ctx, pattern1_0) { - // Rule at src/isa/aarch64/inst.isle line 1991. + // Rule at src/isa/aarch64/inst.isle line 1986. let expr0_0 = C::put_in_reg(ctx, pattern0_0); let expr1_0: bool = true; let expr2_0 = C::ty_bits(ctx, pattern2_0); @@ -3290,17 +3290,17 @@ pub fn constructor_put_in_reg_zext32(ctx: &mut C, arg0: Value) -> Op let pattern0_0 = arg0; let pattern1_0 = C::value_type(ctx, pattern0_0); if pattern1_0 == I32 { - // Rule at src/isa/aarch64/inst.isle line 2004. + // Rule at src/isa/aarch64/inst.isle line 1999. let expr0_0 = C::put_in_reg(ctx, pattern0_0); return Some(expr0_0); } if pattern1_0 == I64 { - // Rule at src/isa/aarch64/inst.isle line 2005. + // Rule at src/isa/aarch64/inst.isle line 2000. let expr0_0 = C::put_in_reg(ctx, pattern0_0); return Some(expr0_0); } if let Some(pattern2_0) = C::fits_in_32(ctx, pattern1_0) { - // Rule at src/isa/aarch64/inst.isle line 2000. + // Rule at src/isa/aarch64/inst.isle line 1995. let expr0_0 = C::put_in_reg(ctx, pattern0_0); let expr1_0: bool = false; let expr2_0 = C::ty_bits(ctx, pattern2_0); @@ -3316,12 +3316,12 @@ pub fn constructor_put_in_reg_sext64(ctx: &mut C, arg0: Value) -> Op let pattern0_0 = arg0; let pattern1_0 = C::value_type(ctx, pattern0_0); if pattern1_0 == I64 { - // Rule at src/isa/aarch64/inst.isle line 2013. + // Rule at src/isa/aarch64/inst.isle line 2008. let expr0_0 = C::put_in_reg(ctx, pattern0_0); return Some(expr0_0); } if let Some(pattern2_0) = C::fits_in_32(ctx, pattern1_0) { - // Rule at src/isa/aarch64/inst.isle line 2009. + // Rule at src/isa/aarch64/inst.isle line 2004. let expr0_0 = C::put_in_reg(ctx, pattern0_0); let expr1_0: bool = true; let expr2_0 = C::ty_bits(ctx, pattern2_0); @@ -3337,12 +3337,12 @@ pub fn constructor_put_in_reg_zext64(ctx: &mut C, arg0: Value) -> Op let pattern0_0 = arg0; let pattern1_0 = C::value_type(ctx, pattern0_0); if pattern1_0 == I64 { - // Rule at src/isa/aarch64/inst.isle line 2021. + // Rule at src/isa/aarch64/inst.isle line 2016. let expr0_0 = C::put_in_reg(ctx, pattern0_0); return Some(expr0_0); } if let Some(pattern2_0) = C::fits_in_32(ctx, pattern1_0) { - // Rule at src/isa/aarch64/inst.isle line 2017. + // Rule at src/isa/aarch64/inst.isle line 2012. let expr0_0 = C::put_in_reg(ctx, pattern0_0); let expr1_0: bool = false; let expr2_0 = C::ty_bits(ctx, pattern2_0); @@ -3356,7 +3356,7 @@ pub fn constructor_put_in_reg_zext64(ctx: &mut C, arg0: Value) -> Op // Generated as internal constructor for term trap_if_zero_divisor. pub fn constructor_trap_if_zero_divisor(ctx: &mut C, arg0: Reg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/aarch64/inst.isle line 2026. + // Rule at src/isa/aarch64/inst.isle line 2021. let expr0_0 = C::cond_br_zero(ctx, pattern0_0); let expr1_0 = C::trap_code_division_by_zero(ctx); let expr2_0 = MInst::TrapIf { @@ -3371,12 +3371,12 @@ pub fn constructor_trap_if_zero_divisor(ctx: &mut C, arg0: Reg) -> O pub fn constructor_size_from_ty(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I64 { - // Rule at src/isa/aarch64/inst.isle line 2032. + // Rule at src/isa/aarch64/inst.isle line 2027. let expr0_0 = OperandSize::Size64; return Some(expr0_0); } if let Some(pattern1_0) = C::fits_in_32(ctx, pattern0_0) { - // Rule at src/isa/aarch64/inst.isle line 2031. + // Rule at src/isa/aarch64/inst.isle line 2026. let expr0_0 = OperandSize::Size32; return Some(expr0_0); } @@ -3393,7 +3393,7 @@ pub fn constructor_trap_if_div_overflow( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 2038. + // Rule at src/isa/aarch64/inst.isle line 2033. let expr0_0 = constructor_adds_op(ctx, pattern0_0)?; let expr1_0 = C::writable_zero_reg(ctx); let expr2_0: u8 = 1; @@ -3437,12 +3437,12 @@ pub fn constructor_trap_if_div_overflow( pub fn constructor_adds_op(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I64 { - // Rule at src/isa/aarch64/inst.isle line 2058. + // Rule at src/isa/aarch64/inst.isle line 2053. let expr0_0 = ALUOp::AddS64; return Some(expr0_0); } if let Some(pattern1_0) = C::fits_in_32(ctx, pattern0_0) { - // Rule at src/isa/aarch64/inst.isle line 2057. + // Rule at src/isa/aarch64/inst.isle line 2052. let expr0_0 = ALUOp::AddS32; return Some(expr0_0); } @@ -3476,7 +3476,7 @@ pub fn constructor_alu_rs_imm_logic_commutative( C::imm_logic_from_imm64(ctx, pattern5_1, pattern7_0) { let pattern9_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 2088. + // Rule at src/isa/aarch64/inst.isle line 2083. let expr0_0 = C::put_in_reg(ctx, pattern9_0); let expr1_0 = constructor_alu_rr_imm_logic(ctx, pattern0_0, expr0_0, pattern8_0)?; @@ -3507,7 +3507,7 @@ pub fn constructor_alu_rs_imm_logic_commutative( C::lshl_from_imm64(ctx, pattern10_1, pattern12_0) { let pattern14_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 2094. + // Rule at src/isa/aarch64/inst.isle line 2089. let expr0_0 = C::put_in_reg(ctx, pattern14_0); let expr1_0 = C::put_in_reg(ctx, pattern7_0); let expr2_0 = constructor_alu_rrr_shift( @@ -3544,7 +3544,7 @@ pub fn constructor_alu_rs_imm_logic_commutative( if let Some(pattern9_0) = C::imm_logic_from_imm64(ctx, pattern6_1, pattern8_0) { - // Rule at src/isa/aarch64/inst.isle line 2086. + // Rule at src/isa/aarch64/inst.isle line 2081. let expr0_0 = C::put_in_reg(ctx, pattern2_0); let expr1_0 = constructor_alu_rr_imm_logic(ctx, pattern0_0, expr0_0, pattern9_0)?; @@ -3574,7 +3574,7 @@ pub fn constructor_alu_rs_imm_logic_commutative( if let Some(pattern14_0) = C::lshl_from_imm64(ctx, pattern11_1, pattern13_0) { - // Rule at src/isa/aarch64/inst.isle line 2092. + // Rule at src/isa/aarch64/inst.isle line 2087. let expr0_0 = C::put_in_reg(ctx, pattern2_0); let expr1_0 = C::put_in_reg(ctx, pattern8_0); let expr2_0 = constructor_alu_rrr_shift( @@ -3595,7 +3595,7 @@ pub fn constructor_alu_rs_imm_logic_commutative( _ => {} } } - // Rule at src/isa/aarch64/inst.isle line 2082. + // Rule at src/isa/aarch64/inst.isle line 2077. let expr0_0 = C::put_in_reg(ctx, pattern2_0); let expr1_0 = C::put_in_reg(ctx, pattern3_0); let expr2_0 = constructor_alu_rrr(ctx, pattern0_0, expr0_0, expr1_0)?; @@ -3629,7 +3629,7 @@ pub fn constructor_alu_rs_imm_logic( if let Some(pattern9_0) = C::imm_logic_from_imm64(ctx, pattern6_1, pattern8_0) { - // Rule at src/isa/aarch64/inst.isle line 2102. + // Rule at src/isa/aarch64/inst.isle line 2097. let expr0_0 = C::put_in_reg(ctx, pattern2_0); let expr1_0 = constructor_alu_rr_imm_logic(ctx, pattern0_0, expr0_0, pattern9_0)?; @@ -3659,7 +3659,7 @@ pub fn constructor_alu_rs_imm_logic( if let Some(pattern14_0) = C::lshl_from_imm64(ctx, pattern11_1, pattern13_0) { - // Rule at src/isa/aarch64/inst.isle line 2104. + // Rule at src/isa/aarch64/inst.isle line 2099. let expr0_0 = C::put_in_reg(ctx, pattern2_0); let expr1_0 = C::put_in_reg(ctx, pattern8_0); let expr2_0 = constructor_alu_rrr_shift( @@ -3680,7 +3680,7 @@ pub fn constructor_alu_rs_imm_logic( _ => {} } } - // Rule at src/isa/aarch64/inst.isle line 2100. + // Rule at src/isa/aarch64/inst.isle line 2095. let expr0_0 = C::put_in_reg(ctx, pattern2_0); let expr1_0 = C::put_in_reg(ctx, pattern3_0); let expr2_0 = constructor_alu_rrr(ctx, pattern0_0, expr0_0, expr1_0)?; @@ -3697,7 +3697,7 @@ pub fn constructor_i128_alu_bitop( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 2112. + // Rule at src/isa/aarch64/inst.isle line 2107. let expr0_0 = C::put_in_regs(ctx, pattern1_0); let expr1_0: usize = 0; let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); diff --git a/cranelift/codegen/src/isa/s390x/inst.isle b/cranelift/codegen/src/isa/s390x/inst.isle index 0e96092ef1bb..f5122148bc0a 100644 --- a/cranelift/codegen/src/isa/s390x/inst.isle +++ b/cranelift/codegen/src/isa/s390x/inst.isle @@ -678,11 +678,7 @@ (type BoxCallInfo (primitive BoxCallInfo)) (type BoxCallIndInfo (primitive BoxCallIndInfo)) -(type MachLabel (primitive MachLabel)) (type BoxJTSequenceInfo (primitive BoxJTSequenceInfo)) -(type BoxExternalName (primitive BoxExternalName)) -(type ValueLabel (primitive ValueLabel)) -(type UnwindInst (primitive UnwindInst)) ;; An ALU operation. (type ALUOp @@ -1041,10 +1037,6 @@ ;; Helpers for machine label vectors ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;; VecMachLabel needs to be passed by reference, so it cannot be -;; declared as primitive type. Declare as extern enum instead. -(type VecMachLabel extern (enum)) - (decl vec_length_minus1 (VecMachLabel) u32) (extern constructor vec_length_minus1 vec_length_minus1) @@ -2963,5 +2955,3 @@ (decl fcmp_reg (Type Reg Reg) ProducesFlags) (rule (fcmp_reg $F32 src1 src2) (fpu_cmp32 src1 src2)) (rule (fcmp_reg $F64 src1 src2) (fpu_cmp64 src1 src2)) - - diff --git a/cranelift/codegen/src/isa/s390x/lower/isle/generated_code.manifest b/cranelift/codegen/src/isa/s390x/lower/isle/generated_code.manifest index 9cbb4eaa96fa..eee9fc5608a3 100644 --- a/cranelift/codegen/src/isa/s390x/lower/isle/generated_code.manifest +++ b/cranelift/codegen/src/isa/s390x/lower/isle/generated_code.manifest @@ -1,4 +1,4 @@ src/clif.isle 9ea75a6f790b5c03 -src/prelude.isle 2bfcafbef6b29358 -src/isa/s390x/inst.isle 1d525c87f7c77c26 +src/prelude.isle 6aaf8ce0f5a5c2ec +src/isa/s390x/inst.isle f5af3708848ef1aa src/isa/s390x/lower.isle 57dcc39cbab2d1c6 diff --git a/cranelift/codegen/src/isa/s390x/lower/isle/generated_code.rs b/cranelift/codegen/src/isa/s390x/lower/isle/generated_code.rs index 96b59f69b2d5..2c52f5f2b96f 100644 --- a/cranelift/codegen/src/isa/s390x/lower/isle/generated_code.rs +++ b/cranelift/codegen/src/isa/s390x/lower/isle/generated_code.rs @@ -127,19 +127,19 @@ pub trait Context { fn sink_inst(&mut self, arg0: Inst) -> Unit; } -/// Internal type SideEffectNoResult: defined at src/prelude.isle line 295. +/// Internal type SideEffectNoResult: defined at src/prelude.isle line 307. #[derive(Clone, Debug)] pub enum SideEffectNoResult { Inst { inst: MInst }, } -/// Internal type ProducesFlags: defined at src/prelude.isle line 314. +/// Internal type ProducesFlags: defined at src/prelude.isle line 326. #[derive(Clone, Debug)] pub enum ProducesFlags { ProducesFlags { inst: MInst, result: Reg }, } -/// Internal type ConsumesFlags: defined at src/prelude.isle line 317. +/// Internal type ConsumesFlags: defined at src/prelude.isle line 329. #[derive(Clone, Debug)] pub enum ConsumesFlags { ConsumesFlags { inst: MInst, result: Reg }, @@ -652,8 +652,8 @@ pub enum MInst { }, } -/// Internal type ALUOp: defined at src/isa/s390x/inst.isle line 688. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type ALUOp: defined at src/isa/s390x/inst.isle line 684. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum ALUOp { Add32, Add32Ext16, @@ -690,8 +690,8 @@ pub enum ALUOp { XorNot64, } -/// Internal type UnaryOp: defined at src/isa/s390x/inst.isle line 729. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type UnaryOp: defined at src/isa/s390x/inst.isle line 725. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum UnaryOp { Abs32, Abs64, @@ -703,8 +703,8 @@ pub enum UnaryOp { PopcntReg, } -/// Internal type ShiftOp: defined at src/isa/s390x/inst.isle line 742. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type ShiftOp: defined at src/isa/s390x/inst.isle line 738. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum ShiftOp { RotL32, RotL64, @@ -716,8 +716,8 @@ pub enum ShiftOp { AShR64, } -/// Internal type CmpOp: defined at src/isa/s390x/inst.isle line 755. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type CmpOp: defined at src/isa/s390x/inst.isle line 751. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum CmpOp { CmpS32, CmpS32Ext16, @@ -731,8 +731,8 @@ pub enum CmpOp { CmpL64Ext32, } -/// Internal type FPUOp1: defined at src/isa/s390x/inst.isle line 770. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type FPUOp1: defined at src/isa/s390x/inst.isle line 766. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum FPUOp1 { Abs32, Abs64, @@ -746,8 +746,8 @@ pub enum FPUOp1 { Cvt64To32, } -/// Internal type FPUOp2: defined at src/isa/s390x/inst.isle line 785. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type FPUOp2: defined at src/isa/s390x/inst.isle line 781. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum FPUOp2 { Add32, Add64, @@ -763,8 +763,8 @@ pub enum FPUOp2 { Min64, } -/// Internal type FPUOp3: defined at src/isa/s390x/inst.isle line 802. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type FPUOp3: defined at src/isa/s390x/inst.isle line 798. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum FPUOp3 { MAdd32, MAdd64, @@ -772,8 +772,8 @@ pub enum FPUOp3 { MSub64, } -/// Internal type FpuToIntOp: defined at src/isa/s390x/inst.isle line 811. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type FpuToIntOp: defined at src/isa/s390x/inst.isle line 807. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum FpuToIntOp { F32ToU32, F32ToI32, @@ -785,8 +785,8 @@ pub enum FpuToIntOp { F64ToI64, } -/// Internal type IntToFpuOp: defined at src/isa/s390x/inst.isle line 824. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type IntToFpuOp: defined at src/isa/s390x/inst.isle line 820. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum IntToFpuOp { U32ToF32, I32ToF32, @@ -798,8 +798,8 @@ pub enum IntToFpuOp { I64ToF64, } -/// Internal type FpuRoundMode: defined at src/isa/s390x/inst.isle line 838. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type FpuRoundMode: defined at src/isa/s390x/inst.isle line 834. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum FpuRoundMode { Minus32, Minus64, @@ -811,19 +811,19 @@ pub enum FpuRoundMode { Nearest64, } -/// Internal type WritableRegPair: defined at src/isa/s390x/inst.isle line 1248. +/// Internal type WritableRegPair: defined at src/isa/s390x/inst.isle line 1240. #[derive(Clone, Debug)] pub enum WritableRegPair { WritableRegPair { hi: WritableReg, lo: WritableReg }, } -/// Internal type RegPair: defined at src/isa/s390x/inst.isle line 1270. +/// Internal type RegPair: defined at src/isa/s390x/inst.isle line 1262. #[derive(Clone, Debug)] pub enum RegPair { RegPair { hi: Reg, lo: Reg }, } -/// Internal type ProducesBool: defined at src/isa/s390x/inst.isle line 2195. +/// Internal type ProducesBool: defined at src/isa/s390x/inst.isle line 2187. #[derive(Clone, Debug)] pub enum ProducesBool { ProducesBool { producer: ProducesFlags, cond: Cond }, @@ -832,7 +832,7 @@ pub enum ProducesBool { // Generated as internal constructor for term temp_reg. pub fn constructor_temp_reg(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; - // Rule at src/prelude.isle line 70. + // Rule at src/prelude.isle line 73. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); return Some(expr1_0); @@ -841,7 +841,7 @@ pub fn constructor_temp_reg(ctx: &mut C, arg0: Type) -> Option // Generated as internal constructor for term lo_reg. pub fn constructor_lo_reg(ctx: &mut C, arg0: Value) -> Option { let pattern0_0 = arg0; - // Rule at src/prelude.isle line 105. + // Rule at src/prelude.isle line 108. let expr0_0 = C::put_in_regs(ctx, pattern0_0); let expr1_0: usize = 0; let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); @@ -858,7 +858,7 @@ pub fn constructor_value_regs_none( inst: ref pattern1_0, } = pattern0_0 { - // Rule at src/prelude.isle line 300. + // Rule at src/prelude.isle line 312. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::value_regs_invalid(ctx); return Some(expr1_0); @@ -876,7 +876,7 @@ pub fn constructor_safepoint( inst: ref pattern1_0, } = pattern0_0 { - // Rule at src/prelude.isle line 306. + // Rule at src/prelude.isle line 318. let expr0_0 = C::emit_safepoint(ctx, &pattern1_0); let expr1_0 = C::value_regs_invalid(ctx); return Some(expr1_0); @@ -902,7 +902,7 @@ pub fn constructor_with_flags( result: pattern3_1, } = pattern2_0 { - // Rule at src/prelude.isle line 327. + // Rule at src/prelude.isle line 339. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern3_0); let expr2_0 = C::value_regs(ctx, pattern1_1, pattern3_1); @@ -930,7 +930,7 @@ pub fn constructor_with_flags_1( result: pattern3_1, } = pattern2_0 { - // Rule at src/prelude.isle line 335. + // Rule at src/prelude.isle line 347. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern3_0); return Some(pattern3_1); @@ -964,7 +964,7 @@ pub fn constructor_with_flags_2( result: pattern5_1, } = pattern4_0 { - // Rule at src/prelude.isle line 345. + // Rule at src/prelude.isle line 357. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern5_0); let expr2_0 = C::emit(ctx, &pattern3_0); @@ -981,7 +981,7 @@ pub fn constructor_mask_amt_reg(ctx: &mut C, arg0: Type, arg1: Reg) let pattern0_0 = arg0; if let Some(pattern1_0) = C::gpr32_ty(ctx, pattern0_0) { let pattern2_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1013. + // Rule at src/isa/s390x/inst.isle line 1009. let expr0_0: i64 = -1; let expr1_0 = C::mask_amt_imm(ctx, pattern1_0, expr0_0); let expr2_0 = C::u8_as_u16(ctx, expr1_0); @@ -992,7 +992,7 @@ pub fn constructor_mask_amt_reg(ctx: &mut C, arg0: Type, arg1: Reg) } if let Some(pattern1_0) = C::gpr64_ty(ctx, pattern0_0) { let pattern2_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1016. + // Rule at src/isa/s390x/inst.isle line 1012. return Some(pattern2_0); } return None; @@ -1019,7 +1019,7 @@ pub fn constructor_lower_address( let pattern7_0 = arg2; let pattern8_0 = C::i64_from_offset(ctx, pattern7_0); if pattern8_0 == 0 { - // Rule at src/isa/s390x/inst.isle line 1112. + // Rule at src/isa/s390x/inst.isle line 1104. let expr0_0 = C::put_in_reg(ctx, pattern6_0); let expr1_0 = C::put_in_reg(ctx, pattern6_1); let expr2_0 = C::memarg_reg_plus_reg(ctx, expr0_0, expr1_0, pattern0_0); @@ -1038,7 +1038,7 @@ pub fn constructor_lower_address( if let Some(pattern8_0) = C::memarg_symbol_offset_sum(ctx, pattern6_0, pattern7_0) { - // Rule at src/isa/s390x/inst.isle line 1115. + // Rule at src/isa/s390x/inst.isle line 1107. let expr0_0 = C::memarg_symbol(ctx, pattern3_0, pattern8_0, pattern0_0); return Some(expr0_0); } @@ -1048,7 +1048,7 @@ pub fn constructor_lower_address( } let pattern2_0 = arg2; let pattern3_0 = C::i64_from_offset(ctx, pattern2_0); - // Rule at src/isa/s390x/inst.isle line 1109. + // Rule at src/isa/s390x/inst.isle line 1101. let expr0_0 = C::put_in_reg(ctx, pattern1_0); let expr1_0 = C::memarg_reg_plus_off(ctx, expr0_0, pattern3_0, pattern0_0); return Some(expr1_0); @@ -1064,7 +1064,7 @@ pub fn constructor_stack_addr_impl( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1142. + // Rule at src/isa/s390x/inst.isle line 1134. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::abi_stackslot_addr(ctx, expr0_0, pattern1_0, pattern2_0); let expr2_0 = C::emit(ctx, &expr1_0); @@ -1084,7 +1084,7 @@ pub fn constructor_sink_load(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C) -> Option { - // Rule at src/isa/s390x/inst.isle line 1253. + // Rule at src/isa/s390x/inst.isle line 1245. let expr0_0: u8 = 0; let expr1_0 = C::writable_gpr(ctx, expr0_0); let expr2_0: u8 = 1; @@ -1197,7 +1197,7 @@ pub fn constructor_copy_writable_regpair( arg0: &RegPair, ) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/s390x/inst.isle line 1259. + // Rule at src/isa/s390x/inst.isle line 1251. let expr0_0 = constructor_temp_writable_regpair(ctx)?; return Some(expr0_0); } @@ -1213,7 +1213,7 @@ pub fn constructor_writable_regpair_hi( lo: pattern1_1, } = pattern0_0 { - // Rule at src/isa/s390x/inst.isle line 1263. + // Rule at src/isa/s390x/inst.isle line 1255. return Some(pattern1_0); } return None; @@ -1230,7 +1230,7 @@ pub fn constructor_writable_regpair_lo( lo: pattern1_1, } = pattern0_0 { - // Rule at src/isa/s390x/inst.isle line 1267. + // Rule at src/isa/s390x/inst.isle line 1259. return Some(pattern1_1); } return None; @@ -1247,7 +1247,7 @@ pub fn constructor_writable_regpair_to_regpair( lo: pattern1_1, } = pattern0_0 { - // Rule at src/isa/s390x/inst.isle line 1274. + // Rule at src/isa/s390x/inst.isle line 1266. let expr0_0 = C::writable_reg_to_reg(ctx, pattern1_0); let expr1_0 = C::writable_reg_to_reg(ctx, pattern1_1); let expr2_0 = RegPair::RegPair { @@ -1261,7 +1261,7 @@ pub fn constructor_writable_regpair_to_regpair( // Generated as internal constructor for term uninitialized_regpair. pub fn constructor_uninitialized_regpair(ctx: &mut C) -> Option { - // Rule at src/isa/s390x/inst.isle line 1279. + // Rule at src/isa/s390x/inst.isle line 1271. let expr0_0 = constructor_temp_writable_regpair(ctx)?; let expr1_0 = constructor_writable_regpair_to_regpair(ctx, &expr0_0)?; return Some(expr1_0); @@ -1275,7 +1275,7 @@ pub fn constructor_regpair_hi(ctx: &mut C, arg0: &RegPair) -> Option lo: pattern1_1, } = pattern0_0 { - // Rule at src/isa/s390x/inst.isle line 1284. + // Rule at src/isa/s390x/inst.isle line 1276. return Some(pattern1_0); } return None; @@ -1289,7 +1289,7 @@ pub fn constructor_regpair_lo(ctx: &mut C, arg0: &RegPair) -> Option lo: pattern1_1, } = pattern0_0 { - // Rule at src/isa/s390x/inst.isle line 1288. + // Rule at src/isa/s390x/inst.isle line 1280. return Some(pattern1_1); } return None; @@ -1307,7 +1307,7 @@ pub fn constructor_alu_rrr( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 1295. + // Rule at src/isa/s390x/inst.isle line 1287. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::AluRRR { alu_op: pattern1_0.clone(), @@ -1332,7 +1332,7 @@ pub fn constructor_alu_rrsimm16( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 1302. + // Rule at src/isa/s390x/inst.isle line 1294. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::AluRRSImm16 { alu_op: pattern1_0.clone(), @@ -1357,7 +1357,7 @@ pub fn constructor_alu_rr( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 1309. + // Rule at src/isa/s390x/inst.isle line 1301. let expr0_0 = constructor_copy_writable_reg(ctx, pattern0_0, pattern2_0)?; let expr1_0 = MInst::AluRR { alu_op: pattern1_0.clone(), @@ -1381,7 +1381,7 @@ pub fn constructor_alu_rx( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 1316. + // Rule at src/isa/s390x/inst.isle line 1308. let expr0_0 = constructor_copy_writable_reg(ctx, pattern0_0, pattern2_0)?; let expr1_0 = MInst::AluRX { alu_op: pattern1_0.clone(), @@ -1405,7 +1405,7 @@ pub fn constructor_alu_rsimm16( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 1323. + // Rule at src/isa/s390x/inst.isle line 1315. let expr0_0 = constructor_copy_writable_reg(ctx, pattern0_0, pattern2_0)?; let expr1_0 = MInst::AluRSImm16 { alu_op: pattern1_0.clone(), @@ -1429,7 +1429,7 @@ pub fn constructor_alu_rsimm32( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 1330. + // Rule at src/isa/s390x/inst.isle line 1322. let expr0_0 = constructor_copy_writable_reg(ctx, pattern0_0, pattern2_0)?; let expr1_0 = MInst::AluRSImm32 { alu_op: pattern1_0.clone(), @@ -1453,7 +1453,7 @@ pub fn constructor_alu_ruimm32( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 1337. + // Rule at src/isa/s390x/inst.isle line 1329. let expr0_0 = constructor_copy_writable_reg(ctx, pattern0_0, pattern2_0)?; let expr1_0 = MInst::AluRUImm32 { alu_op: pattern1_0.clone(), @@ -1477,7 +1477,7 @@ pub fn constructor_alu_ruimm16shifted( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 1344. + // Rule at src/isa/s390x/inst.isle line 1336. let expr0_0 = constructor_copy_writable_reg(ctx, pattern0_0, pattern2_0)?; let expr1_0 = MInst::AluRUImm16Shifted { alu_op: pattern1_0.clone(), @@ -1501,7 +1501,7 @@ pub fn constructor_alu_ruimm32shifted( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 1351. + // Rule at src/isa/s390x/inst.isle line 1343. let expr0_0 = constructor_copy_writable_reg(ctx, pattern0_0, pattern2_0)?; let expr1_0 = MInst::AluRUImm32Shifted { alu_op: pattern1_0.clone(), @@ -1517,7 +1517,7 @@ pub fn constructor_alu_ruimm32shifted( pub fn constructor_smul_wide(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1358. + // Rule at src/isa/s390x/inst.isle line 1350. let expr0_0 = constructor_temp_writable_regpair(ctx)?; let expr1_0 = MInst::SMulWide { rn: pattern0_0, @@ -1532,7 +1532,7 @@ pub fn constructor_smul_wide(ctx: &mut C, arg0: Reg, arg1: Reg) -> O pub fn constructor_umul_wide(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1365. + // Rule at src/isa/s390x/inst.isle line 1357. let expr0_0 = constructor_temp_writable_regpair(ctx)?; let expr1_0 = constructor_writable_regpair_lo(ctx, &expr0_0)?; let expr2_0 = MInst::Mov64 { @@ -1554,7 +1554,7 @@ pub fn constructor_sdivmod32( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1373. + // Rule at src/isa/s390x/inst.isle line 1365. let expr0_0 = constructor_copy_writable_regpair(ctx, pattern0_0)?; let expr1_0 = MInst::SDivMod32 { rn: pattern1_0 }; let expr2_0 = C::emit(ctx, &expr1_0); @@ -1570,7 +1570,7 @@ pub fn constructor_sdivmod64( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1380. + // Rule at src/isa/s390x/inst.isle line 1372. let expr0_0 = constructor_copy_writable_regpair(ctx, pattern0_0)?; let expr1_0 = MInst::SDivMod64 { rn: pattern1_0 }; let expr2_0 = C::emit(ctx, &expr1_0); @@ -1586,7 +1586,7 @@ pub fn constructor_udivmod32( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1387. + // Rule at src/isa/s390x/inst.isle line 1379. let expr0_0 = constructor_copy_writable_regpair(ctx, pattern0_0)?; let expr1_0 = MInst::UDivMod32 { rn: pattern1_0 }; let expr2_0 = C::emit(ctx, &expr1_0); @@ -1602,7 +1602,7 @@ pub fn constructor_udivmod64( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1394. + // Rule at src/isa/s390x/inst.isle line 1386. let expr0_0 = constructor_copy_writable_regpair(ctx, pattern0_0)?; let expr1_0 = MInst::UDivMod64 { rn: pattern1_0 }; let expr2_0 = C::emit(ctx, &expr1_0); @@ -1624,7 +1624,7 @@ pub fn constructor_shift_rr( let pattern2_0 = arg2; let pattern3_0 = arg3; let pattern4_0 = arg4; - // Rule at src/isa/s390x/inst.isle line 1401. + // Rule at src/isa/s390x/inst.isle line 1393. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::ShiftRR { shift_op: pattern1_0.clone(), @@ -1648,7 +1648,7 @@ pub fn constructor_unary_rr( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1408. + // Rule at src/isa/s390x/inst.isle line 1400. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::UnaryRR { op: pattern1_0.clone(), @@ -1670,7 +1670,7 @@ pub fn constructor_cmp_rr( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1415. + // Rule at src/isa/s390x/inst.isle line 1407. let expr0_0 = MInst::CmpRR { op: pattern0_0.clone(), rn: pattern1_0, @@ -1694,7 +1694,7 @@ pub fn constructor_cmp_rx( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1421. + // Rule at src/isa/s390x/inst.isle line 1413. let expr0_0 = MInst::CmpRX { op: pattern0_0.clone(), rn: pattern1_0, @@ -1718,7 +1718,7 @@ pub fn constructor_cmp_rsimm16( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1427. + // Rule at src/isa/s390x/inst.isle line 1419. let expr0_0 = MInst::CmpRSImm16 { op: pattern0_0.clone(), rn: pattern1_0, @@ -1742,7 +1742,7 @@ pub fn constructor_cmp_rsimm32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1433. + // Rule at src/isa/s390x/inst.isle line 1425. let expr0_0 = MInst::CmpRSImm32 { op: pattern0_0.clone(), rn: pattern1_0, @@ -1766,7 +1766,7 @@ pub fn constructor_cmp_ruimm32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1439. + // Rule at src/isa/s390x/inst.isle line 1431. let expr0_0 = MInst::CmpRUImm32 { op: pattern0_0.clone(), rn: pattern1_0, @@ -1792,7 +1792,7 @@ pub fn constructor_atomic_rmw_impl( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 1445. + // Rule at src/isa/s390x/inst.isle line 1437. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::AtomicRmw { alu_op: pattern1_0.clone(), @@ -1815,7 +1815,7 @@ pub fn constructor_atomic_cas32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1452. + // Rule at src/isa/s390x/inst.isle line 1444. let expr0_0: Type = I32; let expr1_0 = constructor_copy_writable_reg(ctx, expr0_0, pattern0_0)?; let expr2_0 = MInst::AtomicCas32 { @@ -1838,7 +1838,7 @@ pub fn constructor_atomic_cas64( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1459. + // Rule at src/isa/s390x/inst.isle line 1451. let expr0_0: Type = I64; let expr1_0 = constructor_copy_writable_reg(ctx, expr0_0, pattern0_0)?; let expr2_0 = MInst::AtomicCas64 { @@ -1853,7 +1853,7 @@ pub fn constructor_atomic_cas64( // Generated as internal constructor for term fence_impl. pub fn constructor_fence_impl(ctx: &mut C) -> Option { - // Rule at src/isa/s390x/inst.isle line 1466. + // Rule at src/isa/s390x/inst.isle line 1458. let expr0_0 = MInst::Fence; let expr1_0 = SideEffectNoResult::Inst { inst: expr0_0 }; return Some(expr1_0); @@ -1862,7 +1862,7 @@ pub fn constructor_fence_impl(ctx: &mut C) -> Option(ctx: &mut C, arg0: &MemArg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/s390x/inst.isle line 1471. + // Rule at src/isa/s390x/inst.isle line 1463. let expr0_0: Type = I32; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::Load32 { @@ -1877,7 +1877,7 @@ pub fn constructor_load32(ctx: &mut C, arg0: &MemArg) -> Option // Generated as internal constructor for term load64. pub fn constructor_load64(ctx: &mut C, arg0: &MemArg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/s390x/inst.isle line 1478. + // Rule at src/isa/s390x/inst.isle line 1470. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::Load64 { @@ -1892,7 +1892,7 @@ pub fn constructor_load64(ctx: &mut C, arg0: &MemArg) -> Option // Generated as internal constructor for term loadrev16. pub fn constructor_loadrev16(ctx: &mut C, arg0: &MemArg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/s390x/inst.isle line 1485. + // Rule at src/isa/s390x/inst.isle line 1477. let expr0_0: Type = I32; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::LoadRev16 { @@ -1907,7 +1907,7 @@ pub fn constructor_loadrev16(ctx: &mut C, arg0: &MemArg) -> Option(ctx: &mut C, arg0: &MemArg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/s390x/inst.isle line 1492. + // Rule at src/isa/s390x/inst.isle line 1484. let expr0_0: Type = I32; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::LoadRev32 { @@ -1922,7 +1922,7 @@ pub fn constructor_loadrev32(ctx: &mut C, arg0: &MemArg) -> Option(ctx: &mut C, arg0: &MemArg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/s390x/inst.isle line 1499. + // Rule at src/isa/s390x/inst.isle line 1491. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::LoadRev64 { @@ -1942,7 +1942,7 @@ pub fn constructor_store8( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1506. + // Rule at src/isa/s390x/inst.isle line 1498. let expr0_0 = MInst::Store8 { rd: pattern0_0, mem: pattern1_0.clone(), @@ -1959,7 +1959,7 @@ pub fn constructor_store16( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1511. + // Rule at src/isa/s390x/inst.isle line 1503. let expr0_0 = MInst::Store16 { rd: pattern0_0, mem: pattern1_0.clone(), @@ -1976,7 +1976,7 @@ pub fn constructor_store32( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1516. + // Rule at src/isa/s390x/inst.isle line 1508. let expr0_0 = MInst::Store32 { rd: pattern0_0, mem: pattern1_0.clone(), @@ -1993,7 +1993,7 @@ pub fn constructor_store64( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1521. + // Rule at src/isa/s390x/inst.isle line 1513. let expr0_0 = MInst::Store64 { rd: pattern0_0, mem: pattern1_0.clone(), @@ -2010,7 +2010,7 @@ pub fn constructor_store8_imm( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1526. + // Rule at src/isa/s390x/inst.isle line 1518. let expr0_0 = MInst::StoreImm8 { imm: pattern0_0, mem: pattern1_0.clone(), @@ -2027,7 +2027,7 @@ pub fn constructor_store16_imm( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1531. + // Rule at src/isa/s390x/inst.isle line 1523. let expr0_0 = MInst::StoreImm16 { imm: pattern0_0, mem: pattern1_0.clone(), @@ -2044,7 +2044,7 @@ pub fn constructor_store32_simm16( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1536. + // Rule at src/isa/s390x/inst.isle line 1528. let expr0_0 = MInst::StoreImm32SExt16 { imm: pattern0_0, mem: pattern1_0.clone(), @@ -2061,7 +2061,7 @@ pub fn constructor_store64_simm16( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1541. + // Rule at src/isa/s390x/inst.isle line 1533. let expr0_0 = MInst::StoreImm64SExt16 { imm: pattern0_0, mem: pattern1_0.clone(), @@ -2078,7 +2078,7 @@ pub fn constructor_storerev16( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1546. + // Rule at src/isa/s390x/inst.isle line 1538. let expr0_0 = MInst::StoreRev16 { rd: pattern0_0, mem: pattern1_0.clone(), @@ -2095,7 +2095,7 @@ pub fn constructor_storerev32( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1551. + // Rule at src/isa/s390x/inst.isle line 1543. let expr0_0 = MInst::StoreRev32 { rd: pattern0_0, mem: pattern1_0.clone(), @@ -2112,7 +2112,7 @@ pub fn constructor_storerev64( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1556. + // Rule at src/isa/s390x/inst.isle line 1548. let expr0_0 = MInst::StoreRev64 { rd: pattern0_0, mem: pattern1_0.clone(), @@ -2131,7 +2131,7 @@ pub fn constructor_fpu_rr( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1561. + // Rule at src/isa/s390x/inst.isle line 1553. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::FpuRR { fpu_op: pattern1_0.clone(), @@ -2155,7 +2155,7 @@ pub fn constructor_fpu_rrr( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 1568. + // Rule at src/isa/s390x/inst.isle line 1560. let expr0_0 = constructor_copy_writable_reg(ctx, pattern0_0, pattern2_0)?; let expr1_0 = MInst::FpuRRR { fpu_op: pattern1_0.clone(), @@ -2181,7 +2181,7 @@ pub fn constructor_fpu_rrrr( let pattern2_0 = arg2; let pattern3_0 = arg3; let pattern4_0 = arg4; - // Rule at src/isa/s390x/inst.isle line 1575. + // Rule at src/isa/s390x/inst.isle line 1567. let expr0_0 = constructor_copy_writable_reg(ctx, pattern0_0, pattern2_0)?; let expr1_0 = MInst::FpuRRRR { fpu_op: pattern1_0.clone(), @@ -2204,7 +2204,7 @@ pub fn constructor_fpu_copysign( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1582. + // Rule at src/isa/s390x/inst.isle line 1574. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::FpuCopysign { rd: expr0_0, @@ -2224,7 +2224,7 @@ pub fn constructor_fpu_cmp32( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1589. + // Rule at src/isa/s390x/inst.isle line 1581. let expr0_0 = MInst::FpuCmp32 { rn: pattern0_0, rm: pattern1_0, @@ -2245,7 +2245,7 @@ pub fn constructor_fpu_cmp64( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1595. + // Rule at src/isa/s390x/inst.isle line 1587. let expr0_0 = MInst::FpuCmp64 { rn: pattern0_0, rm: pattern1_0, @@ -2268,7 +2268,7 @@ pub fn constructor_fpu_to_int( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1601. + // Rule at src/isa/s390x/inst.isle line 1593. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::FpuToInt { op: pattern1_0.clone(), @@ -2293,7 +2293,7 @@ pub fn constructor_int_to_fpu( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1608. + // Rule at src/isa/s390x/inst.isle line 1600. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::IntToFpu { op: pattern1_0.clone(), @@ -2315,7 +2315,7 @@ pub fn constructor_fpu_round( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1615. + // Rule at src/isa/s390x/inst.isle line 1607. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::FpuRound { op: pattern1_0.clone(), @@ -2339,7 +2339,7 @@ pub fn constructor_fpuvec_rrr( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 1622. + // Rule at src/isa/s390x/inst.isle line 1614. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::FpuVecRRR { fpu_op: pattern1_0.clone(), @@ -2355,7 +2355,7 @@ pub fn constructor_fpuvec_rrr( // Generated as internal constructor for term mov_to_fpr. pub fn constructor_mov_to_fpr(ctx: &mut C, arg0: Reg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/s390x/inst.isle line 1629. + // Rule at src/isa/s390x/inst.isle line 1621. let expr0_0: Type = F64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::MovToFpr { @@ -2370,7 +2370,7 @@ pub fn constructor_mov_to_fpr(ctx: &mut C, arg0: Reg) -> Option // Generated as internal constructor for term mov_from_fpr. pub fn constructor_mov_from_fpr(ctx: &mut C, arg0: Reg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/s390x/inst.isle line 1636. + // Rule at src/isa/s390x/inst.isle line 1628. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::MovFromFpr { @@ -2385,7 +2385,7 @@ pub fn constructor_mov_from_fpr(ctx: &mut C, arg0: Reg) -> Option(ctx: &mut C, arg0: &MemArg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/s390x/inst.isle line 1643. + // Rule at src/isa/s390x/inst.isle line 1635. let expr0_0: Type = F32; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::FpuLoad32 { @@ -2400,7 +2400,7 @@ pub fn constructor_fpu_load32(ctx: &mut C, arg0: &MemArg) -> Option< // Generated as internal constructor for term fpu_load64. pub fn constructor_fpu_load64(ctx: &mut C, arg0: &MemArg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/s390x/inst.isle line 1650. + // Rule at src/isa/s390x/inst.isle line 1642. let expr0_0: Type = F64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::FpuLoad64 { @@ -2415,7 +2415,7 @@ pub fn constructor_fpu_load64(ctx: &mut C, arg0: &MemArg) -> Option< // Generated as internal constructor for term fpu_loadrev32. pub fn constructor_fpu_loadrev32(ctx: &mut C, arg0: &MemArg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/s390x/inst.isle line 1657. + // Rule at src/isa/s390x/inst.isle line 1649. let expr0_0: Type = F32; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::FpuLoadRev32 { @@ -2430,7 +2430,7 @@ pub fn constructor_fpu_loadrev32(ctx: &mut C, arg0: &MemArg) -> Opti // Generated as internal constructor for term fpu_loadrev64. pub fn constructor_fpu_loadrev64(ctx: &mut C, arg0: &MemArg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/s390x/inst.isle line 1664. + // Rule at src/isa/s390x/inst.isle line 1656. let expr0_0: Type = F64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::FpuLoadRev64 { @@ -2450,7 +2450,7 @@ pub fn constructor_fpu_store32( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1671. + // Rule at src/isa/s390x/inst.isle line 1663. let expr0_0 = MInst::FpuStore32 { rd: pattern0_0, mem: pattern1_0.clone(), @@ -2467,7 +2467,7 @@ pub fn constructor_fpu_store64( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1676. + // Rule at src/isa/s390x/inst.isle line 1668. let expr0_0 = MInst::FpuStore64 { rd: pattern0_0, mem: pattern1_0.clone(), @@ -2484,7 +2484,7 @@ pub fn constructor_fpu_storerev32( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1681. + // Rule at src/isa/s390x/inst.isle line 1673. let expr0_0 = MInst::FpuStoreRev32 { rd: pattern0_0, mem: pattern1_0.clone(), @@ -2501,7 +2501,7 @@ pub fn constructor_fpu_storerev64( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1686. + // Rule at src/isa/s390x/inst.isle line 1678. let expr0_0 = MInst::FpuStoreRev64 { rd: pattern0_0, mem: pattern1_0.clone(), @@ -2518,7 +2518,7 @@ pub fn constructor_load_ext_name_far( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1691. + // Rule at src/isa/s390x/inst.isle line 1683. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::LoadExtNameFar { @@ -2534,7 +2534,7 @@ pub fn constructor_load_ext_name_far( // Generated as internal constructor for term load_addr. pub fn constructor_load_addr(ctx: &mut C, arg0: &MemArg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/s390x/inst.isle line 1698. + // Rule at src/isa/s390x/inst.isle line 1690. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::LoadAddr { @@ -2552,7 +2552,7 @@ pub fn constructor_jump_impl( arg0: MachLabel, ) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/s390x/inst.isle line 1705. + // Rule at src/isa/s390x/inst.isle line 1697. let expr0_0 = MInst::Jump { dest: pattern0_0 }; let expr1_0 = SideEffectNoResult::Inst { inst: expr0_0 }; return Some(expr1_0); @@ -2568,7 +2568,7 @@ pub fn constructor_cond_br( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1710. + // Rule at src/isa/s390x/inst.isle line 1702. let expr0_0 = MInst::CondBr { taken: pattern0_0, not_taken: pattern1_0, @@ -2586,7 +2586,7 @@ pub fn constructor_oneway_cond_br( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1715. + // Rule at src/isa/s390x/inst.isle line 1707. let expr0_0 = MInst::OneWayCondBr { target: pattern0_0, cond: pattern1_0.clone(), @@ -2603,7 +2603,7 @@ pub fn constructor_jt_sequence( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1720. + // Rule at src/isa/s390x/inst.isle line 1712. let expr0_0 = MInst::JTSequence { ridx: pattern0_0, targets: pattern1_0.clone(), @@ -2620,7 +2620,7 @@ pub fn constructor_drop_flags(ctx: &mut C, arg0: &ProducesFlags) -> result: pattern1_1, } = pattern0_0 { - // Rule at src/isa/s390x/inst.isle line 1725. + // Rule at src/isa/s390x/inst.isle line 1717. let expr0_0 = C::emit(ctx, &pattern1_0); return Some(pattern1_1); } @@ -2638,7 +2638,7 @@ pub fn constructor_emit_mov( if pattern0_0 == F32 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1741. + // Rule at src/isa/s390x/inst.isle line 1733. let expr0_0 = MInst::FpuMove32 { rd: pattern2_0, rn: pattern3_0, @@ -2649,7 +2649,7 @@ pub fn constructor_emit_mov( if pattern0_0 == F64 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1744. + // Rule at src/isa/s390x/inst.isle line 1736. let expr0_0 = MInst::FpuMove64 { rd: pattern2_0, rn: pattern3_0, @@ -2660,7 +2660,7 @@ pub fn constructor_emit_mov( if let Some(pattern1_0) = C::gpr32_ty(ctx, pattern0_0) { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1735. + // Rule at src/isa/s390x/inst.isle line 1727. let expr0_0 = MInst::Mov32 { rd: pattern2_0, rm: pattern3_0, @@ -2671,7 +2671,7 @@ pub fn constructor_emit_mov( if let Some(pattern1_0) = C::gpr64_ty(ctx, pattern0_0) { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1738. + // Rule at src/isa/s390x/inst.isle line 1730. let expr0_0 = MInst::Mov64 { rd: pattern2_0, rm: pattern3_0, @@ -2690,7 +2690,7 @@ pub fn constructor_copy_writable_reg( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1749. + // Rule at src/isa/s390x/inst.isle line 1741. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = constructor_emit_mov(ctx, pattern0_0, expr0_0, pattern1_0)?; return Some(expr0_0); @@ -2700,7 +2700,7 @@ pub fn constructor_copy_writable_reg( pub fn constructor_copy_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1756. + // Rule at src/isa/s390x/inst.isle line 1748. let expr0_0 = constructor_copy_writable_reg(ctx, pattern0_0, pattern1_0)?; let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); return Some(expr1_0); @@ -2717,7 +2717,7 @@ pub fn constructor_emit_imm( if pattern0_0 == F32 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1811. + // Rule at src/isa/s390x/inst.isle line 1803. let expr0_0 = C::u64_as_u32(ctx, pattern3_0); let expr1_0 = MInst::LoadFpuConst32 { rd: pattern2_0, @@ -2729,7 +2729,7 @@ pub fn constructor_emit_imm( if pattern0_0 == F64 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1816. + // Rule at src/isa/s390x/inst.isle line 1808. let expr0_0 = MInst::LoadFpuConst64 { rd: pattern2_0, const_data: pattern3_0, @@ -2740,7 +2740,7 @@ pub fn constructor_emit_imm( if let Some(pattern1_0) = C::fits_in_16(ctx, pattern0_0) { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1765. + // Rule at src/isa/s390x/inst.isle line 1757. let expr0_0 = C::u64_as_i16(ctx, pattern3_0); let expr1_0 = MInst::Mov32SImm16 { rd: pattern2_0, @@ -2753,7 +2753,7 @@ pub fn constructor_emit_imm( let pattern2_0 = arg1; let pattern3_0 = arg2; if let Some(pattern4_0) = C::i16_from_u64(ctx, pattern3_0) { - // Rule at src/isa/s390x/inst.isle line 1769. + // Rule at src/isa/s390x/inst.isle line 1761. let expr0_0 = MInst::Mov32SImm16 { rd: pattern2_0, imm: pattern4_0, @@ -2761,7 +2761,7 @@ pub fn constructor_emit_imm( let expr1_0 = C::emit(ctx, &expr0_0); return Some(expr1_0); } - // Rule at src/isa/s390x/inst.isle line 1773. + // Rule at src/isa/s390x/inst.isle line 1765. let expr0_0 = C::u64_as_u32(ctx, pattern3_0); let expr1_0 = MInst::Mov32Imm { rd: pattern2_0, @@ -2775,14 +2775,14 @@ pub fn constructor_emit_imm( let pattern3_0 = arg2; if let Some(pattern4_0) = C::u64_nonzero_hipart(ctx, pattern3_0) { if let Some(pattern5_0) = C::u64_nonzero_lopart(ctx, pattern3_0) { - // Rule at src/isa/s390x/inst.isle line 1793. + // Rule at src/isa/s390x/inst.isle line 1785. let expr0_0 = constructor_emit_imm(ctx, pattern1_0, pattern2_0, pattern4_0)?; let expr1_0 = constructor_emit_insert_imm(ctx, pattern2_0, pattern5_0)?; return Some(expr1_0); } } if let Some(pattern4_0) = C::i16_from_u64(ctx, pattern3_0) { - // Rule at src/isa/s390x/inst.isle line 1777. + // Rule at src/isa/s390x/inst.isle line 1769. let expr0_0 = MInst::Mov64SImm16 { rd: pattern2_0, imm: pattern4_0, @@ -2791,7 +2791,7 @@ pub fn constructor_emit_imm( return Some(expr1_0); } if let Some(pattern4_0) = C::i32_from_u64(ctx, pattern3_0) { - // Rule at src/isa/s390x/inst.isle line 1781. + // Rule at src/isa/s390x/inst.isle line 1773. let expr0_0 = MInst::Mov64SImm32 { rd: pattern2_0, imm: pattern4_0, @@ -2800,7 +2800,7 @@ pub fn constructor_emit_imm( return Some(expr1_0); } if let Some(pattern4_0) = C::uimm32shifted_from_u64(ctx, pattern3_0) { - // Rule at src/isa/s390x/inst.isle line 1789. + // Rule at src/isa/s390x/inst.isle line 1781. let expr0_0 = MInst::Mov64UImm32Shifted { rd: pattern2_0, imm: pattern4_0, @@ -2809,7 +2809,7 @@ pub fn constructor_emit_imm( return Some(expr1_0); } if let Some(pattern4_0) = C::uimm16shifted_from_u64(ctx, pattern3_0) { - // Rule at src/isa/s390x/inst.isle line 1785. + // Rule at src/isa/s390x/inst.isle line 1777. let expr0_0 = MInst::Mov64UImm16Shifted { rd: pattern2_0, imm: pattern4_0, @@ -2830,7 +2830,7 @@ pub fn constructor_emit_insert_imm( let pattern0_0 = arg0; let pattern1_0 = arg1; if let Some(pattern2_0) = C::uimm32shifted_from_u64(ctx, pattern1_0) { - // Rule at src/isa/s390x/inst.isle line 1806. + // Rule at src/isa/s390x/inst.isle line 1798. let expr0_0 = MInst::Insert64UImm32Shifted { rd: pattern0_0, imm: pattern2_0, @@ -2839,7 +2839,7 @@ pub fn constructor_emit_insert_imm( return Some(expr1_0); } if let Some(pattern2_0) = C::uimm16shifted_from_u64(ctx, pattern1_0) { - // Rule at src/isa/s390x/inst.isle line 1802. + // Rule at src/isa/s390x/inst.isle line 1794. let expr0_0 = MInst::Insert64UImm16Shifted { rd: pattern0_0, imm: pattern2_0, @@ -2854,7 +2854,7 @@ pub fn constructor_emit_insert_imm( pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1821. + // Rule at src/isa/s390x/inst.isle line 1813. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = constructor_emit_imm(ctx, pattern0_0, expr0_0, pattern1_0)?; let expr2_0 = C::writable_reg_to_reg(ctx, expr0_0); @@ -2871,7 +2871,7 @@ pub fn constructor_imm_regpair_lo( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1829. + // Rule at src/isa/s390x/inst.isle line 1821. let expr0_0 = constructor_copy_writable_regpair(ctx, pattern2_0)?; let expr1_0 = constructor_writable_regpair_lo(ctx, &expr0_0)?; let expr2_0 = constructor_emit_imm(ctx, pattern0_0, expr1_0, pattern1_0)?; @@ -2889,7 +2889,7 @@ pub fn constructor_imm_regpair_hi( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1837. + // Rule at src/isa/s390x/inst.isle line 1829. let expr0_0 = constructor_copy_writable_regpair(ctx, pattern2_0)?; let expr1_0 = constructor_writable_regpair_hi(ctx, &expr0_0)?; let expr2_0 = constructor_emit_imm(ctx, pattern0_0, expr1_0, pattern1_0)?; @@ -2901,22 +2901,22 @@ pub fn constructor_imm_regpair_hi( pub fn constructor_ty_ext32(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I8 { - // Rule at src/isa/s390x/inst.isle line 1847. + // Rule at src/isa/s390x/inst.isle line 1839. let expr0_0: Type = I32; return Some(expr0_0); } if pattern0_0 == I16 { - // Rule at src/isa/s390x/inst.isle line 1848. + // Rule at src/isa/s390x/inst.isle line 1840. let expr0_0: Type = I32; return Some(expr0_0); } if pattern0_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 1849. + // Rule at src/isa/s390x/inst.isle line 1841. let expr0_0: Type = I32; return Some(expr0_0); } if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 1850. + // Rule at src/isa/s390x/inst.isle line 1842. let expr0_0: Type = I64; return Some(expr0_0); } @@ -2927,22 +2927,22 @@ pub fn constructor_ty_ext32(ctx: &mut C, arg0: Type) -> Option pub fn constructor_ty_ext64(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I8 { - // Rule at src/isa/s390x/inst.isle line 1854. + // Rule at src/isa/s390x/inst.isle line 1846. let expr0_0: Type = I64; return Some(expr0_0); } if pattern0_0 == I16 { - // Rule at src/isa/s390x/inst.isle line 1855. + // Rule at src/isa/s390x/inst.isle line 1847. let expr0_0: Type = I64; return Some(expr0_0); } if pattern0_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 1856. + // Rule at src/isa/s390x/inst.isle line 1848. let expr0_0: Type = I64; return Some(expr0_0); } if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 1857. + // Rule at src/isa/s390x/inst.isle line 1849. let expr0_0: Type = I64; return Some(expr0_0); } @@ -2959,7 +2959,7 @@ pub fn constructor_emit_zext32_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1862. + // Rule at src/isa/s390x/inst.isle line 1854. let expr0_0: bool = false; let expr1_0 = C::ty_bits(ctx, pattern1_0); let expr2_0: u8 = 32; @@ -2984,7 +2984,7 @@ pub fn constructor_emit_sext32_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1868. + // Rule at src/isa/s390x/inst.isle line 1860. let expr0_0: bool = true; let expr1_0 = C::ty_bits(ctx, pattern1_0); let expr2_0: u8 = 32; @@ -3009,7 +3009,7 @@ pub fn constructor_emit_zext64_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1874. + // Rule at src/isa/s390x/inst.isle line 1866. let expr0_0: bool = false; let expr1_0 = C::ty_bits(ctx, pattern1_0); let expr2_0: u8 = 64; @@ -3034,7 +3034,7 @@ pub fn constructor_emit_sext64_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1880. + // Rule at src/isa/s390x/inst.isle line 1872. let expr0_0: bool = true; let expr1_0 = C::ty_bits(ctx, pattern1_0); let expr2_0: u8 = 64; @@ -3053,7 +3053,7 @@ pub fn constructor_emit_sext64_reg( pub fn constructor_zext32_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1886. + // Rule at src/isa/s390x/inst.isle line 1878. let expr0_0: Type = I32; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = constructor_emit_zext32_reg(ctx, expr1_0, pattern0_0, pattern1_0)?; @@ -3065,7 +3065,7 @@ pub fn constructor_zext32_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> pub fn constructor_sext32_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1894. + // Rule at src/isa/s390x/inst.isle line 1886. let expr0_0: Type = I32; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = constructor_emit_sext32_reg(ctx, expr1_0, pattern0_0, pattern1_0)?; @@ -3077,7 +3077,7 @@ pub fn constructor_sext32_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> pub fn constructor_zext64_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1902. + // Rule at src/isa/s390x/inst.isle line 1894. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = constructor_emit_zext64_reg(ctx, expr1_0, pattern0_0, pattern1_0)?; @@ -3089,7 +3089,7 @@ pub fn constructor_zext64_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> pub fn constructor_sext64_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1910. + // Rule at src/isa/s390x/inst.isle line 1902. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = constructor_emit_sext64_reg(ctx, expr1_0, pattern0_0, pattern1_0)?; @@ -3108,7 +3108,7 @@ pub fn constructor_emit_zext32_mem( let pattern1_0 = arg1; if pattern1_0 == I8 { let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1918. + // Rule at src/isa/s390x/inst.isle line 1910. let expr0_0 = MInst::Load32ZExt8 { rd: pattern0_0, mem: pattern3_0.clone(), @@ -3118,7 +3118,7 @@ pub fn constructor_emit_zext32_mem( } if pattern1_0 == I16 { let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1919. + // Rule at src/isa/s390x/inst.isle line 1911. let expr0_0 = MInst::Load32ZExt16 { rd: pattern0_0, mem: pattern3_0.clone(), @@ -3140,7 +3140,7 @@ pub fn constructor_emit_sext32_mem( let pattern1_0 = arg1; if pattern1_0 == I8 { let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1923. + // Rule at src/isa/s390x/inst.isle line 1915. let expr0_0 = MInst::Load32SExt8 { rd: pattern0_0, mem: pattern3_0.clone(), @@ -3150,7 +3150,7 @@ pub fn constructor_emit_sext32_mem( } if pattern1_0 == I16 { let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1924. + // Rule at src/isa/s390x/inst.isle line 1916. let expr0_0 = MInst::Load32SExt16 { rd: pattern0_0, mem: pattern3_0.clone(), @@ -3172,7 +3172,7 @@ pub fn constructor_emit_zext64_mem( let pattern1_0 = arg1; if pattern1_0 == I8 { let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1928. + // Rule at src/isa/s390x/inst.isle line 1920. let expr0_0 = MInst::Load64ZExt8 { rd: pattern0_0, mem: pattern3_0.clone(), @@ -3182,7 +3182,7 @@ pub fn constructor_emit_zext64_mem( } if pattern1_0 == I16 { let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1929. + // Rule at src/isa/s390x/inst.isle line 1921. let expr0_0 = MInst::Load64ZExt16 { rd: pattern0_0, mem: pattern3_0.clone(), @@ -3192,7 +3192,7 @@ pub fn constructor_emit_zext64_mem( } if pattern1_0 == I32 { let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1930. + // Rule at src/isa/s390x/inst.isle line 1922. let expr0_0 = MInst::Load64ZExt32 { rd: pattern0_0, mem: pattern3_0.clone(), @@ -3214,7 +3214,7 @@ pub fn constructor_emit_sext64_mem( let pattern1_0 = arg1; if pattern1_0 == I8 { let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1934. + // Rule at src/isa/s390x/inst.isle line 1926. let expr0_0 = MInst::Load64SExt8 { rd: pattern0_0, mem: pattern3_0.clone(), @@ -3224,7 +3224,7 @@ pub fn constructor_emit_sext64_mem( } if pattern1_0 == I16 { let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1935. + // Rule at src/isa/s390x/inst.isle line 1927. let expr0_0 = MInst::Load64SExt16 { rd: pattern0_0, mem: pattern3_0.clone(), @@ -3234,7 +3234,7 @@ pub fn constructor_emit_sext64_mem( } if pattern1_0 == I32 { let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1936. + // Rule at src/isa/s390x/inst.isle line 1928. let expr0_0 = MInst::Load64SExt32 { rd: pattern0_0, mem: pattern3_0.clone(), @@ -3249,7 +3249,7 @@ pub fn constructor_emit_sext64_mem( pub fn constructor_zext32_mem(ctx: &mut C, arg0: Type, arg1: &MemArg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1940. + // Rule at src/isa/s390x/inst.isle line 1932. let expr0_0: Type = I32; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = constructor_emit_zext32_mem(ctx, expr1_0, pattern0_0, pattern1_0)?; @@ -3261,7 +3261,7 @@ pub fn constructor_zext32_mem(ctx: &mut C, arg0: Type, arg1: &MemArg pub fn constructor_sext32_mem(ctx: &mut C, arg0: Type, arg1: &MemArg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1947. + // Rule at src/isa/s390x/inst.isle line 1939. let expr0_0: Type = I32; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = constructor_emit_sext32_mem(ctx, expr1_0, pattern0_0, pattern1_0)?; @@ -3273,7 +3273,7 @@ pub fn constructor_sext32_mem(ctx: &mut C, arg0: Type, arg1: &MemArg pub fn constructor_zext64_mem(ctx: &mut C, arg0: Type, arg1: &MemArg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1954. + // Rule at src/isa/s390x/inst.isle line 1946. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = constructor_emit_zext64_mem(ctx, expr1_0, pattern0_0, pattern1_0)?; @@ -3285,7 +3285,7 @@ pub fn constructor_zext64_mem(ctx: &mut C, arg0: Type, arg1: &MemArg pub fn constructor_sext64_mem(ctx: &mut C, arg0: Type, arg1: &MemArg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1961. + // Rule at src/isa/s390x/inst.isle line 1953. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = constructor_emit_sext64_mem(ctx, expr1_0, pattern0_0, pattern1_0)?; @@ -3303,7 +3303,7 @@ pub fn constructor_emit_put_in_reg_zext32( let pattern1_0 = arg1; let pattern2_0 = C::value_type(ctx, pattern1_0); if let Some(pattern3_0) = C::u64_from_value(ctx, pattern1_0) { - // Rule at src/isa/s390x/inst.isle line 1969. + // Rule at src/isa/s390x/inst.isle line 1961. let expr0_0 = constructor_ty_ext32(ctx, pattern2_0)?; let expr1_0 = constructor_emit_imm(ctx, expr0_0, pattern0_0, pattern3_0)?; return Some(expr1_0); @@ -3320,7 +3320,7 @@ pub fn constructor_emit_put_in_reg_zext32( { if let &Opcode::Load = &pattern6_0 { if let Some(()) = C::bigendian(ctx, pattern6_2) { - // Rule at src/isa/s390x/inst.isle line 1971. + // Rule at src/isa/s390x/inst.isle line 1963. let expr0_0 = constructor_sink_load(ctx, pattern4_0)?; let expr1_0 = constructor_emit_zext32_mem(ctx, pattern0_0, pattern3_0, &expr0_0)?; @@ -3329,13 +3329,13 @@ pub fn constructor_emit_put_in_reg_zext32( } } } - // Rule at src/isa/s390x/inst.isle line 1973. + // Rule at src/isa/s390x/inst.isle line 1965. let expr0_0 = C::put_in_reg(ctx, pattern1_0); let expr1_0 = constructor_emit_zext32_reg(ctx, pattern0_0, pattern3_0, expr0_0)?; return Some(expr1_0); } if let Some(pattern3_0) = C::ty_32_or_64(ctx, pattern2_0) { - // Rule at src/isa/s390x/inst.isle line 1975. + // Rule at src/isa/s390x/inst.isle line 1967. let expr0_0 = C::put_in_reg(ctx, pattern1_0); let expr1_0 = constructor_emit_mov(ctx, pattern3_0, pattern0_0, expr0_0)?; return Some(expr1_0); @@ -3353,7 +3353,7 @@ pub fn constructor_emit_put_in_reg_sext32( let pattern1_0 = arg1; let pattern2_0 = C::value_type(ctx, pattern1_0); if let Some(pattern3_0) = C::u64_from_signed_value(ctx, pattern1_0) { - // Rule at src/isa/s390x/inst.isle line 1980. + // Rule at src/isa/s390x/inst.isle line 1972. let expr0_0 = constructor_ty_ext32(ctx, pattern2_0)?; let expr1_0 = constructor_emit_imm(ctx, expr0_0, pattern0_0, pattern3_0)?; return Some(expr1_0); @@ -3370,7 +3370,7 @@ pub fn constructor_emit_put_in_reg_sext32( { if let &Opcode::Load = &pattern6_0 { if let Some(()) = C::bigendian(ctx, pattern6_2) { - // Rule at src/isa/s390x/inst.isle line 1982. + // Rule at src/isa/s390x/inst.isle line 1974. let expr0_0 = constructor_sink_load(ctx, pattern4_0)?; let expr1_0 = constructor_emit_sext32_mem(ctx, pattern0_0, pattern3_0, &expr0_0)?; @@ -3379,13 +3379,13 @@ pub fn constructor_emit_put_in_reg_sext32( } } } - // Rule at src/isa/s390x/inst.isle line 1984. + // Rule at src/isa/s390x/inst.isle line 1976. let expr0_0 = C::put_in_reg(ctx, pattern1_0); let expr1_0 = constructor_emit_sext32_reg(ctx, pattern0_0, pattern3_0, expr0_0)?; return Some(expr1_0); } if let Some(pattern3_0) = C::ty_32_or_64(ctx, pattern2_0) { - // Rule at src/isa/s390x/inst.isle line 1986. + // Rule at src/isa/s390x/inst.isle line 1978. let expr0_0 = C::put_in_reg(ctx, pattern1_0); let expr1_0 = constructor_emit_mov(ctx, pattern3_0, pattern0_0, expr0_0)?; return Some(expr1_0); @@ -3403,7 +3403,7 @@ pub fn constructor_emit_put_in_reg_zext64( let pattern1_0 = arg1; let pattern2_0 = C::value_type(ctx, pattern1_0); if let Some(pattern3_0) = C::u64_from_value(ctx, pattern1_0) { - // Rule at src/isa/s390x/inst.isle line 1991. + // Rule at src/isa/s390x/inst.isle line 1983. let expr0_0 = constructor_ty_ext64(ctx, pattern2_0)?; let expr1_0 = constructor_emit_imm(ctx, expr0_0, pattern0_0, pattern3_0)?; return Some(expr1_0); @@ -3420,7 +3420,7 @@ pub fn constructor_emit_put_in_reg_zext64( { if let &Opcode::Load = &pattern6_0 { if let Some(()) = C::bigendian(ctx, pattern6_2) { - // Rule at src/isa/s390x/inst.isle line 1993. + // Rule at src/isa/s390x/inst.isle line 1985. let expr0_0 = constructor_sink_load(ctx, pattern4_0)?; let expr1_0 = constructor_emit_zext64_mem(ctx, pattern0_0, pattern3_0, &expr0_0)?; @@ -3429,13 +3429,13 @@ pub fn constructor_emit_put_in_reg_zext64( } } } - // Rule at src/isa/s390x/inst.isle line 1995. + // Rule at src/isa/s390x/inst.isle line 1987. let expr0_0 = C::put_in_reg(ctx, pattern1_0); let expr1_0 = constructor_emit_zext64_reg(ctx, pattern0_0, pattern3_0, expr0_0)?; return Some(expr1_0); } if let Some(pattern3_0) = C::gpr64_ty(ctx, pattern2_0) { - // Rule at src/isa/s390x/inst.isle line 1997. + // Rule at src/isa/s390x/inst.isle line 1989. let expr0_0 = C::put_in_reg(ctx, pattern1_0); let expr1_0 = constructor_emit_mov(ctx, pattern3_0, pattern0_0, expr0_0)?; return Some(expr1_0); @@ -3453,7 +3453,7 @@ pub fn constructor_emit_put_in_reg_sext64( let pattern1_0 = arg1; let pattern2_0 = C::value_type(ctx, pattern1_0); if let Some(pattern3_0) = C::u64_from_signed_value(ctx, pattern1_0) { - // Rule at src/isa/s390x/inst.isle line 2002. + // Rule at src/isa/s390x/inst.isle line 1994. let expr0_0 = constructor_ty_ext64(ctx, pattern2_0)?; let expr1_0 = constructor_emit_imm(ctx, expr0_0, pattern0_0, pattern3_0)?; return Some(expr1_0); @@ -3470,7 +3470,7 @@ pub fn constructor_emit_put_in_reg_sext64( { if let &Opcode::Load = &pattern6_0 { if let Some(()) = C::bigendian(ctx, pattern6_2) { - // Rule at src/isa/s390x/inst.isle line 2004. + // Rule at src/isa/s390x/inst.isle line 1996. let expr0_0 = constructor_sink_load(ctx, pattern4_0)?; let expr1_0 = constructor_emit_sext64_mem(ctx, pattern0_0, pattern3_0, &expr0_0)?; @@ -3479,13 +3479,13 @@ pub fn constructor_emit_put_in_reg_sext64( } } } - // Rule at src/isa/s390x/inst.isle line 2006. + // Rule at src/isa/s390x/inst.isle line 1998. let expr0_0 = C::put_in_reg(ctx, pattern1_0); let expr1_0 = constructor_emit_sext64_reg(ctx, pattern0_0, pattern3_0, expr0_0)?; return Some(expr1_0); } if let Some(pattern3_0) = C::gpr64_ty(ctx, pattern2_0) { - // Rule at src/isa/s390x/inst.isle line 2008. + // Rule at src/isa/s390x/inst.isle line 2000. let expr0_0 = C::put_in_reg(ctx, pattern1_0); let expr1_0 = constructor_emit_mov(ctx, pattern3_0, pattern0_0, expr0_0)?; return Some(expr1_0); @@ -3498,7 +3498,7 @@ pub fn constructor_put_in_reg_zext32(ctx: &mut C, arg0: Value) -> Op let pattern0_0 = arg0; let pattern1_0 = C::value_type(ctx, pattern0_0); if let Some(pattern2_0) = C::u64_from_value(ctx, pattern0_0) { - // Rule at src/isa/s390x/inst.isle line 2013. + // Rule at src/isa/s390x/inst.isle line 2005. let expr0_0 = constructor_ty_ext32(ctx, pattern1_0)?; let expr1_0 = constructor_imm(ctx, expr0_0, pattern2_0)?; return Some(expr1_0); @@ -3515,7 +3515,7 @@ pub fn constructor_put_in_reg_zext32(ctx: &mut C, arg0: Value) -> Op { if let &Opcode::Load = &pattern5_0 { if let Some(()) = C::bigendian(ctx, pattern5_2) { - // Rule at src/isa/s390x/inst.isle line 2015. + // Rule at src/isa/s390x/inst.isle line 2007. let expr0_0 = constructor_sink_load(ctx, pattern3_0)?; let expr1_0 = constructor_zext32_mem(ctx, pattern2_0, &expr0_0)?; return Some(expr1_0); @@ -3523,13 +3523,13 @@ pub fn constructor_put_in_reg_zext32(ctx: &mut C, arg0: Value) -> Op } } } - // Rule at src/isa/s390x/inst.isle line 2017. + // Rule at src/isa/s390x/inst.isle line 2009. let expr0_0 = C::put_in_reg(ctx, pattern0_0); let expr1_0 = constructor_zext32_reg(ctx, pattern2_0, expr0_0)?; return Some(expr1_0); } if let Some(pattern2_0) = C::ty_32_or_64(ctx, pattern1_0) { - // Rule at src/isa/s390x/inst.isle line 2019. + // Rule at src/isa/s390x/inst.isle line 2011. let expr0_0 = C::put_in_reg(ctx, pattern0_0); return Some(expr0_0); } @@ -3541,7 +3541,7 @@ pub fn constructor_put_in_reg_sext32(ctx: &mut C, arg0: Value) -> Op let pattern0_0 = arg0; let pattern1_0 = C::value_type(ctx, pattern0_0); if let Some(pattern2_0) = C::u64_from_signed_value(ctx, pattern0_0) { - // Rule at src/isa/s390x/inst.isle line 2024. + // Rule at src/isa/s390x/inst.isle line 2016. let expr0_0 = constructor_ty_ext32(ctx, pattern1_0)?; let expr1_0 = constructor_imm(ctx, expr0_0, pattern2_0)?; return Some(expr1_0); @@ -3558,7 +3558,7 @@ pub fn constructor_put_in_reg_sext32(ctx: &mut C, arg0: Value) -> Op { if let &Opcode::Load = &pattern5_0 { if let Some(()) = C::bigendian(ctx, pattern5_2) { - // Rule at src/isa/s390x/inst.isle line 2026. + // Rule at src/isa/s390x/inst.isle line 2018. let expr0_0 = constructor_sink_load(ctx, pattern3_0)?; let expr1_0 = constructor_sext32_mem(ctx, pattern2_0, &expr0_0)?; return Some(expr1_0); @@ -3566,13 +3566,13 @@ pub fn constructor_put_in_reg_sext32(ctx: &mut C, arg0: Value) -> Op } } } - // Rule at src/isa/s390x/inst.isle line 2028. + // Rule at src/isa/s390x/inst.isle line 2020. let expr0_0 = C::put_in_reg(ctx, pattern0_0); let expr1_0 = constructor_sext32_reg(ctx, pattern2_0, expr0_0)?; return Some(expr1_0); } if let Some(pattern2_0) = C::ty_32_or_64(ctx, pattern1_0) { - // Rule at src/isa/s390x/inst.isle line 2030. + // Rule at src/isa/s390x/inst.isle line 2022. let expr0_0 = C::put_in_reg(ctx, pattern0_0); return Some(expr0_0); } @@ -3584,7 +3584,7 @@ pub fn constructor_put_in_reg_zext64(ctx: &mut C, arg0: Value) -> Op let pattern0_0 = arg0; let pattern1_0 = C::value_type(ctx, pattern0_0); if let Some(pattern2_0) = C::u64_from_value(ctx, pattern0_0) { - // Rule at src/isa/s390x/inst.isle line 2035. + // Rule at src/isa/s390x/inst.isle line 2027. let expr0_0 = constructor_ty_ext64(ctx, pattern1_0)?; let expr1_0 = constructor_imm(ctx, expr0_0, pattern2_0)?; return Some(expr1_0); @@ -3601,7 +3601,7 @@ pub fn constructor_put_in_reg_zext64(ctx: &mut C, arg0: Value) -> Op { if let &Opcode::Load = &pattern5_0 { if let Some(()) = C::bigendian(ctx, pattern5_2) { - // Rule at src/isa/s390x/inst.isle line 2037. + // Rule at src/isa/s390x/inst.isle line 2029. let expr0_0 = constructor_sink_load(ctx, pattern3_0)?; let expr1_0 = constructor_zext64_mem(ctx, pattern2_0, &expr0_0)?; return Some(expr1_0); @@ -3609,13 +3609,13 @@ pub fn constructor_put_in_reg_zext64(ctx: &mut C, arg0: Value) -> Op } } } - // Rule at src/isa/s390x/inst.isle line 2039. + // Rule at src/isa/s390x/inst.isle line 2031. let expr0_0 = C::put_in_reg(ctx, pattern0_0); let expr1_0 = constructor_zext64_reg(ctx, pattern2_0, expr0_0)?; return Some(expr1_0); } if let Some(pattern2_0) = C::gpr64_ty(ctx, pattern1_0) { - // Rule at src/isa/s390x/inst.isle line 2041. + // Rule at src/isa/s390x/inst.isle line 2033. let expr0_0 = C::put_in_reg(ctx, pattern0_0); return Some(expr0_0); } @@ -3627,7 +3627,7 @@ pub fn constructor_put_in_reg_sext64(ctx: &mut C, arg0: Value) -> Op let pattern0_0 = arg0; let pattern1_0 = C::value_type(ctx, pattern0_0); if let Some(pattern2_0) = C::u64_from_signed_value(ctx, pattern0_0) { - // Rule at src/isa/s390x/inst.isle line 2046. + // Rule at src/isa/s390x/inst.isle line 2038. let expr0_0 = constructor_ty_ext64(ctx, pattern1_0)?; let expr1_0 = constructor_imm(ctx, expr0_0, pattern2_0)?; return Some(expr1_0); @@ -3644,7 +3644,7 @@ pub fn constructor_put_in_reg_sext64(ctx: &mut C, arg0: Value) -> Op { if let &Opcode::Load = &pattern5_0 { if let Some(()) = C::bigendian(ctx, pattern5_2) { - // Rule at src/isa/s390x/inst.isle line 2048. + // Rule at src/isa/s390x/inst.isle line 2040. let expr0_0 = constructor_sink_load(ctx, pattern3_0)?; let expr1_0 = constructor_sext64_mem(ctx, pattern2_0, &expr0_0)?; return Some(expr1_0); @@ -3652,13 +3652,13 @@ pub fn constructor_put_in_reg_sext64(ctx: &mut C, arg0: Value) -> Op } } } - // Rule at src/isa/s390x/inst.isle line 2050. + // Rule at src/isa/s390x/inst.isle line 2042. let expr0_0 = C::put_in_reg(ctx, pattern0_0); let expr1_0 = constructor_sext64_reg(ctx, pattern2_0, expr0_0)?; return Some(expr1_0); } if let Some(pattern2_0) = C::gpr64_ty(ctx, pattern1_0) { - // Rule at src/isa/s390x/inst.isle line 2052. + // Rule at src/isa/s390x/inst.isle line 2044. let expr0_0 = C::put_in_reg(ctx, pattern0_0); return Some(expr0_0); } @@ -3673,7 +3673,7 @@ pub fn constructor_put_in_regpair_lo_zext32( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2058. + // Rule at src/isa/s390x/inst.isle line 2050. let expr0_0 = constructor_copy_writable_regpair(ctx, pattern1_0)?; let expr1_0 = constructor_writable_regpair_lo(ctx, &expr0_0)?; let expr2_0 = constructor_emit_put_in_reg_zext32(ctx, expr1_0, pattern0_0)?; @@ -3689,7 +3689,7 @@ pub fn constructor_put_in_regpair_lo_sext32( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2066. + // Rule at src/isa/s390x/inst.isle line 2058. let expr0_0 = constructor_copy_writable_regpair(ctx, pattern1_0)?; let expr1_0 = constructor_writable_regpair_lo(ctx, &expr0_0)?; let expr2_0 = constructor_emit_put_in_reg_sext32(ctx, expr1_0, pattern0_0)?; @@ -3705,7 +3705,7 @@ pub fn constructor_put_in_regpair_lo_zext64( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2074. + // Rule at src/isa/s390x/inst.isle line 2066. let expr0_0 = constructor_copy_writable_regpair(ctx, pattern1_0)?; let expr1_0 = constructor_writable_regpair_lo(ctx, &expr0_0)?; let expr2_0 = constructor_emit_put_in_reg_zext64(ctx, expr1_0, pattern0_0)?; @@ -3721,7 +3721,7 @@ pub fn constructor_put_in_regpair_lo_sext64( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2082. + // Rule at src/isa/s390x/inst.isle line 2074. let expr0_0 = constructor_copy_writable_regpair(ctx, pattern1_0)?; let expr1_0 = constructor_writable_regpair_lo(ctx, &expr0_0)?; let expr2_0 = constructor_emit_put_in_reg_sext64(ctx, expr1_0, pattern0_0)?; @@ -3742,7 +3742,7 @@ pub fn constructor_emit_cmov_imm( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 2092. + // Rule at src/isa/s390x/inst.isle line 2084. let expr0_0 = MInst::CMov32SImm16 { rd: pattern2_0, cond: pattern3_0.clone(), @@ -3759,7 +3759,7 @@ pub fn constructor_emit_cmov_imm( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 2095. + // Rule at src/isa/s390x/inst.isle line 2087. let expr0_0 = MInst::CMov64SImm16 { rd: pattern2_0, cond: pattern3_0.clone(), @@ -3787,7 +3787,7 @@ pub fn constructor_cmov_imm( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 2101. + // Rule at src/isa/s390x/inst.isle line 2093. let expr0_0 = constructor_copy_writable_reg(ctx, pattern0_0, pattern3_0)?; let expr1_0 = constructor_emit_cmov_imm(ctx, pattern0_0, expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -3807,7 +3807,7 @@ pub fn constructor_cmov_imm_regpair_lo( let pattern2_0 = arg2; let pattern3_0 = arg3; let pattern4_0 = arg4; - // Rule at src/isa/s390x/inst.isle line 2108. + // Rule at src/isa/s390x/inst.isle line 2100. let expr0_0 = constructor_copy_writable_regpair(ctx, pattern4_0)?; let expr1_0 = constructor_writable_regpair_lo(ctx, &expr0_0)?; let expr2_0 = constructor_emit_cmov_imm(ctx, pattern0_0, expr1_0, pattern2_0, pattern3_0)?; @@ -3830,7 +3830,7 @@ pub fn constructor_cmov_imm_regpair_hi( let pattern2_0 = arg2; let pattern3_0 = arg3; let pattern4_0 = arg4; - // Rule at src/isa/s390x/inst.isle line 2117. + // Rule at src/isa/s390x/inst.isle line 2109. let expr0_0 = constructor_copy_writable_regpair(ctx, pattern4_0)?; let expr1_0 = constructor_writable_regpair_hi(ctx, &expr0_0)?; let expr2_0 = constructor_emit_cmov_imm(ctx, pattern0_0, expr1_0, pattern2_0, pattern3_0)?; @@ -3852,7 +3852,7 @@ pub fn constructor_emit_cmov_reg( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 2131. + // Rule at src/isa/s390x/inst.isle line 2123. let expr0_0 = MInst::FpuCMov32 { rd: pattern2_0, cond: pattern3_0.clone(), @@ -3869,7 +3869,7 @@ pub fn constructor_emit_cmov_reg( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 2134. + // Rule at src/isa/s390x/inst.isle line 2126. let expr0_0 = MInst::FpuCMov64 { rd: pattern2_0, cond: pattern3_0.clone(), @@ -3886,7 +3886,7 @@ pub fn constructor_emit_cmov_reg( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 2125. + // Rule at src/isa/s390x/inst.isle line 2117. let expr0_0 = MInst::CMov32 { rd: pattern2_0, cond: pattern3_0.clone(), @@ -3903,7 +3903,7 @@ pub fn constructor_emit_cmov_reg( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 2128. + // Rule at src/isa/s390x/inst.isle line 2120. let expr0_0 = MInst::CMov64 { rd: pattern2_0, cond: pattern3_0.clone(), @@ -3931,7 +3931,7 @@ pub fn constructor_cmov_reg( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 2140. + // Rule at src/isa/s390x/inst.isle line 2132. let expr0_0 = constructor_copy_writable_reg(ctx, pattern0_0, pattern3_0)?; let expr1_0 = constructor_emit_cmov_reg(ctx, pattern0_0, expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -3952,7 +3952,7 @@ pub fn constructor_trap_if( { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2148. + // Rule at src/isa/s390x/inst.isle line 2140. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = MInst::TrapIf { cond: pattern2_0.clone(), @@ -3978,7 +3978,7 @@ pub fn constructor_icmps_reg_and_trap( let pattern2_0 = arg2; let pattern3_0 = arg3; let pattern4_0 = arg4; - // Rule at src/isa/s390x/inst.isle line 2154. + // Rule at src/isa/s390x/inst.isle line 2146. let expr0_0 = constructor_cmpop_cmps(ctx, pattern0_0)?; let expr1_0 = MInst::CmpTrapRR { op: expr0_0, @@ -4006,7 +4006,7 @@ pub fn constructor_icmps_simm16_and_trap( let pattern2_0 = arg2; let pattern3_0 = arg3; let pattern4_0 = arg4; - // Rule at src/isa/s390x/inst.isle line 2160. + // Rule at src/isa/s390x/inst.isle line 2152. let expr0_0 = constructor_cmpop_cmps(ctx, pattern0_0)?; let expr1_0 = MInst::CmpTrapRSImm16 { op: expr0_0, @@ -4034,7 +4034,7 @@ pub fn constructor_icmpu_reg_and_trap( let pattern2_0 = arg2; let pattern3_0 = arg3; let pattern4_0 = arg4; - // Rule at src/isa/s390x/inst.isle line 2166. + // Rule at src/isa/s390x/inst.isle line 2158. let expr0_0 = constructor_cmpop_cmpu(ctx, pattern0_0)?; let expr1_0 = MInst::CmpTrapRR { op: expr0_0, @@ -4062,7 +4062,7 @@ pub fn constructor_icmpu_uimm16_and_trap( let pattern2_0 = arg2; let pattern3_0 = arg3; let pattern4_0 = arg4; - // Rule at src/isa/s390x/inst.isle line 2172. + // Rule at src/isa/s390x/inst.isle line 2164. let expr0_0 = constructor_cmpop_cmpu(ctx, pattern0_0)?; let expr1_0 = MInst::CmpTrapRUImm16 { op: expr0_0, @@ -4082,7 +4082,7 @@ pub fn constructor_trap_impl( arg0: &TrapCode, ) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/s390x/inst.isle line 2178. + // Rule at src/isa/s390x/inst.isle line 2170. let expr0_0 = MInst::Trap { trap_code: pattern0_0.clone(), }; @@ -4098,7 +4098,7 @@ pub fn constructor_trap_if_impl( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2182. + // Rule at src/isa/s390x/inst.isle line 2174. let expr0_0 = MInst::TrapIf { cond: pattern0_0.clone(), trap_code: pattern1_0.clone(), @@ -4109,7 +4109,7 @@ pub fn constructor_trap_if_impl( // Generated as internal constructor for term debugtrap_impl. pub fn constructor_debugtrap_impl(ctx: &mut C) -> Option { - // Rule at src/isa/s390x/inst.isle line 2186. + // Rule at src/isa/s390x/inst.isle line 2178. let expr0_0 = MInst::Debugtrap; let expr1_0 = SideEffectNoResult::Inst { inst: expr0_0 }; return Some(expr1_0); @@ -4123,7 +4123,7 @@ pub fn constructor_bool( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2197. + // Rule at src/isa/s390x/inst.isle line 2189. let expr0_0 = ProducesBool::ProducesBool { producer: pattern0_0.clone(), cond: pattern1_0.clone(), @@ -4142,7 +4142,7 @@ pub fn constructor_invert_bool( cond: ref pattern1_1, } = pattern0_0 { - // Rule at src/isa/s390x/inst.isle line 2201. + // Rule at src/isa/s390x/inst.isle line 2193. let expr0_0 = C::invert_cond(ctx, &pattern1_1); let expr1_0 = constructor_bool(ctx, &pattern1_0, &expr0_0)?; return Some(expr1_0); @@ -4158,7 +4158,7 @@ pub fn constructor_emit_producer(ctx: &mut C, arg0: &ProducesFlags) result: pattern1_1, } = pattern0_0 { - // Rule at src/isa/s390x/inst.isle line 2210. + // Rule at src/isa/s390x/inst.isle line 2202. let expr0_0 = C::emit(ctx, &pattern1_0); return Some(expr0_0); } @@ -4173,7 +4173,7 @@ pub fn constructor_emit_consumer(ctx: &mut C, arg0: &ConsumesFlags) result: pattern1_1, } = pattern0_0 { - // Rule at src/isa/s390x/inst.isle line 2212. + // Rule at src/isa/s390x/inst.isle line 2204. let expr0_0 = C::emit(ctx, &pattern1_0); return Some(expr0_0); } @@ -4197,7 +4197,7 @@ pub fn constructor_select_bool_reg( { let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 2216. + // Rule at src/isa/s390x/inst.isle line 2208. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = constructor_emit_producer(ctx, &pattern2_0)?; let expr2_0 = constructor_emit_mov(ctx, pattern0_0, expr0_0, pattern4_0)?; @@ -4226,7 +4226,7 @@ pub fn constructor_select_bool_imm( { let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 2225. + // Rule at src/isa/s390x/inst.isle line 2217. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = constructor_emit_producer(ctx, &pattern2_0)?; let expr2_0 = constructor_emit_imm(ctx, pattern0_0, expr0_0, pattern4_0)?; @@ -4247,7 +4247,7 @@ pub fn constructor_lower_bool( let pattern0_0 = arg0; if pattern0_0 == B1 { let pattern2_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2235. + // Rule at src/isa/s390x/inst.isle line 2227. let expr0_0: Type = B1; let expr1_0: i16 = 1; let expr2_0: u64 = 0; @@ -4256,7 +4256,7 @@ pub fn constructor_lower_bool( } if pattern0_0 == B8 { let pattern2_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2236. + // Rule at src/isa/s390x/inst.isle line 2228. let expr0_0: Type = B8; let expr1_0: i16 = -1; let expr2_0: u64 = 0; @@ -4265,7 +4265,7 @@ pub fn constructor_lower_bool( } if pattern0_0 == B16 { let pattern2_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2237. + // Rule at src/isa/s390x/inst.isle line 2229. let expr0_0: Type = B16; let expr1_0: i16 = -1; let expr2_0: u64 = 0; @@ -4274,7 +4274,7 @@ pub fn constructor_lower_bool( } if pattern0_0 == B32 { let pattern2_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2238. + // Rule at src/isa/s390x/inst.isle line 2230. let expr0_0: Type = B32; let expr1_0: i16 = -1; let expr2_0: u64 = 0; @@ -4283,7 +4283,7 @@ pub fn constructor_lower_bool( } if pattern0_0 == B64 { let pattern2_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2239. + // Rule at src/isa/s390x/inst.isle line 2231. let expr0_0: Type = B64; let expr1_0: i16 = -1; let expr2_0: u64 = 0; @@ -4308,7 +4308,7 @@ pub fn constructor_cond_br_bool( { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2243. + // Rule at src/isa/s390x/inst.isle line 2235. let expr0_0 = constructor_emit_producer(ctx, &pattern1_0)?; let expr1_0 = constructor_cond_br(ctx, pattern2_0, pattern3_0, &pattern1_1)?; return Some(expr1_0); @@ -4329,7 +4329,7 @@ pub fn constructor_oneway_cond_br_bool( } = pattern0_0 { let pattern2_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2249. + // Rule at src/isa/s390x/inst.isle line 2241. let expr0_0 = constructor_emit_producer(ctx, &pattern1_0)?; let expr1_0 = constructor_oneway_cond_br(ctx, pattern2_0, &pattern1_1)?; return Some(expr1_0); @@ -4350,7 +4350,7 @@ pub fn constructor_trap_if_bool( } = pattern0_0 { let pattern2_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2255. + // Rule at src/isa/s390x/inst.isle line 2247. let expr0_0 = constructor_emit_producer(ctx, &pattern1_0)?; let expr1_0 = constructor_trap_if_impl(ctx, &pattern1_1, pattern2_0)?; return Some(expr1_0); @@ -4363,7 +4363,7 @@ pub fn constructor_clz_reg(ctx: &mut C, arg0: i16, arg1: Reg) -> Opt let pattern0_0 = arg0; if pattern0_0 == 64 { let pattern2_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2266. + // Rule at src/isa/s390x/inst.isle line 2258. let expr0_0 = constructor_temp_writable_regpair(ctx)?; let expr1_0 = MInst::Flogr { rn: pattern2_0 }; let expr2_0 = C::emit(ctx, &expr1_0); @@ -4371,7 +4371,7 @@ pub fn constructor_clz_reg(ctx: &mut C, arg0: i16, arg1: Reg) -> Opt return Some(expr3_0); } let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2275. + // Rule at src/isa/s390x/inst.isle line 2267. let expr0_0 = constructor_temp_writable_regpair(ctx)?; let expr1_0 = MInst::Flogr { rn: pattern1_0 }; let expr2_0 = C::emit(ctx, &expr1_0); @@ -4392,22 +4392,22 @@ pub fn constructor_clz_reg(ctx: &mut C, arg0: i16, arg1: Reg) -> Opt pub fn constructor_aluop_add(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I8 { - // Rule at src/isa/s390x/inst.isle line 2286. + // Rule at src/isa/s390x/inst.isle line 2278. let expr0_0 = ALUOp::Add32; return Some(expr0_0); } if pattern0_0 == I16 { - // Rule at src/isa/s390x/inst.isle line 2287. + // Rule at src/isa/s390x/inst.isle line 2279. let expr0_0 = ALUOp::Add32; return Some(expr0_0); } if pattern0_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 2288. + // Rule at src/isa/s390x/inst.isle line 2280. let expr0_0 = ALUOp::Add32; return Some(expr0_0); } if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2289. + // Rule at src/isa/s390x/inst.isle line 2281. let expr0_0 = ALUOp::Add64; return Some(expr0_0); } @@ -4418,17 +4418,17 @@ pub fn constructor_aluop_add(ctx: &mut C, arg0: Type) -> Option(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I16 { - // Rule at src/isa/s390x/inst.isle line 2292. + // Rule at src/isa/s390x/inst.isle line 2284. let expr0_0 = ALUOp::Add32Ext16; return Some(expr0_0); } if pattern0_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 2293. + // Rule at src/isa/s390x/inst.isle line 2285. let expr0_0 = ALUOp::Add32Ext16; return Some(expr0_0); } if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2294. + // Rule at src/isa/s390x/inst.isle line 2286. let expr0_0 = ALUOp::Add64Ext16; return Some(expr0_0); } @@ -4439,7 +4439,7 @@ pub fn constructor_aluop_add_sext16(ctx: &mut C, arg0: Type) -> Opti pub fn constructor_aluop_add_sext32(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2297. + // Rule at src/isa/s390x/inst.isle line 2289. let expr0_0 = ALUOp::Add64Ext32; return Some(expr0_0); } @@ -4456,7 +4456,7 @@ pub fn constructor_add_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2300. + // Rule at src/isa/s390x/inst.isle line 2292. let expr0_0 = constructor_aluop_add(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rrr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4472,7 +4472,7 @@ pub fn constructor_add_reg_sext32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2303. + // Rule at src/isa/s390x/inst.isle line 2295. let expr0_0 = constructor_aluop_add_sext32(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4488,7 +4488,7 @@ pub fn constructor_add_simm16( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2306. + // Rule at src/isa/s390x/inst.isle line 2298. let expr0_0 = constructor_aluop_add(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rrsimm16(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4504,7 +4504,7 @@ pub fn constructor_add_simm32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2309. + // Rule at src/isa/s390x/inst.isle line 2301. let expr0_0 = constructor_aluop_add(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rsimm32(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4520,7 +4520,7 @@ pub fn constructor_add_mem( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2312. + // Rule at src/isa/s390x/inst.isle line 2304. let expr0_0 = constructor_aluop_add(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rx(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4536,7 +4536,7 @@ pub fn constructor_add_mem_sext16( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2315. + // Rule at src/isa/s390x/inst.isle line 2307. let expr0_0 = constructor_aluop_add_sext16(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rx(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4552,7 +4552,7 @@ pub fn constructor_add_mem_sext32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2318. + // Rule at src/isa/s390x/inst.isle line 2310. let expr0_0 = constructor_aluop_add_sext32(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rx(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4562,12 +4562,12 @@ pub fn constructor_add_mem_sext32( pub fn constructor_aluop_add_logical(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 2324. + // Rule at src/isa/s390x/inst.isle line 2316. let expr0_0 = ALUOp::AddLogical32; return Some(expr0_0); } if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2325. + // Rule at src/isa/s390x/inst.isle line 2317. let expr0_0 = ALUOp::AddLogical64; return Some(expr0_0); } @@ -4578,7 +4578,7 @@ pub fn constructor_aluop_add_logical(ctx: &mut C, arg0: Type) -> Opt pub fn constructor_aluop_add_logical_zext32(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2328. + // Rule at src/isa/s390x/inst.isle line 2320. let expr0_0 = ALUOp::AddLogical64Ext32; return Some(expr0_0); } @@ -4595,7 +4595,7 @@ pub fn constructor_add_logical_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2331. + // Rule at src/isa/s390x/inst.isle line 2323. let expr0_0 = constructor_aluop_add_logical(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rrr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4611,7 +4611,7 @@ pub fn constructor_add_logical_reg_zext32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2334. + // Rule at src/isa/s390x/inst.isle line 2326. let expr0_0 = constructor_aluop_add_logical_zext32(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4627,7 +4627,7 @@ pub fn constructor_add_logical_zimm32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2337. + // Rule at src/isa/s390x/inst.isle line 2329. let expr0_0 = constructor_aluop_add_logical(ctx, pattern0_0)?; let expr1_0 = constructor_alu_ruimm32(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4643,7 +4643,7 @@ pub fn constructor_add_logical_mem( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2340. + // Rule at src/isa/s390x/inst.isle line 2332. let expr0_0 = constructor_aluop_add_logical(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rx(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4659,7 +4659,7 @@ pub fn constructor_add_logical_mem_zext32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2343. + // Rule at src/isa/s390x/inst.isle line 2335. let expr0_0 = constructor_aluop_add_logical_zext32(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rx(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4669,22 +4669,22 @@ pub fn constructor_add_logical_mem_zext32( pub fn constructor_aluop_sub(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I8 { - // Rule at src/isa/s390x/inst.isle line 2349. + // Rule at src/isa/s390x/inst.isle line 2341. let expr0_0 = ALUOp::Sub32; return Some(expr0_0); } if pattern0_0 == I16 { - // Rule at src/isa/s390x/inst.isle line 2350. + // Rule at src/isa/s390x/inst.isle line 2342. let expr0_0 = ALUOp::Sub32; return Some(expr0_0); } if pattern0_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 2351. + // Rule at src/isa/s390x/inst.isle line 2343. let expr0_0 = ALUOp::Sub32; return Some(expr0_0); } if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2352. + // Rule at src/isa/s390x/inst.isle line 2344. let expr0_0 = ALUOp::Sub64; return Some(expr0_0); } @@ -4695,17 +4695,17 @@ pub fn constructor_aluop_sub(ctx: &mut C, arg0: Type) -> Option(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I16 { - // Rule at src/isa/s390x/inst.isle line 2355. + // Rule at src/isa/s390x/inst.isle line 2347. let expr0_0 = ALUOp::Sub32Ext16; return Some(expr0_0); } if pattern0_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 2356. + // Rule at src/isa/s390x/inst.isle line 2348. let expr0_0 = ALUOp::Sub32Ext16; return Some(expr0_0); } if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2357. + // Rule at src/isa/s390x/inst.isle line 2349. let expr0_0 = ALUOp::Sub64Ext16; return Some(expr0_0); } @@ -4716,7 +4716,7 @@ pub fn constructor_aluop_sub_sext16(ctx: &mut C, arg0: Type) -> Opti pub fn constructor_aluop_sub_sext32(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2360. + // Rule at src/isa/s390x/inst.isle line 2352. let expr0_0 = ALUOp::Sub64Ext32; return Some(expr0_0); } @@ -4733,7 +4733,7 @@ pub fn constructor_sub_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2363. + // Rule at src/isa/s390x/inst.isle line 2355. let expr0_0 = constructor_aluop_sub(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rrr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4749,7 +4749,7 @@ pub fn constructor_sub_reg_sext32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2366. + // Rule at src/isa/s390x/inst.isle line 2358. let expr0_0 = constructor_aluop_sub_sext32(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4765,7 +4765,7 @@ pub fn constructor_sub_mem( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2369. + // Rule at src/isa/s390x/inst.isle line 2361. let expr0_0 = constructor_aluop_sub(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rx(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4781,7 +4781,7 @@ pub fn constructor_sub_mem_sext16( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2372. + // Rule at src/isa/s390x/inst.isle line 2364. let expr0_0 = constructor_aluop_sub_sext16(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rx(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4797,7 +4797,7 @@ pub fn constructor_sub_mem_sext32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2375. + // Rule at src/isa/s390x/inst.isle line 2367. let expr0_0 = constructor_aluop_sub_sext32(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rx(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4807,12 +4807,12 @@ pub fn constructor_sub_mem_sext32( pub fn constructor_aluop_sub_logical(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 2381. + // Rule at src/isa/s390x/inst.isle line 2373. let expr0_0 = ALUOp::SubLogical32; return Some(expr0_0); } if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2382. + // Rule at src/isa/s390x/inst.isle line 2374. let expr0_0 = ALUOp::SubLogical64; return Some(expr0_0); } @@ -4823,7 +4823,7 @@ pub fn constructor_aluop_sub_logical(ctx: &mut C, arg0: Type) -> Opt pub fn constructor_aluop_sub_logical_zext32(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2385. + // Rule at src/isa/s390x/inst.isle line 2377. let expr0_0 = ALUOp::SubLogical64Ext32; return Some(expr0_0); } @@ -4840,7 +4840,7 @@ pub fn constructor_sub_logical_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2388. + // Rule at src/isa/s390x/inst.isle line 2380. let expr0_0 = constructor_aluop_sub_logical(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rrr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4856,7 +4856,7 @@ pub fn constructor_sub_logical_reg_zext32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2391. + // Rule at src/isa/s390x/inst.isle line 2383. let expr0_0 = constructor_aluop_sub_logical_zext32(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4872,7 +4872,7 @@ pub fn constructor_sub_logical_zimm32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2394. + // Rule at src/isa/s390x/inst.isle line 2386. let expr0_0 = constructor_aluop_sub_logical(ctx, pattern0_0)?; let expr1_0 = constructor_alu_ruimm32(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4888,7 +4888,7 @@ pub fn constructor_sub_logical_mem( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2397. + // Rule at src/isa/s390x/inst.isle line 2389. let expr0_0 = constructor_aluop_sub_logical(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rx(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4904,7 +4904,7 @@ pub fn constructor_sub_logical_mem_zext32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2400. + // Rule at src/isa/s390x/inst.isle line 2392. let expr0_0 = constructor_aluop_sub_logical(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rx(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4914,22 +4914,22 @@ pub fn constructor_sub_logical_mem_zext32( pub fn constructor_aluop_mul(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I8 { - // Rule at src/isa/s390x/inst.isle line 2406. + // Rule at src/isa/s390x/inst.isle line 2398. let expr0_0 = ALUOp::Mul32; return Some(expr0_0); } if pattern0_0 == I16 { - // Rule at src/isa/s390x/inst.isle line 2407. + // Rule at src/isa/s390x/inst.isle line 2399. let expr0_0 = ALUOp::Mul32; return Some(expr0_0); } if pattern0_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 2408. + // Rule at src/isa/s390x/inst.isle line 2400. let expr0_0 = ALUOp::Mul32; return Some(expr0_0); } if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2409. + // Rule at src/isa/s390x/inst.isle line 2401. let expr0_0 = ALUOp::Mul64; return Some(expr0_0); } @@ -4940,17 +4940,17 @@ pub fn constructor_aluop_mul(ctx: &mut C, arg0: Type) -> Option(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I16 { - // Rule at src/isa/s390x/inst.isle line 2412. + // Rule at src/isa/s390x/inst.isle line 2404. let expr0_0 = ALUOp::Mul32Ext16; return Some(expr0_0); } if pattern0_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 2413. + // Rule at src/isa/s390x/inst.isle line 2405. let expr0_0 = ALUOp::Mul32Ext16; return Some(expr0_0); } if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2414. + // Rule at src/isa/s390x/inst.isle line 2406. let expr0_0 = ALUOp::Mul64Ext16; return Some(expr0_0); } @@ -4961,7 +4961,7 @@ pub fn constructor_aluop_mul_sext16(ctx: &mut C, arg0: Type) -> Opti pub fn constructor_aluop_mul_sext32(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2417. + // Rule at src/isa/s390x/inst.isle line 2409. let expr0_0 = ALUOp::Mul64Ext32; return Some(expr0_0); } @@ -4978,7 +4978,7 @@ pub fn constructor_mul_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2420. + // Rule at src/isa/s390x/inst.isle line 2412. let expr0_0 = constructor_aluop_mul(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rrr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4994,7 +4994,7 @@ pub fn constructor_mul_reg_sext32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2423. + // Rule at src/isa/s390x/inst.isle line 2415. let expr0_0 = constructor_aluop_mul_sext32(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -5010,7 +5010,7 @@ pub fn constructor_mul_simm16( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2426. + // Rule at src/isa/s390x/inst.isle line 2418. let expr0_0 = constructor_aluop_mul(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rsimm16(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -5026,7 +5026,7 @@ pub fn constructor_mul_simm32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2429. + // Rule at src/isa/s390x/inst.isle line 2421. let expr0_0 = constructor_aluop_mul(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rsimm32(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -5042,7 +5042,7 @@ pub fn constructor_mul_mem( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2432. + // Rule at src/isa/s390x/inst.isle line 2424. let expr0_0 = constructor_aluop_mul(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rx(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -5058,7 +5058,7 @@ pub fn constructor_mul_mem_sext16( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2435. + // Rule at src/isa/s390x/inst.isle line 2427. let expr0_0 = constructor_aluop_mul_sext16(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rx(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -5074,7 +5074,7 @@ pub fn constructor_mul_mem_sext32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2438. + // Rule at src/isa/s390x/inst.isle line 2430. let expr0_0 = constructor_aluop_mul_sext32(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rx(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -5091,14 +5091,14 @@ pub fn constructor_udivmod( if pattern0_0 == I32 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2444. + // Rule at src/isa/s390x/inst.isle line 2436. let expr0_0 = constructor_udivmod32(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if pattern0_0 == I64 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2445. + // Rule at src/isa/s390x/inst.isle line 2437. let expr0_0 = constructor_udivmod64(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } @@ -5116,14 +5116,14 @@ pub fn constructor_sdivmod( if pattern0_0 == I32 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2451. + // Rule at src/isa/s390x/inst.isle line 2443. let expr0_0 = constructor_sdivmod32(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if pattern0_0 == I64 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2452. + // Rule at src/isa/s390x/inst.isle line 2444. let expr0_0 = constructor_sdivmod64(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } @@ -5134,12 +5134,12 @@ pub fn constructor_sdivmod( pub fn constructor_aluop_and(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if let Some(pattern1_0) = C::gpr32_ty(ctx, pattern0_0) { - // Rule at src/isa/s390x/inst.isle line 2458. + // Rule at src/isa/s390x/inst.isle line 2450. let expr0_0 = ALUOp::And32; return Some(expr0_0); } if let Some(pattern1_0) = C::gpr64_ty(ctx, pattern0_0) { - // Rule at src/isa/s390x/inst.isle line 2459. + // Rule at src/isa/s390x/inst.isle line 2451. let expr0_0 = ALUOp::And64; return Some(expr0_0); } @@ -5156,7 +5156,7 @@ pub fn constructor_and_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2462. + // Rule at src/isa/s390x/inst.isle line 2454. let expr0_0 = constructor_aluop_and(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rrr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -5172,7 +5172,7 @@ pub fn constructor_and_uimm16shifted( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2465. + // Rule at src/isa/s390x/inst.isle line 2457. let expr0_0 = constructor_aluop_and(ctx, pattern0_0)?; let expr1_0 = constructor_alu_ruimm16shifted(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; @@ -5189,7 +5189,7 @@ pub fn constructor_and_uimm32shifted( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2468. + // Rule at src/isa/s390x/inst.isle line 2460. let expr0_0 = constructor_aluop_and(ctx, pattern0_0)?; let expr1_0 = constructor_alu_ruimm32shifted(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; @@ -5206,7 +5206,7 @@ pub fn constructor_and_mem( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2471. + // Rule at src/isa/s390x/inst.isle line 2463. let expr0_0 = constructor_aluop_and(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rx(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -5216,12 +5216,12 @@ pub fn constructor_and_mem( pub fn constructor_aluop_or(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if let Some(pattern1_0) = C::gpr32_ty(ctx, pattern0_0) { - // Rule at src/isa/s390x/inst.isle line 2477. + // Rule at src/isa/s390x/inst.isle line 2469. let expr0_0 = ALUOp::Orr32; return Some(expr0_0); } if let Some(pattern1_0) = C::gpr64_ty(ctx, pattern0_0) { - // Rule at src/isa/s390x/inst.isle line 2478. + // Rule at src/isa/s390x/inst.isle line 2470. let expr0_0 = ALUOp::Orr64; return Some(expr0_0); } @@ -5238,7 +5238,7 @@ pub fn constructor_or_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2481. + // Rule at src/isa/s390x/inst.isle line 2473. let expr0_0 = constructor_aluop_or(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rrr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -5254,7 +5254,7 @@ pub fn constructor_or_uimm16shifted( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2484. + // Rule at src/isa/s390x/inst.isle line 2476. let expr0_0 = constructor_aluop_or(ctx, pattern0_0)?; let expr1_0 = constructor_alu_ruimm16shifted(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; @@ -5271,7 +5271,7 @@ pub fn constructor_or_uimm32shifted( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2487. + // Rule at src/isa/s390x/inst.isle line 2479. let expr0_0 = constructor_aluop_or(ctx, pattern0_0)?; let expr1_0 = constructor_alu_ruimm32shifted(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; @@ -5288,7 +5288,7 @@ pub fn constructor_or_mem( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2490. + // Rule at src/isa/s390x/inst.isle line 2482. let expr0_0 = constructor_aluop_or(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rx(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -5298,12 +5298,12 @@ pub fn constructor_or_mem( pub fn constructor_aluop_xor(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if let Some(pattern1_0) = C::gpr32_ty(ctx, pattern0_0) { - // Rule at src/isa/s390x/inst.isle line 2496. + // Rule at src/isa/s390x/inst.isle line 2488. let expr0_0 = ALUOp::Xor32; return Some(expr0_0); } if let Some(pattern1_0) = C::gpr64_ty(ctx, pattern0_0) { - // Rule at src/isa/s390x/inst.isle line 2497. + // Rule at src/isa/s390x/inst.isle line 2489. let expr0_0 = ALUOp::Xor64; return Some(expr0_0); } @@ -5320,7 +5320,7 @@ pub fn constructor_xor_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2500. + // Rule at src/isa/s390x/inst.isle line 2492. let expr0_0 = constructor_aluop_xor(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rrr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -5336,7 +5336,7 @@ pub fn constructor_xor_uimm32shifted( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2503. + // Rule at src/isa/s390x/inst.isle line 2495. let expr0_0 = constructor_aluop_xor(ctx, pattern0_0)?; let expr1_0 = constructor_alu_ruimm32shifted(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; @@ -5353,7 +5353,7 @@ pub fn constructor_xor_mem( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2506. + // Rule at src/isa/s390x/inst.isle line 2498. let expr0_0 = constructor_aluop_xor(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rx(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -5364,7 +5364,7 @@ pub fn constructor_not_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> Op let pattern0_0 = arg0; if let Some(pattern1_0) = C::gpr32_ty(ctx, pattern0_0) { let pattern2_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2512. + // Rule at src/isa/s390x/inst.isle line 2504. let expr0_0: u32 = 4294967295; let expr1_0: u8 = 0; let expr2_0 = C::uimm32shifted(ctx, expr0_0, expr1_0); @@ -5373,7 +5373,7 @@ pub fn constructor_not_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> Op } if let Some(pattern1_0) = C::gpr64_ty(ctx, pattern0_0) { let pattern2_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2514. + // Rule at src/isa/s390x/inst.isle line 2506. let expr0_0: u32 = 4294967295; let expr1_0: u8 = 0; let expr2_0 = C::uimm32shifted(ctx, expr0_0, expr1_0); @@ -5391,12 +5391,12 @@ pub fn constructor_not_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> Op pub fn constructor_aluop_and_not(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if let Some(pattern1_0) = C::gpr32_ty(ctx, pattern0_0) { - // Rule at src/isa/s390x/inst.isle line 2523. + // Rule at src/isa/s390x/inst.isle line 2515. let expr0_0 = ALUOp::AndNot32; return Some(expr0_0); } if let Some(pattern1_0) = C::gpr64_ty(ctx, pattern0_0) { - // Rule at src/isa/s390x/inst.isle line 2524. + // Rule at src/isa/s390x/inst.isle line 2516. let expr0_0 = ALUOp::AndNot64; return Some(expr0_0); } @@ -5413,7 +5413,7 @@ pub fn constructor_and_not_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2527. + // Rule at src/isa/s390x/inst.isle line 2519. let expr0_0 = constructor_aluop_and_not(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rrr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -5423,12 +5423,12 @@ pub fn constructor_and_not_reg( pub fn constructor_aluop_or_not(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if let Some(pattern1_0) = C::gpr32_ty(ctx, pattern0_0) { - // Rule at src/isa/s390x/inst.isle line 2533. + // Rule at src/isa/s390x/inst.isle line 2525. let expr0_0 = ALUOp::OrrNot32; return Some(expr0_0); } if let Some(pattern1_0) = C::gpr64_ty(ctx, pattern0_0) { - // Rule at src/isa/s390x/inst.isle line 2534. + // Rule at src/isa/s390x/inst.isle line 2526. let expr0_0 = ALUOp::OrrNot64; return Some(expr0_0); } @@ -5445,7 +5445,7 @@ pub fn constructor_or_not_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2537. + // Rule at src/isa/s390x/inst.isle line 2529. let expr0_0 = constructor_aluop_or_not(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rrr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -5455,12 +5455,12 @@ pub fn constructor_or_not_reg( pub fn constructor_aluop_xor_not(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if let Some(pattern1_0) = C::gpr32_ty(ctx, pattern0_0) { - // Rule at src/isa/s390x/inst.isle line 2543. + // Rule at src/isa/s390x/inst.isle line 2535. let expr0_0 = ALUOp::XorNot32; return Some(expr0_0); } if let Some(pattern1_0) = C::gpr64_ty(ctx, pattern0_0) { - // Rule at src/isa/s390x/inst.isle line 2544. + // Rule at src/isa/s390x/inst.isle line 2536. let expr0_0 = ALUOp::XorNot64; return Some(expr0_0); } @@ -5477,7 +5477,7 @@ pub fn constructor_xor_not_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2547. + // Rule at src/isa/s390x/inst.isle line 2539. let expr0_0 = constructor_aluop_xor_not(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rrr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -5487,12 +5487,12 @@ pub fn constructor_xor_not_reg( pub fn constructor_unaryop_abs(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 2553. + // Rule at src/isa/s390x/inst.isle line 2545. let expr0_0 = UnaryOp::Abs32; return Some(expr0_0); } if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2554. + // Rule at src/isa/s390x/inst.isle line 2546. let expr0_0 = UnaryOp::Abs64; return Some(expr0_0); } @@ -5503,7 +5503,7 @@ pub fn constructor_unaryop_abs(ctx: &mut C, arg0: Type) -> Option(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2557. + // Rule at src/isa/s390x/inst.isle line 2549. let expr0_0 = UnaryOp::Abs64Ext32; return Some(expr0_0); } @@ -5514,7 +5514,7 @@ pub fn constructor_unaryop_abs_sext32(ctx: &mut C, arg0: Type) -> Op pub fn constructor_abs_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2560. + // Rule at src/isa/s390x/inst.isle line 2552. let expr0_0 = constructor_unaryop_abs(ctx, pattern0_0)?; let expr1_0 = constructor_unary_rr(ctx, pattern0_0, &expr0_0, pattern1_0)?; return Some(expr1_0); @@ -5524,7 +5524,7 @@ pub fn constructor_abs_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> Op pub fn constructor_abs_reg_sext32(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2563. + // Rule at src/isa/s390x/inst.isle line 2555. let expr0_0 = constructor_unaryop_abs_sext32(ctx, pattern0_0)?; let expr1_0 = constructor_unary_rr(ctx, pattern0_0, &expr0_0, pattern1_0)?; return Some(expr1_0); @@ -5534,22 +5534,22 @@ pub fn constructor_abs_reg_sext32(ctx: &mut C, arg0: Type, arg1: Reg pub fn constructor_unaryop_neg(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I8 { - // Rule at src/isa/s390x/inst.isle line 2569. + // Rule at src/isa/s390x/inst.isle line 2561. let expr0_0 = UnaryOp::Neg32; return Some(expr0_0); } if pattern0_0 == I16 { - // Rule at src/isa/s390x/inst.isle line 2570. + // Rule at src/isa/s390x/inst.isle line 2562. let expr0_0 = UnaryOp::Neg32; return Some(expr0_0); } if pattern0_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 2571. + // Rule at src/isa/s390x/inst.isle line 2563. let expr0_0 = UnaryOp::Neg32; return Some(expr0_0); } if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2572. + // Rule at src/isa/s390x/inst.isle line 2564. let expr0_0 = UnaryOp::Neg64; return Some(expr0_0); } @@ -5560,7 +5560,7 @@ pub fn constructor_unaryop_neg(ctx: &mut C, arg0: Type) -> Option(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2575. + // Rule at src/isa/s390x/inst.isle line 2567. let expr0_0 = UnaryOp::Neg64Ext32; return Some(expr0_0); } @@ -5571,7 +5571,7 @@ pub fn constructor_unaryop_neg_sext32(ctx: &mut C, arg0: Type) -> Op pub fn constructor_neg_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2578. + // Rule at src/isa/s390x/inst.isle line 2570. let expr0_0 = constructor_unaryop_neg(ctx, pattern0_0)?; let expr1_0 = constructor_unary_rr(ctx, pattern0_0, &expr0_0, pattern1_0)?; return Some(expr1_0); @@ -5581,7 +5581,7 @@ pub fn constructor_neg_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> Op pub fn constructor_neg_reg_sext32(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2581. + // Rule at src/isa/s390x/inst.isle line 2573. let expr0_0 = constructor_unaryop_neg_sext32(ctx, pattern0_0)?; let expr1_0 = constructor_unary_rr(ctx, pattern0_0, &expr0_0, pattern1_0)?; return Some(expr1_0); @@ -5591,12 +5591,12 @@ pub fn constructor_neg_reg_sext32(ctx: &mut C, arg0: Type, arg1: Reg pub fn constructor_shiftop_rot(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 2587. + // Rule at src/isa/s390x/inst.isle line 2579. let expr0_0 = ShiftOp::RotL32; return Some(expr0_0); } if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2588. + // Rule at src/isa/s390x/inst.isle line 2580. let expr0_0 = ShiftOp::RotL64; return Some(expr0_0); } @@ -5613,7 +5613,7 @@ pub fn constructor_rot_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2591. + // Rule at src/isa/s390x/inst.isle line 2583. let expr0_0 = constructor_shiftop_rot(ctx, pattern0_0)?; let expr1_0: u8 = 0; let expr2_0 = constructor_shift_rr(ctx, pattern0_0, &expr0_0, pattern1_0, expr1_0, pattern2_0)?; @@ -5630,7 +5630,7 @@ pub fn constructor_rot_imm( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2595. + // Rule at src/isa/s390x/inst.isle line 2587. let expr0_0 = constructor_shiftop_rot(ctx, pattern0_0)?; let expr1_0 = C::zero_reg(ctx); let expr2_0 = constructor_shift_rr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0, expr1_0)?; @@ -5641,22 +5641,22 @@ pub fn constructor_rot_imm( pub fn constructor_shiftop_lshl(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I8 { - // Rule at src/isa/s390x/inst.isle line 2602. + // Rule at src/isa/s390x/inst.isle line 2594. let expr0_0 = ShiftOp::LShL32; return Some(expr0_0); } if pattern0_0 == I16 { - // Rule at src/isa/s390x/inst.isle line 2603. + // Rule at src/isa/s390x/inst.isle line 2595. let expr0_0 = ShiftOp::LShL32; return Some(expr0_0); } if pattern0_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 2604. + // Rule at src/isa/s390x/inst.isle line 2596. let expr0_0 = ShiftOp::LShL32; return Some(expr0_0); } if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2605. + // Rule at src/isa/s390x/inst.isle line 2597. let expr0_0 = ShiftOp::LShL64; return Some(expr0_0); } @@ -5673,7 +5673,7 @@ pub fn constructor_lshl_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2608. + // Rule at src/isa/s390x/inst.isle line 2600. let expr0_0 = constructor_shiftop_lshl(ctx, pattern0_0)?; let expr1_0: u8 = 0; let expr2_0 = constructor_shift_rr(ctx, pattern0_0, &expr0_0, pattern1_0, expr1_0, pattern2_0)?; @@ -5690,7 +5690,7 @@ pub fn constructor_lshl_imm( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2612. + // Rule at src/isa/s390x/inst.isle line 2604. let expr0_0 = constructor_shiftop_lshl(ctx, pattern0_0)?; let expr1_0 = C::zero_reg(ctx); let expr2_0 = constructor_shift_rr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0, expr1_0)?; @@ -5701,12 +5701,12 @@ pub fn constructor_lshl_imm( pub fn constructor_shiftop_lshr(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 2619. + // Rule at src/isa/s390x/inst.isle line 2611. let expr0_0 = ShiftOp::LShR32; return Some(expr0_0); } if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2620. + // Rule at src/isa/s390x/inst.isle line 2612. let expr0_0 = ShiftOp::LShR64; return Some(expr0_0); } @@ -5723,7 +5723,7 @@ pub fn constructor_lshr_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2623. + // Rule at src/isa/s390x/inst.isle line 2615. let expr0_0 = constructor_shiftop_lshr(ctx, pattern0_0)?; let expr1_0: u8 = 0; let expr2_0 = constructor_shift_rr(ctx, pattern0_0, &expr0_0, pattern1_0, expr1_0, pattern2_0)?; @@ -5740,7 +5740,7 @@ pub fn constructor_lshr_imm( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2627. + // Rule at src/isa/s390x/inst.isle line 2619. let expr0_0 = constructor_shiftop_lshr(ctx, pattern0_0)?; let expr1_0 = C::zero_reg(ctx); let expr2_0 = constructor_shift_rr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0, expr1_0)?; @@ -5751,12 +5751,12 @@ pub fn constructor_lshr_imm( pub fn constructor_shiftop_ashr(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 2634. + // Rule at src/isa/s390x/inst.isle line 2626. let expr0_0 = ShiftOp::AShR32; return Some(expr0_0); } if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2635. + // Rule at src/isa/s390x/inst.isle line 2627. let expr0_0 = ShiftOp::AShR64; return Some(expr0_0); } @@ -5773,7 +5773,7 @@ pub fn constructor_ashr_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2638. + // Rule at src/isa/s390x/inst.isle line 2630. let expr0_0 = constructor_shiftop_ashr(ctx, pattern0_0)?; let expr1_0: u8 = 0; let expr2_0 = constructor_shift_rr(ctx, pattern0_0, &expr0_0, pattern1_0, expr1_0, pattern2_0)?; @@ -5790,7 +5790,7 @@ pub fn constructor_ashr_imm( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2642. + // Rule at src/isa/s390x/inst.isle line 2634. let expr0_0 = constructor_shiftop_ashr(ctx, pattern0_0)?; let expr1_0 = C::zero_reg(ctx); let expr2_0 = constructor_shift_rr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0, expr1_0)?; @@ -5800,7 +5800,7 @@ pub fn constructor_ashr_imm( // Generated as internal constructor for term popcnt_byte. pub fn constructor_popcnt_byte(ctx: &mut C, arg0: Reg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/s390x/inst.isle line 2649. + // Rule at src/isa/s390x/inst.isle line 2641. let expr0_0: Type = I64; let expr1_0 = UnaryOp::PopcntByte; let expr2_0 = constructor_unary_rr(ctx, expr0_0, &expr1_0, pattern0_0)?; @@ -5810,7 +5810,7 @@ pub fn constructor_popcnt_byte(ctx: &mut C, arg0: Reg) -> Option(ctx: &mut C, arg0: Reg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/s390x/inst.isle line 2652. + // Rule at src/isa/s390x/inst.isle line 2644. let expr0_0: Type = I64; let expr1_0 = UnaryOp::PopcntReg; let expr2_0 = constructor_unary_rr(ctx, expr0_0, &expr1_0, pattern0_0)?; @@ -5828,7 +5828,7 @@ pub fn constructor_atomic_rmw_and( if pattern0_0 == I32 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2658. + // Rule at src/isa/s390x/inst.isle line 2650. let expr0_0: Type = I32; let expr1_0 = ALUOp::And32; let expr2_0 = constructor_atomic_rmw_impl(ctx, expr0_0, &expr1_0, pattern2_0, pattern3_0)?; @@ -5837,7 +5837,7 @@ pub fn constructor_atomic_rmw_and( if pattern0_0 == I64 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2659. + // Rule at src/isa/s390x/inst.isle line 2651. let expr0_0: Type = I64; let expr1_0 = ALUOp::And64; let expr2_0 = constructor_atomic_rmw_impl(ctx, expr0_0, &expr1_0, pattern2_0, pattern3_0)?; @@ -5857,7 +5857,7 @@ pub fn constructor_atomic_rmw_or( if pattern0_0 == I32 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2662. + // Rule at src/isa/s390x/inst.isle line 2654. let expr0_0: Type = I32; let expr1_0 = ALUOp::Orr32; let expr2_0 = constructor_atomic_rmw_impl(ctx, expr0_0, &expr1_0, pattern2_0, pattern3_0)?; @@ -5866,7 +5866,7 @@ pub fn constructor_atomic_rmw_or( if pattern0_0 == I64 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2663. + // Rule at src/isa/s390x/inst.isle line 2655. let expr0_0: Type = I64; let expr1_0 = ALUOp::Orr64; let expr2_0 = constructor_atomic_rmw_impl(ctx, expr0_0, &expr1_0, pattern2_0, pattern3_0)?; @@ -5886,7 +5886,7 @@ pub fn constructor_atomic_rmw_xor( if pattern0_0 == I32 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2666. + // Rule at src/isa/s390x/inst.isle line 2658. let expr0_0: Type = I32; let expr1_0 = ALUOp::Xor32; let expr2_0 = constructor_atomic_rmw_impl(ctx, expr0_0, &expr1_0, pattern2_0, pattern3_0)?; @@ -5895,7 +5895,7 @@ pub fn constructor_atomic_rmw_xor( if pattern0_0 == I64 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2667. + // Rule at src/isa/s390x/inst.isle line 2659. let expr0_0: Type = I64; let expr1_0 = ALUOp::Xor64; let expr2_0 = constructor_atomic_rmw_impl(ctx, expr0_0, &expr1_0, pattern2_0, pattern3_0)?; @@ -5915,7 +5915,7 @@ pub fn constructor_atomic_rmw_add( if pattern0_0 == I32 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2670. + // Rule at src/isa/s390x/inst.isle line 2662. let expr0_0: Type = I32; let expr1_0 = ALUOp::Add32; let expr2_0 = constructor_atomic_rmw_impl(ctx, expr0_0, &expr1_0, pattern2_0, pattern3_0)?; @@ -5924,7 +5924,7 @@ pub fn constructor_atomic_rmw_add( if pattern0_0 == I64 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2671. + // Rule at src/isa/s390x/inst.isle line 2663. let expr0_0: Type = I64; let expr1_0 = ALUOp::Add64; let expr2_0 = constructor_atomic_rmw_impl(ctx, expr0_0, &expr1_0, pattern2_0, pattern3_0)?; @@ -5937,12 +5937,12 @@ pub fn constructor_atomic_rmw_add( pub fn constructor_fpuop2_add(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32 { - // Rule at src/isa/s390x/inst.isle line 2677. + // Rule at src/isa/s390x/inst.isle line 2669. let expr0_0 = FPUOp2::Add32; return Some(expr0_0); } if pattern0_0 == F64 { - // Rule at src/isa/s390x/inst.isle line 2678. + // Rule at src/isa/s390x/inst.isle line 2670. let expr0_0 = FPUOp2::Add64; return Some(expr0_0); } @@ -5959,7 +5959,7 @@ pub fn constructor_fadd_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2681. + // Rule at src/isa/s390x/inst.isle line 2673. let expr0_0 = constructor_fpuop2_add(ctx, pattern0_0)?; let expr1_0 = constructor_fpu_rrr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -5969,12 +5969,12 @@ pub fn constructor_fadd_reg( pub fn constructor_fpuop2_sub(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32 { - // Rule at src/isa/s390x/inst.isle line 2687. + // Rule at src/isa/s390x/inst.isle line 2679. let expr0_0 = FPUOp2::Sub32; return Some(expr0_0); } if pattern0_0 == F64 { - // Rule at src/isa/s390x/inst.isle line 2688. + // Rule at src/isa/s390x/inst.isle line 2680. let expr0_0 = FPUOp2::Sub64; return Some(expr0_0); } @@ -5991,7 +5991,7 @@ pub fn constructor_fsub_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2691. + // Rule at src/isa/s390x/inst.isle line 2683. let expr0_0 = constructor_fpuop2_sub(ctx, pattern0_0)?; let expr1_0 = constructor_fpu_rrr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -6001,12 +6001,12 @@ pub fn constructor_fsub_reg( pub fn constructor_fpuop2_mul(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32 { - // Rule at src/isa/s390x/inst.isle line 2697. + // Rule at src/isa/s390x/inst.isle line 2689. let expr0_0 = FPUOp2::Mul32; return Some(expr0_0); } if pattern0_0 == F64 { - // Rule at src/isa/s390x/inst.isle line 2698. + // Rule at src/isa/s390x/inst.isle line 2690. let expr0_0 = FPUOp2::Mul64; return Some(expr0_0); } @@ -6023,7 +6023,7 @@ pub fn constructor_fmul_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2701. + // Rule at src/isa/s390x/inst.isle line 2693. let expr0_0 = constructor_fpuop2_mul(ctx, pattern0_0)?; let expr1_0 = constructor_fpu_rrr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -6033,12 +6033,12 @@ pub fn constructor_fmul_reg( pub fn constructor_fpuop2_div(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32 { - // Rule at src/isa/s390x/inst.isle line 2707. + // Rule at src/isa/s390x/inst.isle line 2699. let expr0_0 = FPUOp2::Div32; return Some(expr0_0); } if pattern0_0 == F64 { - // Rule at src/isa/s390x/inst.isle line 2708. + // Rule at src/isa/s390x/inst.isle line 2700. let expr0_0 = FPUOp2::Div64; return Some(expr0_0); } @@ -6055,7 +6055,7 @@ pub fn constructor_fdiv_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2711. + // Rule at src/isa/s390x/inst.isle line 2703. let expr0_0 = constructor_fpuop2_div(ctx, pattern0_0)?; let expr1_0 = constructor_fpu_rrr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -6065,12 +6065,12 @@ pub fn constructor_fdiv_reg( pub fn constructor_fpuop2_min(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32 { - // Rule at src/isa/s390x/inst.isle line 2717. + // Rule at src/isa/s390x/inst.isle line 2709. let expr0_0 = FPUOp2::Min32; return Some(expr0_0); } if pattern0_0 == F64 { - // Rule at src/isa/s390x/inst.isle line 2718. + // Rule at src/isa/s390x/inst.isle line 2710. let expr0_0 = FPUOp2::Min64; return Some(expr0_0); } @@ -6087,7 +6087,7 @@ pub fn constructor_fmin_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2721. + // Rule at src/isa/s390x/inst.isle line 2713. let expr0_0 = constructor_fpuop2_min(ctx, pattern0_0)?; let expr1_0 = constructor_fpuvec_rrr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -6097,12 +6097,12 @@ pub fn constructor_fmin_reg( pub fn constructor_fpuop2_max(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32 { - // Rule at src/isa/s390x/inst.isle line 2727. + // Rule at src/isa/s390x/inst.isle line 2719. let expr0_0 = FPUOp2::Max32; return Some(expr0_0); } if pattern0_0 == F64 { - // Rule at src/isa/s390x/inst.isle line 2728. + // Rule at src/isa/s390x/inst.isle line 2720. let expr0_0 = FPUOp2::Max64; return Some(expr0_0); } @@ -6119,7 +6119,7 @@ pub fn constructor_fmax_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2731. + // Rule at src/isa/s390x/inst.isle line 2723. let expr0_0 = constructor_fpuop2_max(ctx, pattern0_0)?; let expr1_0 = constructor_fpuvec_rrr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -6129,12 +6129,12 @@ pub fn constructor_fmax_reg( pub fn constructor_fpuop3_fma(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32 { - // Rule at src/isa/s390x/inst.isle line 2737. + // Rule at src/isa/s390x/inst.isle line 2729. let expr0_0 = FPUOp3::MAdd32; return Some(expr0_0); } if pattern0_0 == F64 { - // Rule at src/isa/s390x/inst.isle line 2738. + // Rule at src/isa/s390x/inst.isle line 2730. let expr0_0 = FPUOp3::MAdd64; return Some(expr0_0); } @@ -6153,7 +6153,7 @@ pub fn constructor_fma_reg( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 2741. + // Rule at src/isa/s390x/inst.isle line 2733. let expr0_0 = constructor_fpuop3_fma(ctx, pattern0_0)?; let expr1_0 = constructor_fpu_rrrr( ctx, pattern0_0, &expr0_0, pattern3_0, pattern1_0, pattern2_0, @@ -6165,12 +6165,12 @@ pub fn constructor_fma_reg( pub fn constructor_fpuop1_sqrt(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32 { - // Rule at src/isa/s390x/inst.isle line 2747. + // Rule at src/isa/s390x/inst.isle line 2739. let expr0_0 = FPUOp1::Sqrt32; return Some(expr0_0); } if pattern0_0 == F64 { - // Rule at src/isa/s390x/inst.isle line 2748. + // Rule at src/isa/s390x/inst.isle line 2740. let expr0_0 = FPUOp1::Sqrt64; return Some(expr0_0); } @@ -6181,7 +6181,7 @@ pub fn constructor_fpuop1_sqrt(ctx: &mut C, arg0: Type) -> Option(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2751. + // Rule at src/isa/s390x/inst.isle line 2743. let expr0_0 = constructor_fpuop1_sqrt(ctx, pattern0_0)?; let expr1_0 = constructor_fpu_rr(ctx, pattern0_0, &expr0_0, pattern1_0)?; return Some(expr1_0); @@ -6191,12 +6191,12 @@ pub fn constructor_sqrt_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> O pub fn constructor_fpuop1_neg(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32 { - // Rule at src/isa/s390x/inst.isle line 2757. + // Rule at src/isa/s390x/inst.isle line 2749. let expr0_0 = FPUOp1::Neg32; return Some(expr0_0); } if pattern0_0 == F64 { - // Rule at src/isa/s390x/inst.isle line 2758. + // Rule at src/isa/s390x/inst.isle line 2750. let expr0_0 = FPUOp1::Neg64; return Some(expr0_0); } @@ -6207,7 +6207,7 @@ pub fn constructor_fpuop1_neg(ctx: &mut C, arg0: Type) -> Option(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2761. + // Rule at src/isa/s390x/inst.isle line 2753. let expr0_0 = constructor_fpuop1_neg(ctx, pattern0_0)?; let expr1_0 = constructor_fpu_rr(ctx, pattern0_0, &expr0_0, pattern1_0)?; return Some(expr1_0); @@ -6217,12 +6217,12 @@ pub fn constructor_fneg_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> O pub fn constructor_fpuop1_abs(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32 { - // Rule at src/isa/s390x/inst.isle line 2767. + // Rule at src/isa/s390x/inst.isle line 2759. let expr0_0 = FPUOp1::Abs32; return Some(expr0_0); } if pattern0_0 == F64 { - // Rule at src/isa/s390x/inst.isle line 2768. + // Rule at src/isa/s390x/inst.isle line 2760. let expr0_0 = FPUOp1::Abs64; return Some(expr0_0); } @@ -6233,7 +6233,7 @@ pub fn constructor_fpuop1_abs(ctx: &mut C, arg0: Type) -> Option(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2771. + // Rule at src/isa/s390x/inst.isle line 2763. let expr0_0 = constructor_fpuop1_abs(ctx, pattern0_0)?; let expr1_0 = constructor_fpu_rr(ctx, pattern0_0, &expr0_0, pattern1_0)?; return Some(expr1_0); @@ -6243,12 +6243,12 @@ pub fn constructor_fabs_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> O pub fn constructor_fpuroundmode_ceil(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32 { - // Rule at src/isa/s390x/inst.isle line 2777. + // Rule at src/isa/s390x/inst.isle line 2769. let expr0_0 = FpuRoundMode::Plus32; return Some(expr0_0); } if pattern0_0 == F64 { - // Rule at src/isa/s390x/inst.isle line 2778. + // Rule at src/isa/s390x/inst.isle line 2770. let expr0_0 = FpuRoundMode::Plus64; return Some(expr0_0); } @@ -6259,7 +6259,7 @@ pub fn constructor_fpuroundmode_ceil(ctx: &mut C, arg0: Type) -> Opt pub fn constructor_ceil_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2781. + // Rule at src/isa/s390x/inst.isle line 2773. let expr0_0 = constructor_fpuroundmode_ceil(ctx, pattern0_0)?; let expr1_0 = constructor_fpu_round(ctx, pattern0_0, &expr0_0, pattern1_0)?; return Some(expr1_0); @@ -6269,12 +6269,12 @@ pub fn constructor_ceil_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> O pub fn constructor_fpuroundmode_floor(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32 { - // Rule at src/isa/s390x/inst.isle line 2787. + // Rule at src/isa/s390x/inst.isle line 2779. let expr0_0 = FpuRoundMode::Minus32; return Some(expr0_0); } if pattern0_0 == F64 { - // Rule at src/isa/s390x/inst.isle line 2788. + // Rule at src/isa/s390x/inst.isle line 2780. let expr0_0 = FpuRoundMode::Minus64; return Some(expr0_0); } @@ -6285,7 +6285,7 @@ pub fn constructor_fpuroundmode_floor(ctx: &mut C, arg0: Type) -> Op pub fn constructor_floor_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2791. + // Rule at src/isa/s390x/inst.isle line 2783. let expr0_0 = constructor_fpuroundmode_floor(ctx, pattern0_0)?; let expr1_0 = constructor_fpu_round(ctx, pattern0_0, &expr0_0, pattern1_0)?; return Some(expr1_0); @@ -6295,12 +6295,12 @@ pub fn constructor_floor_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> pub fn constructor_fpuroundmode_trunc(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32 { - // Rule at src/isa/s390x/inst.isle line 2797. + // Rule at src/isa/s390x/inst.isle line 2789. let expr0_0 = FpuRoundMode::Zero32; return Some(expr0_0); } if pattern0_0 == F64 { - // Rule at src/isa/s390x/inst.isle line 2798. + // Rule at src/isa/s390x/inst.isle line 2790. let expr0_0 = FpuRoundMode::Zero64; return Some(expr0_0); } @@ -6311,7 +6311,7 @@ pub fn constructor_fpuroundmode_trunc(ctx: &mut C, arg0: Type) -> Op pub fn constructor_trunc_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2801. + // Rule at src/isa/s390x/inst.isle line 2793. let expr0_0 = constructor_fpuroundmode_trunc(ctx, pattern0_0)?; let expr1_0 = constructor_fpu_round(ctx, pattern0_0, &expr0_0, pattern1_0)?; return Some(expr1_0); @@ -6324,12 +6324,12 @@ pub fn constructor_fpuroundmode_nearest( ) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32 { - // Rule at src/isa/s390x/inst.isle line 2807. + // Rule at src/isa/s390x/inst.isle line 2799. let expr0_0 = FpuRoundMode::Nearest32; return Some(expr0_0); } if pattern0_0 == F64 { - // Rule at src/isa/s390x/inst.isle line 2808. + // Rule at src/isa/s390x/inst.isle line 2800. let expr0_0 = FpuRoundMode::Nearest64; return Some(expr0_0); } @@ -6340,7 +6340,7 @@ pub fn constructor_fpuroundmode_nearest( pub fn constructor_nearest_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2811. + // Rule at src/isa/s390x/inst.isle line 2803. let expr0_0 = constructor_fpuroundmode_nearest(ctx, pattern0_0)?; let expr1_0 = constructor_fpu_round(ctx, pattern0_0, &expr0_0, pattern1_0)?; return Some(expr1_0); @@ -6356,7 +6356,7 @@ pub fn constructor_fpuop1_promote( if pattern0_0 == F64 { let pattern2_0 = arg1; if pattern2_0 == F32 { - // Rule at src/isa/s390x/inst.isle line 2817. + // Rule at src/isa/s390x/inst.isle line 2809. let expr0_0 = FPUOp1::Cvt32To64; return Some(expr0_0); } @@ -6374,7 +6374,7 @@ pub fn constructor_fpromote_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2820. + // Rule at src/isa/s390x/inst.isle line 2812. let expr0_0 = constructor_fpuop1_promote(ctx, pattern0_0, pattern1_0)?; let expr1_0 = constructor_fpu_rr(ctx, pattern0_0, &expr0_0, pattern2_0)?; return Some(expr1_0); @@ -6390,7 +6390,7 @@ pub fn constructor_fpuop1_demote( if pattern0_0 == F32 { let pattern2_0 = arg1; if pattern2_0 == F64 { - // Rule at src/isa/s390x/inst.isle line 2827. + // Rule at src/isa/s390x/inst.isle line 2819. let expr0_0 = FPUOp1::Cvt64To32; return Some(expr0_0); } @@ -6408,7 +6408,7 @@ pub fn constructor_fdemote_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2830. + // Rule at src/isa/s390x/inst.isle line 2822. let expr0_0 = constructor_fpuop1_demote(ctx, pattern0_0, pattern1_0)?; let expr1_0 = constructor_fpu_rr(ctx, pattern0_0, &expr0_0, pattern2_0)?; return Some(expr1_0); @@ -6424,12 +6424,12 @@ pub fn constructor_uint_to_fpu_op( if pattern0_0 == F32 { let pattern2_0 = arg1; if pattern2_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 2837. + // Rule at src/isa/s390x/inst.isle line 2829. let expr0_0 = IntToFpuOp::U32ToF32; return Some(expr0_0); } if pattern2_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2839. + // Rule at src/isa/s390x/inst.isle line 2831. let expr0_0 = IntToFpuOp::U64ToF32; return Some(expr0_0); } @@ -6437,12 +6437,12 @@ pub fn constructor_uint_to_fpu_op( if pattern0_0 == F64 { let pattern2_0 = arg1; if pattern2_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 2838. + // Rule at src/isa/s390x/inst.isle line 2830. let expr0_0 = IntToFpuOp::U32ToF64; return Some(expr0_0); } if pattern2_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2840. + // Rule at src/isa/s390x/inst.isle line 2832. let expr0_0 = IntToFpuOp::U64ToF64; return Some(expr0_0); } @@ -6460,7 +6460,7 @@ pub fn constructor_fcvt_from_uint_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2843. + // Rule at src/isa/s390x/inst.isle line 2835. let expr0_0 = constructor_uint_to_fpu_op(ctx, pattern0_0, pattern1_0)?; let expr1_0 = constructor_int_to_fpu(ctx, pattern0_0, &expr0_0, pattern2_0)?; return Some(expr1_0); @@ -6476,12 +6476,12 @@ pub fn constructor_sint_to_fpu_op( if pattern0_0 == F32 { let pattern2_0 = arg1; if pattern2_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 2850. + // Rule at src/isa/s390x/inst.isle line 2842. let expr0_0 = IntToFpuOp::I32ToF32; return Some(expr0_0); } if pattern2_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2852. + // Rule at src/isa/s390x/inst.isle line 2844. let expr0_0 = IntToFpuOp::I64ToF32; return Some(expr0_0); } @@ -6489,12 +6489,12 @@ pub fn constructor_sint_to_fpu_op( if pattern0_0 == F64 { let pattern2_0 = arg1; if pattern2_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 2851. + // Rule at src/isa/s390x/inst.isle line 2843. let expr0_0 = IntToFpuOp::I32ToF64; return Some(expr0_0); } if pattern2_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2853. + // Rule at src/isa/s390x/inst.isle line 2845. let expr0_0 = IntToFpuOp::I64ToF64; return Some(expr0_0); } @@ -6512,7 +6512,7 @@ pub fn constructor_fcvt_from_sint_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2856. + // Rule at src/isa/s390x/inst.isle line 2848. let expr0_0 = constructor_sint_to_fpu_op(ctx, pattern0_0, pattern1_0)?; let expr1_0 = constructor_int_to_fpu(ctx, pattern0_0, &expr0_0, pattern2_0)?; return Some(expr1_0); @@ -6528,12 +6528,12 @@ pub fn constructor_fpu_to_uint_op( if pattern0_0 == I32 { let pattern2_0 = arg1; if pattern2_0 == F32 { - // Rule at src/isa/s390x/inst.isle line 2863. + // Rule at src/isa/s390x/inst.isle line 2855. let expr0_0 = FpuToIntOp::F32ToU32; return Some(expr0_0); } if pattern2_0 == F64 { - // Rule at src/isa/s390x/inst.isle line 2864. + // Rule at src/isa/s390x/inst.isle line 2856. let expr0_0 = FpuToIntOp::F64ToU32; return Some(expr0_0); } @@ -6541,12 +6541,12 @@ pub fn constructor_fpu_to_uint_op( if pattern0_0 == I64 { let pattern2_0 = arg1; if pattern2_0 == F32 { - // Rule at src/isa/s390x/inst.isle line 2865. + // Rule at src/isa/s390x/inst.isle line 2857. let expr0_0 = FpuToIntOp::F32ToU64; return Some(expr0_0); } if pattern2_0 == F64 { - // Rule at src/isa/s390x/inst.isle line 2866. + // Rule at src/isa/s390x/inst.isle line 2858. let expr0_0 = FpuToIntOp::F64ToU64; return Some(expr0_0); } @@ -6564,7 +6564,7 @@ pub fn constructor_fcvt_to_uint_reg_with_flags( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2869. + // Rule at src/isa/s390x/inst.isle line 2861. let expr0_0 = constructor_fpu_to_uint_op(ctx, pattern0_0, pattern1_0)?; let expr1_0 = constructor_fpu_to_int(ctx, pattern0_0, &expr0_0, pattern2_0)?; return Some(expr1_0); @@ -6580,7 +6580,7 @@ pub fn constructor_fcvt_to_uint_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2873. + // Rule at src/isa/s390x/inst.isle line 2865. let expr0_0 = constructor_fcvt_to_uint_reg_with_flags(ctx, pattern0_0, pattern1_0, pattern2_0)?; let expr1_0 = constructor_drop_flags(ctx, &expr0_0)?; return Some(expr1_0); @@ -6596,12 +6596,12 @@ pub fn constructor_fpu_to_sint_op( if pattern0_0 == I32 { let pattern2_0 = arg1; if pattern2_0 == F32 { - // Rule at src/isa/s390x/inst.isle line 2880. + // Rule at src/isa/s390x/inst.isle line 2872. let expr0_0 = FpuToIntOp::F32ToI32; return Some(expr0_0); } if pattern2_0 == F64 { - // Rule at src/isa/s390x/inst.isle line 2881. + // Rule at src/isa/s390x/inst.isle line 2873. let expr0_0 = FpuToIntOp::F64ToI32; return Some(expr0_0); } @@ -6609,12 +6609,12 @@ pub fn constructor_fpu_to_sint_op( if pattern0_0 == I64 { let pattern2_0 = arg1; if pattern2_0 == F32 { - // Rule at src/isa/s390x/inst.isle line 2882. + // Rule at src/isa/s390x/inst.isle line 2874. let expr0_0 = FpuToIntOp::F32ToI64; return Some(expr0_0); } if pattern2_0 == F64 { - // Rule at src/isa/s390x/inst.isle line 2883. + // Rule at src/isa/s390x/inst.isle line 2875. let expr0_0 = FpuToIntOp::F64ToI64; return Some(expr0_0); } @@ -6632,7 +6632,7 @@ pub fn constructor_fcvt_to_sint_reg_with_flags( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2886. + // Rule at src/isa/s390x/inst.isle line 2878. let expr0_0 = constructor_fpu_to_sint_op(ctx, pattern0_0, pattern1_0)?; let expr1_0 = constructor_fpu_to_int(ctx, pattern0_0, &expr0_0, pattern2_0)?; return Some(expr1_0); @@ -6648,7 +6648,7 @@ pub fn constructor_fcvt_to_sint_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2890. + // Rule at src/isa/s390x/inst.isle line 2882. let expr0_0 = constructor_fcvt_to_sint_reg_with_flags(ctx, pattern0_0, pattern1_0, pattern2_0)?; let expr1_0 = constructor_drop_flags(ctx, &expr0_0)?; return Some(expr1_0); @@ -6658,12 +6658,12 @@ pub fn constructor_fcvt_to_sint_reg( pub fn constructor_cmpop_cmps(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 2897. + // Rule at src/isa/s390x/inst.isle line 2889. let expr0_0 = CmpOp::CmpS32; return Some(expr0_0); } if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2898. + // Rule at src/isa/s390x/inst.isle line 2890. let expr0_0 = CmpOp::CmpS64; return Some(expr0_0); } @@ -6674,12 +6674,12 @@ pub fn constructor_cmpop_cmps(ctx: &mut C, arg0: Type) -> Option(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 2901. + // Rule at src/isa/s390x/inst.isle line 2893. let expr0_0 = CmpOp::CmpS32Ext16; return Some(expr0_0); } if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2902. + // Rule at src/isa/s390x/inst.isle line 2894. let expr0_0 = CmpOp::CmpS64Ext16; return Some(expr0_0); } @@ -6690,7 +6690,7 @@ pub fn constructor_cmpop_cmps_sext16(ctx: &mut C, arg0: Type) -> Opt pub fn constructor_cmpop_cmps_sext32(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2905. + // Rule at src/isa/s390x/inst.isle line 2897. let expr0_0 = CmpOp::CmpS64Ext32; return Some(expr0_0); } @@ -6707,7 +6707,7 @@ pub fn constructor_icmps_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2908. + // Rule at src/isa/s390x/inst.isle line 2900. let expr0_0 = constructor_cmpop_cmps(ctx, pattern0_0)?; let expr1_0 = constructor_cmp_rr(ctx, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -6723,7 +6723,7 @@ pub fn constructor_icmps_reg_sext32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2911. + // Rule at src/isa/s390x/inst.isle line 2903. let expr0_0 = constructor_cmpop_cmps_sext32(ctx, pattern0_0)?; let expr1_0 = constructor_cmp_rr(ctx, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -6739,7 +6739,7 @@ pub fn constructor_icmps_simm16( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2914. + // Rule at src/isa/s390x/inst.isle line 2906. let expr0_0 = constructor_cmpop_cmps(ctx, pattern0_0)?; let expr1_0 = constructor_cmp_rsimm16(ctx, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -6755,7 +6755,7 @@ pub fn constructor_icmps_simm32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2917. + // Rule at src/isa/s390x/inst.isle line 2909. let expr0_0 = constructor_cmpop_cmps(ctx, pattern0_0)?; let expr1_0 = constructor_cmp_rsimm32(ctx, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -6771,7 +6771,7 @@ pub fn constructor_icmps_mem( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2920. + // Rule at src/isa/s390x/inst.isle line 2912. let expr0_0 = constructor_cmpop_cmps(ctx, pattern0_0)?; let expr1_0 = constructor_cmp_rx(ctx, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -6787,7 +6787,7 @@ pub fn constructor_icmps_mem_sext16( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2923. + // Rule at src/isa/s390x/inst.isle line 2915. let expr0_0 = constructor_cmpop_cmps_sext16(ctx, pattern0_0)?; let expr1_0 = constructor_cmp_rx(ctx, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -6803,7 +6803,7 @@ pub fn constructor_icmps_mem_sext32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2926. + // Rule at src/isa/s390x/inst.isle line 2918. let expr0_0 = constructor_cmpop_cmps_sext32(ctx, pattern0_0)?; let expr1_0 = constructor_cmp_rx(ctx, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -6813,12 +6813,12 @@ pub fn constructor_icmps_mem_sext32( pub fn constructor_cmpop_cmpu(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 2932. + // Rule at src/isa/s390x/inst.isle line 2924. let expr0_0 = CmpOp::CmpL32; return Some(expr0_0); } if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2933. + // Rule at src/isa/s390x/inst.isle line 2925. let expr0_0 = CmpOp::CmpL64; return Some(expr0_0); } @@ -6829,12 +6829,12 @@ pub fn constructor_cmpop_cmpu(ctx: &mut C, arg0: Type) -> Option(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 2936. + // Rule at src/isa/s390x/inst.isle line 2928. let expr0_0 = CmpOp::CmpL32Ext16; return Some(expr0_0); } if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2937. + // Rule at src/isa/s390x/inst.isle line 2929. let expr0_0 = CmpOp::CmpL64Ext16; return Some(expr0_0); } @@ -6845,7 +6845,7 @@ pub fn constructor_cmpop_cmpu_zext16(ctx: &mut C, arg0: Type) -> Opt pub fn constructor_cmpop_cmpu_zext32(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2940. + // Rule at src/isa/s390x/inst.isle line 2932. let expr0_0 = CmpOp::CmpL64Ext32; return Some(expr0_0); } @@ -6862,7 +6862,7 @@ pub fn constructor_icmpu_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2943. + // Rule at src/isa/s390x/inst.isle line 2935. let expr0_0 = constructor_cmpop_cmpu(ctx, pattern0_0)?; let expr1_0 = constructor_cmp_rr(ctx, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -6878,7 +6878,7 @@ pub fn constructor_icmpu_reg_zext32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2946. + // Rule at src/isa/s390x/inst.isle line 2938. let expr0_0 = constructor_cmpop_cmpu_zext32(ctx, pattern0_0)?; let expr1_0 = constructor_cmp_rr(ctx, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -6894,7 +6894,7 @@ pub fn constructor_icmpu_uimm32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2949. + // Rule at src/isa/s390x/inst.isle line 2941. let expr0_0 = constructor_cmpop_cmpu(ctx, pattern0_0)?; let expr1_0 = constructor_cmp_ruimm32(ctx, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -6910,7 +6910,7 @@ pub fn constructor_icmpu_mem( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2952. + // Rule at src/isa/s390x/inst.isle line 2944. let expr0_0 = constructor_cmpop_cmpu(ctx, pattern0_0)?; let expr1_0 = constructor_cmp_rx(ctx, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -6926,7 +6926,7 @@ pub fn constructor_icmpu_mem_zext16( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2955. + // Rule at src/isa/s390x/inst.isle line 2947. let expr0_0 = constructor_cmpop_cmpu_zext16(ctx, pattern0_0)?; let expr1_0 = constructor_cmp_rx(ctx, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -6942,7 +6942,7 @@ pub fn constructor_icmpu_mem_zext32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2958. + // Rule at src/isa/s390x/inst.isle line 2950. let expr0_0 = constructor_cmpop_cmpu_zext32(ctx, pattern0_0)?; let expr1_0 = constructor_cmp_rx(ctx, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -6959,14 +6959,14 @@ pub fn constructor_fcmp_reg( if pattern0_0 == F32 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2964. + // Rule at src/isa/s390x/inst.isle line 2956. let expr0_0 = constructor_fpu_cmp32(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if pattern0_0 == F64 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2965. + // Rule at src/isa/s390x/inst.isle line 2957. let expr0_0 = constructor_fpu_cmp64(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } diff --git a/cranelift/codegen/src/isa/x64/encoding/rex.rs b/cranelift/codegen/src/isa/x64/encoding/rex.rs index 17e5408cc1b9..509309205df5 100644 --- a/cranelift/codegen/src/isa/x64/encoding/rex.rs +++ b/cranelift/codegen/src/isa/x64/encoding/rex.rs @@ -47,7 +47,8 @@ pub(crate) fn encode_sib(shift: u8, enc_index: u8, enc_base: u8) -> u8 { /// Get the encoding number of a GPR. #[inline(always)] -pub(crate) fn int_reg_enc(reg: Reg) -> u8 { +pub(crate) fn int_reg_enc(reg: impl Into) -> u8 { + let reg = reg.into(); debug_assert!(reg.is_real()); debug_assert_eq!(reg.get_class(), RegClass::I64); reg.get_hw_encoding() @@ -55,7 +56,8 @@ pub(crate) fn int_reg_enc(reg: Reg) -> u8 { /// Get the encoding number of any register. #[inline(always)] -pub(crate) fn reg_enc(reg: Reg) -> u8 { +pub(crate) fn reg_enc(reg: impl Into) -> u8 { + let reg = reg.into(); debug_assert!(reg.is_real()); reg.get_hw_encoding() } diff --git a/cranelift/codegen/src/isa/x64/inst.isle b/cranelift/codegen/src/isa/x64/inst.isle index 8c52609cbe03..6f3081a4bbf1 100644 --- a/cranelift/codegen/src/isa/x64/inst.isle +++ b/cranelift/codegen/src/isa/x64/inst.isle @@ -2,82 +2,486 @@ ;;;; `MInst` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -(type MInst extern - (enum (Nop (len u8)) - (Ud2 (trap_code TrapCode)) - (AluRmiR (size OperandSize) - (op AluRmiROpcode) - (src1 Reg) - (src2 RegMemImm) - (dst WritableReg)) - (MulHi (size OperandSize) - (signed bool) - (src1 Reg) - (src2 RegMem) - (dst_lo WritableReg) - (dst_hi WritableReg)) - (XmmRmR (op SseOpcode) - (src1 Reg) - (src2 RegMem) - (dst WritableReg)) - (XmmUnaryRmR (op SseOpcode) - (src RegMem) - (dst WritableReg)) - (XmmUnaryRmREvex (op Avx512Opcode) - (src RegMem) - (dst WritableReg)) - (XmmRmiReg (opcode SseOpcode) - (src1 Reg) - (src2 RegMemImm) +;; Don't build `MInst` variants directly, in general. Instead, use the +;; instruction-emitting helpers defined further down. + +(type MInst nodebug + (enum + ;; Nops of various sizes, including zero. + (Nop (len u8)) + + ;; ========================================= + ;; Integer instructions. + + ;; Integer arithmetic/bit-twiddling. + (AluRmiR (size OperandSize) ;; 4 or 8 + (op AluRmiROpcode) + (src1 Reg) + (src2 RegMemImm) + (dst WritableReg)) + + ;; Instructions on general-purpose registers that only read src and + ;; defines dst (dst is not modified). `bsr`, etc. + (UnaryRmR (size OperandSize) ;; 2, 4, or 8 + (op UnaryRmROpcode) + (src GprMem) + (dst WritableGpr)) + + ;; Bitwise not. + (Not (size OperandSize) ;; 1, 2, 4, or 8 + (src Gpr) + (dst WritableGpr)) + + ;; Integer negation. + (Neg (size OperandSize) ;; 1, 2, 4, or 8 + (src Gpr) + (dst WritableGpr)) + + ;; Integer quotient and remainder: (div idiv) $rax $rdx (reg addr) + (Div (size OperandSize) ;; 1, 2, 4, or 8 + (signed bool) + (divisor RegMem) + (dividend Reg) + (dst_quotient WritableReg) + (dst_remainder WritableReg)) + + ;; The high (and low) bits of a (un)signed multiply: `RDX:RAX := RAX * + ;; rhs`. + (MulHi (size OperandSize) + (signed bool) + (src1 Reg) + (src2 RegMem) + (dst_lo WritableReg) + (dst_hi WritableReg)) + + ;; A synthetic sequence to implement the right inline checks for + ;; remainder and division, assuming the dividend is in %rax. + ;; + ;; Puts the result back into %rax if is_div, %rdx if !is_div, to mimic + ;; what the div instruction does. + ;; + ;; The generated code sequence is described in the emit's function match + ;; arm for this instruction. + ;; + ;; Note: %rdx is marked as modified by this instruction, to avoid an + ;; early clobber problem with the temporary and divisor registers. Make + ;; sure to zero %rdx right before this instruction, or you might run into + ;; regalloc failures where %rdx is live before its first def! + (CheckedDivOrRemSeq (kind DivOrRemKind) + (size OperandSize) + (dividend Reg) + ;; The divisor operand. Note it's marked as modified + ;; so that it gets assigned a register different from + ;; the temporary. + (divisor WritableReg) + (dst_quotient WritableReg) + (dst_remainder WritableReg) + (tmp OptionWritableReg)) + + ;; Do a sign-extend based on the sign of the value in rax into rdx: (cwd + ;; cdq cqo) or al into ah: (cbw) + (SignExtendData (size OperandSize) ;; 1, 2, 4, or 8 + (src Reg) (dst WritableReg)) - (XmmRmRImm (op SseOpcode) - (src1 Reg) - (src2 RegMem) - (dst WritableReg) - (imm u8) - (size OperandSize)) - (XmmUninitializedValue (dst WritableReg)) - (CmpRmiR (size OperandSize) - (opcode CmpOpcode) - (src RegMemImm) - (dst Reg)) - (Imm (dst_size OperandSize) - (simm64 u64) + + ;; Constant materialization: (imm32 imm64) reg. + ;; + ;; Either: movl $imm32, %reg32 or movabsq $imm64, %reg32. + (Imm (dst_size OperandSize) ;; 4 or 8 + (simm64 u64) + (dst WritableReg)) + + ;; GPR to GPR move: mov (64 32) reg reg. + (MovRR (size OperandSize) ;; 4 or 8 + (src Reg) + (dst WritableReg)) + + ;; Zero-extended loads, except for 64 bits: movz (bl bq wl wq lq) addr + ;; reg. + ;; + ;; Note that the lq variant doesn't really exist since the default + ;; zero-extend rule makes it unnecessary. For that case we emit the + ;; equivalent "movl AM, reg32". + (MovzxRmR (ext_mode ExtMode) + (src RegMem) (dst WritableReg)) - (ShiftR (size OperandSize) - (kind ShiftKind) - (src Reg) - (num_bits Imm8Reg) - (dst WritableReg)) - (MovzxRmR (ext_mode ExtMode) - (src RegMem) - (dst WritableReg)) - (MovsxRmR (ext_mode ExtMode) - (src RegMem) - (dst WritableReg)) - (Mov64MR (src SyntheticAmode) - (dst WritableReg)) - (Cmove (size OperandSize) - (cc CC) - (consequent RegMem) - (alternative Reg) - (dst WritableReg)) - (XmmRmREvex (op Avx512Opcode) - (src1 RegMem) - (src2 Reg) - (dst WritableReg)) - (GprToXmm (op SseOpcode) - (src RegMem) - (dst WritableReg) - (src_size OperandSize)) - (Not (size OperandSize) - (src Reg) + + ;; A plain 64-bit integer load, since MovZX_RM_R can't represent that. + (Mov64MR (src SyntheticAmode) + (dst WritableReg)) + + ;; Loads the memory address of addr into dst. + (LoadEffectiveAddress (addr SyntheticAmode) + (dst WritableGpr)) + + ;; Sign-extended loads and moves: movs (bl bq wl wq lq) addr reg. + (MovsxRmR (ext_mode ExtMode) + (src RegMem) (dst WritableReg)) - (Neg (size OperandSize) + + ;; Integer stores: mov (b w l q) reg addr. + (MovRM (size OperandSize) ;; 1, 2, 4, or 8 + (src Reg) + (dst SyntheticAmode)) + + ;; Arithmetic shifts: (shl shr sar) (b w l q) imm reg. + (ShiftR (size OperandSize) ;; 1, 2, 4, or 8 + (kind ShiftKind) + (src Reg) + ;; shift count: `Imm8Reg::Imm8(0 .. #bits-in-type - 1)` or + ;; `Imm8Reg::Reg(r)` where `r` get's move mitosis'd into `%cl`. + (num_bits Imm8Reg) + (dst WritableReg)) + + ;; Arithmetic SIMD shifts. + (XmmRmiReg (opcode SseOpcode) + (src1 Xmm) + (src2 XmmMemImm) + (dst WritableXmm)) + + ;; Integer comparisons/tests: cmp or test (b w l q) (reg addr imm) reg. + (CmpRmiR (size OperandSize) ;; 1, 2, 4, or 8 + (opcode CmpOpcode) + (src RegMemImm) + (dst Reg)) + + ;; Materializes the requested condition code in the destinaton reg. + (Setcc (cc CC) + (dst WritableReg)) + + ;; Integer conditional move. + ;; + ;; Overwrites the destination register. + (Cmove (size OperandSize) + (cc CC) + (consequent RegMem) + (alternative Reg) + (dst WritableReg)) + + ;; ========================================= + ;; Stack manipulation. + + ;; pushq (reg addr imm) + (Push64 (src RegMemImm)) + + ;; popq reg + (Pop64 (dst WritableReg)) + + ;; ========================================= + ;; Floating-point operations. + + ;; XMM (scalar or vector) binary op: (add sub and or xor mul adc? sbb?) + ;; (32 64) (reg addr) reg + (XmmRmR (op SseOpcode) + (src1 Xmm) + (src2 XmmMem) + (dst WritableXmm)) + + ;; XMM (scalar or vector) binary op that relies on the EVEX prefix. + (XmmRmREvex (op Avx512Opcode) + (src1 XmmMem) + (src2 Xmm) + (dst WritableXmm)) + + ;; XMM (scalar or vector) unary op: mov between XMM registers (32 64) + ;; (reg addr) reg, sqrt, etc. + ;; + ;; This differs from XMM_RM_R in that the dst register of XmmUnaryRmR is + ;; not used in the computation of the instruction dst value and so does + ;; not have to be a previously valid value. This is characteristic of mov + ;; instructions. + (XmmUnaryRmR (op SseOpcode) + (src XmmMem) + (dst WritableXmm)) + + ;; XMM (scalar or vector) unary op that relies on the EVEX prefix. + (XmmUnaryRmREvex (op Avx512Opcode) + (src XmmMem) + (dst WritableXmm)) + + ;; XMM (scalar or vector) unary op (from xmm to reg/mem): stores, movd, + ;; movq + (XmmMovRM (op SseOpcode) (src Reg) + (dst SyntheticAmode)) + + ;; XMM (vector) unary op (to move a constant value into an xmm register): + ;; movups + (XmmLoadConst (src VCodeConstant) + (dst WritableReg) + (ty Type)) + + ;; XMM (scalar) unary op (from xmm to integer reg): movd, movq, + ;; cvtts{s,d}2si + (XmmToGpr (op SseOpcode) + (src Xmm) + (dst WritableGpr) + (dst_size OperandSize)) + + ;; XMM (scalar) unary op (from integer to float reg): movd, movq, + ;; cvtsi2s{s,d} + (GprToXmm (op SseOpcode) + (src RegMem) + (dst WritableXmm) + (src_size OperandSize)) + + ;; Converts an unsigned int64 to a float32/float64. + (CvtUint64ToFloatSeq (dst_size OperandSize) ;; 4 or 8 + ;; A copy of the source register, fed by + ;; lowering. It is marked as modified during + ;; register allocation to make sure that the + ;; temporary registers differ from the src register, + ;; since both registers are live at the same time in + ;; the generated code sequence. + (src WritableGpr) + (dst WritableXmm) + (tmp_gpr1 WritableGpr) + (tmp_gpr2 WritableGpr)) + + ;; Converts a scalar xmm to a signed int32/int64. + (CvtFloatToSintSeq (dst_size OperandSize) + (src_size OperandSize) + (is_saturating bool) + ;; A copy of the source register, fed by + ;; lowering. It is marked as modified during + ;; register allocation to make sure that the + ;; temporary registers differ from the src register, + ;; since both registers are live at the same time in + ;; the generated code sequence. + (src WritableXmm) + (dst WritableGpr) + (tmp_gpr WritableGpr) + (tmp_xmm WritableXmm)) + + ;; Converts a scalar xmm to an unsigned int32/int64. + (CvtFloatToUintSeq (dst_size OperandSize) + (src_size OperandSize) + (is_saturating bool) + ;; A copy of the source register, fed by + ;; lowering. It is marked as modified during + ;; register allocation to make sure that the + ;; temporary registers differ from the src register, + ;; since both registers are live at the same time in + ;; the generated code sequence. + (src WritableXmm) + (dst WritableGpr) + (tmp_gpr WritableGpr) + (tmp_xmm WritableXmm)) + + ;; A sequence to compute min/max with the proper NaN semantics for xmm + ;; registers. + (XmmMinMaxSeq (size OperandSize) + (is_min bool) + (lhs Reg) + (rhs_dst WritableReg)) + + ;; XMM (scalar) conditional move. + ;; + ;; Overwrites the destination register if cc is set. + (XmmCmove (size OperandSize) + (cc CC) + (src RegMem) (dst WritableReg)) - (LoadEffectiveAddress (addr SyntheticAmode) - (dst WritableReg)))) + + ;; Float comparisons/tests: cmp (b w l q) (reg addr imm) reg. + (XmmCmpRmR (op SseOpcode) + (src RegMem) + (dst Reg)) + + ;; A binary XMM instruction with an 8-bit immediate: e.g. cmp (ps pd) imm + ;; (reg addr) reg + ;; + ;; Note: this has to use `Reg*`, not `Xmm*`, operands because it is used + ;; in various lane insertion and extraction instructions that move + ;; between XMMs and GPRs. + (XmmRmRImm (op SseOpcode) + (src1 Reg) + (src2 RegMem) + (dst WritableReg) + (imm u8) + (size OperandSize)) + + ;; ========================================= + ;; Control flow instructions. + + ;; Direct call: call simm32. + (CallKnown (dest ExternalName) + (uses VecReg) + (defs VecWritableReg) + (opcode Opcode)) + + ;; Indirect call: callq (reg mem) + (CallUnknown (dest RegMem) + (uses VecReg) + (defs VecWritableReg) + (opcode Opcode)) + + ;; Return. + (Ret) + + ;; A placeholder instruction, generating no code, meaning that a function + ;; epilogue must be inserted there. + (EpiloguePlaceholder) + + ;; Jump to a known target: jmp simm32. + (JmpKnown (dst MachLabel)) + + + ;; One-way conditional branch: jcond cond target. + ;; + ;; This instruction is useful when we have conditional jumps depending on + ;; more than two conditions, see for instance the lowering of Brz/brnz + ;; with Fcmp inputs. + ;; + ;; A note of caution: in contexts where the branch target is another + ;; block, this has to be the same successor as the one specified in the + ;; terminator branch of the current block. Otherwise, this might confuse + ;; register allocation by creating new invisible edges. + (JmpIf (cc CC) + (taken MachLabel)) + + ;; Two-way conditional branch: jcond cond target target. + ;; + ;; Emitted as a compound sequence; the MachBuffer will shrink it as + ;; appropriate. + (JmpCond (cc CC) + (taken MachLabel) + (not_taken MachLabel)) + + ;; Jump-table sequence, as one compound instruction (see note in lower.rs + ;; for rationale). + ;; + ;; The generated code sequence is described in the emit's function match + ;; arm for this instruction. + ;; + ;; See comment in lowering about the temporaries signedness. + (JmpTableSeq (idx Reg) + (tmp1 WritableReg) + (tmp2 WritableReg) + (default_target MachLabel) + (targets VecMachLabel) + (targets_for_term VecMachLabel)) + + ;; Indirect jump: jmpq (reg mem). + (JmpUnknown (target RegMem)) + + ;; Traps if the condition code is set. + (TrapIf (cc CC) + (trap_code TrapCode)) + + ;; A debug trap. + (Hlt) + + ;; An instruction that will always trigger the illegal instruction + ;; exception. + (Ud2 (trap_code TrapCode)) + + ;; Loads an external symbol in a register, with a relocation: + ;; + ;; movq $name@GOTPCREL(%rip), dst if PIC is enabled, or + ;; movabsq $name, dst otherwise. + (LoadExtName (dst WritableReg) + (name BoxExternalName) + (offset i64)) + + ;; ========================================= + ;; Instructions pertaining to atomic memory accesses. + + ;; A standard (native) `lock cmpxchg src, (amode)`, with register + ;; conventions: + ;; + ;; `mem` (read) address + ;; `replacement` (read) replacement value + ;; %rax (modified) in: expected value, out: value that was actually at `dst` + ;; %rflags is written. Do not assume anything about it after the instruction. + ;; + ;; The instruction "succeeded" iff the lowest `ty` bits of %rax + ;; afterwards are the same as they were before. + (LockCmpxchg (ty Type) ;; I8, I16, I32, or I64 + (replacement Reg) + (expected Reg) + (mem SyntheticAmode) + (dst_old WritableReg)) + + ;; A synthetic instruction, based on a loop around a native `lock + ;; cmpxchg` instruction. + ;; + ;; This atomically modifies a value in memory and returns the old value. + ;; The sequence consists of an initial "normal" load from `dst`, followed + ;; by a loop which computes the new value and tries to compare-and-swap + ;; ("CAS") it into `dst`, using the native instruction `lock + ;; cmpxchg{b,w,l,q}` . The loop iterates until the CAS is successful. + ;; If there is no contention, there will be only one pass through the + ;; loop body. The sequence does *not* perform any explicit memory fence + ;; instructions (mfence/sfence/lfence). + ;; + ;; Note that the transaction is atomic in the sense that, as observed by + ;; some other thread, `dst` either has the initial or final value, but no + ;; other. It isn't atomic in the sense of guaranteeing that no other + ;; thread writes to `dst` in between the initial load and the CAS -- but + ;; that would cause the CAS to fail unless the other thread's last write + ;; before the CAS wrote the same value that was already there. In other + ;; words, this implementation suffers (unavoidably) from the A-B-A + ;; problem. + ;; + ;; This instruction sequence has fixed register uses as follows: + ;; + ;; %r9 (read) address + ;; %r10 (read) second operand for `op` + ;; %r11 (written) scratch reg; value afterwards has no meaning + ;; %rax (written) the old value at %r9 + ;; %rflags is written. Do not assume anything about it after the instruction. + (AtomicRmwSeq (ty Type) ;; I8, I16, I32, or I64 + (op AtomicRmwOp) + (address Reg) + (operand Reg) + (temp WritableReg) + (dst_old WritableReg)) + + ;; A memory fence (mfence, lfence or sfence). + (Fence (kind FenceKind)) + + ;; ========================================= + ;; Meta-instructions generating no code. + + ;; Marker, no-op in generated code: SP "virtual offset" is adjusted. + ;; + ;; This controls how `MemArg::NominalSPOffset` args are lowered. + (VirtualSPOffsetAdj (offset i64)) + + ;; Provides a way to tell the register allocator that the upcoming + ;; sequence of instructions will overwrite `dst` so it should be + ;; considered as a `def`; use this with care. + ;; + ;; This is useful when we have a sequence of instructions whose register + ;; usages are nominally `mod`s, but such that the combination of + ;; operations creates a result that is independent of the initial + ;; register value. It's thus semantically a `def`, not a `mod`, when all + ;; the instructions are taken together, so we want to ensure the register + ;; is defined (its live-range starts) prior to the sequence to keep + ;; analyses happy. + ;; + ;; One alternative would be a compound instruction that somehow + ;; encapsulates the others and reports its own `def`s/`use`s/`mod`s; this + ;; adds complexity (the instruction list is no longer flat) and requires + ;; knowledge about semantics and initial-value independence anyway. + (XmmUninitializedValue (dst WritableXmm)) + + ;; A call to the `ElfTlsGetAddr` libcall. Returns address of TLS symbol + ;; in `rax`. + (ElfTlsGetAddr (symbol ExternalName)) + + ;; A Mach-O TLS symbol access. Returns address of the TLS symbol in + ;; `rax`. + (MachOTlsGetAddr (symbol ExternalName)) + + ;; A definition of a value label. + (ValueLabelMarker (reg Reg) + (label ValueLabel)) + + ;; An unwind pseudoinstruction describing the state of the machine at + ;; this program point. + (Unwind (inst UnwindInst)))) (type OperandSize extern (enum Size8 @@ -85,6 +489,13 @@ Size32 Size64)) +(type VCodeConstant (primitive VCodeConstant)) + +(type FenceKind extern + (enum MFence + LFence + SFence)) + ;; Get the `OperandSize` for a given `Type`, rounding smaller types up to 32 bits. (decl operand_size_of_type_32_64 (Type) OperandSize) (extern constructor operand_size_of_type_32_64 operand_size_of_type_32_64) @@ -112,6 +523,19 @@ And8 Or8)) +(type UnaryRmROpcode extern + (enum Bsr + Bsf + Lzcnt + Tzcnt + Popcnt)) + +(type DivOrRemKind extern + (enum SignedDiv + UnsignedDiv + SignedRem + UnsignedRem)) + (type SseOpcode extern (enum Addps Addpd @@ -330,7 +754,7 @@ (type Amode extern (enum)) -(decl amode_imm_reg_reg_shift (u32 Reg Reg u8) Amode) +(decl amode_imm_reg_reg_shift (u32 Gpr Gpr u8) Amode) (extern constructor amode_imm_reg_reg_shift amode_imm_reg_reg_shift) (decl amode_to_synthetic_amode (Amode) SyntheticAmode) @@ -397,13 +821,176 @@ (decl encode_fcmp_imm (FcmpImm) u8) (extern constructor encode_fcmp_imm encode_fcmp_imm) +;;;; Newtypes for Different Register Classes ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(type Gpr (primitive Gpr)) +(type WritableGpr (primitive WritableGpr)) +(type GprMem extern (enum)) +(type GprMemImm extern (enum)) + +(type Xmm (primitive Xmm)) +(type WritableXmm (primitive WritableXmm)) +(type XmmMem extern (enum)) +(type XmmMemImm extern (enum)) + +;; Convert a `WritableGpr` to a `WritableReg`. +(decl writable_gpr_to_reg (WritableGpr) WritableReg) +(extern constructor writable_gpr_to_reg writable_gpr_to_reg) + +;; Convert a `WritableXmm` to a `WritableReg`. +(decl writable_xmm_to_reg (WritableXmm) WritableReg) +(extern constructor writable_xmm_to_reg writable_xmm_to_reg) + +;; Convert a `WritableReg` to a `WritableXmm`. +(decl writable_reg_to_xmm (WritableReg) WritableXmm) +(extern constructor writable_reg_to_xmm writable_reg_to_xmm) + +;; Convert a `WritableXmm` to an `Xmm`. +(decl writable_xmm_to_xmm (WritableXmm) Xmm) +(extern constructor writable_xmm_to_xmm writable_xmm_to_xmm) + +;; Convert a `WritableGpr` to an `Gpr`. +(decl writable_gpr_to_gpr (WritableGpr) Gpr) +(extern constructor writable_gpr_to_gpr writable_gpr_to_gpr) + +;; Convert an `Gpr` to a `Reg`. +(decl gpr_to_reg (Gpr) Reg) +(extern constructor gpr_to_reg gpr_to_reg) + +;; Convert an `Xmm` to a `Reg`. +(decl xmm_to_reg (Xmm) Reg) +(extern constructor xmm_to_reg xmm_to_reg) + +;; Convert an `Xmm` into an `XmmMemImm`. +(decl xmm_to_xmm_mem_imm (Xmm) XmmMemImm) +(extern constructor xmm_to_xmm_mem_imm xmm_to_xmm_mem_imm) + +;; Allocate a new temporary GPR register. +(decl temp_writable_gpr () WritableGpr) +(extern constructor temp_writable_gpr temp_writable_gpr) + +;; Allocate a new temporary XMM register. +(decl temp_writable_xmm () WritableXmm) +(extern constructor temp_writable_xmm temp_writable_xmm) + +;; Construct a new `XmmMem` from the given `RegMem`. +;; +;; Asserts that the `RegMem`'s register, if any, is an XMM register. +(decl xmm_mem_new (RegMem) XmmMem) +(extern constructor xmm_mem_new xmm_mem_new) + +;; Construct a new `GprMemImm` from the given `RegMemImm`. +;; +;; Asserts that the `RegMemImm`'s register, if any, is an GPR register. +(decl gpr_mem_imm_new (RegMemImm) GprMemImm) +(extern constructor gpr_mem_imm_new gpr_mem_imm_new) + +;; Construct a new `XmmMemImm` from the given `RegMemImm`. +;; +;; Asserts that the `RegMemImm`'s register, if any, is an XMM register. +(decl xmm_mem_imm_new (RegMemImm) XmmMemImm) +(extern constructor xmm_mem_imm_new xmm_mem_imm_new) + +;; Construct a new `XmmMem` from an `Xmm`. +(decl xmm_to_xmm_mem (Xmm) XmmMem) +(extern constructor xmm_to_xmm_mem xmm_to_xmm_mem) + +;; Construct a new `XmmMem` from an `RegMem`. +(decl xmm_mem_to_reg_mem (XmmMem) RegMem) +(extern constructor xmm_mem_to_reg_mem xmm_mem_to_reg_mem) + +;; Convert a `GprMem` to a `RegMem`. +(decl gpr_mem_to_reg_mem (GprMem) RegMem) +(extern constructor gpr_mem_to_reg_mem gpr_mem_to_reg_mem) + +;; Construct a new `Xmm` from a `Reg`. +;; +;; Asserts that the register is a XMM. +(decl xmm_new (Reg) Xmm) +(extern constructor xmm_new xmm_new) + +;; Construct a new `Gpr` from a `Reg`. +;; +;; Asserts that the register is a GPR. +(decl gpr_new (Reg) Gpr) +(extern constructor gpr_new gpr_new) + +;; Construct a new `GprMem` from a `RegMem`. +;; +;; Asserts that the `RegMem`'s register, if any, is a GPR. +(decl gpr_mem_new (RegMem) GprMem) +(extern constructor gpr_mem_new gpr_mem_new) + +;; Construct a `GprMem` from a `Reg`. +;; +;; Asserts that the `Reg` is a GPR. +(decl reg_to_gpr_mem (Reg) GprMem) +(extern constructor reg_to_gpr_mem reg_to_gpr_mem) + +;; Put a value into a GPR. +;; +;; Asserts that the value goes into a GPR. +(decl put_in_gpr (Value) Gpr) +(rule (put_in_gpr val) + (gpr_new (put_in_reg val))) + +;; Put a value into a `GprMem`. +;; +;; Asserts that the value goes into a GPR. +(decl put_in_gpr_mem (Value) GprMem) +(rule (put_in_gpr_mem val) + (gpr_mem_new (put_in_reg_mem val))) + +;; Put a value into a `GprMemImm`. +;; +;; Asserts that the value goes into a GPR. +(decl put_in_gpr_mem_imm (Value) GprMemImm) +(rule (put_in_gpr_mem_imm val) + (gpr_mem_imm_new (put_in_reg_mem_imm val))) + +;; Put a value into a XMM. +;; +;; Asserts that the value goes into a XMM. +(decl put_in_xmm (Value) Xmm) +(rule (put_in_xmm val) + (xmm_new (put_in_reg val))) + +;; Put a value into a `XmmMem`. +;; +;; Asserts that the value goes into a XMM. +(decl put_in_xmm_mem (Value) XmmMem) +(rule (put_in_xmm_mem val) + (xmm_mem_new (put_in_reg_mem val))) + +;; Put a value into a `XmmMemImm`. +;; +;; Asserts that the value goes into a XMM. +(decl put_in_xmm_mem_imm (Value) XmmMemImm) +(rule (put_in_xmm_mem_imm val) + (xmm_mem_imm_new (put_in_reg_mem_imm val))) + +;; Construct a `ValueRegs` out of a single GPR register. +(decl value_gpr (Gpr) ValueRegs) +(rule (value_gpr x) + (value_reg (gpr_to_reg x))) + +;; Construct a `ValueRegs` out of two GPR registers. +(decl value_gprs (Gpr Gpr) ValueRegs) +(rule (value_gprs x y) + (value_regs (gpr_to_reg x) (gpr_to_reg y))) + +;; Construct a `ValueRegs` out of a single XMM register. +(decl value_xmm (Xmm) ValueRegs) +(rule (value_xmm x) + (value_reg (xmm_to_reg x))) + ;;;; Helpers for Getting Particular Physical Registers ;;;;;;;;;;;;;;;;;;;;;;;;; ;; ;; These should only be used for legalization purposes, when we can't otherwise ;; rely on something like `Inst::mov_mitosis` to put an operand into the ;; appropriate physical register for whatever reason. -(decl xmm0 () WritableReg) +(decl xmm0 () WritableXmm) (extern constructor xmm0 xmm0) ;;;; Helpers for Querying Enabled ISA Extensions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -507,8 +1094,8 @@ (rule (sse_xor_op $F64X2) (SseOpcode.Xorpd)) (rule (sse_xor_op (multi_lane _bits _lanes)) (SseOpcode.Pxor)) -;; Performs an xor operation of the two operands specified -(decl sse_xor (Type Reg RegMem) Reg) +;; Performs an xor operation of the two operands specified. +(decl sse_xor (Type Xmm XmmMem) Xmm) (rule (sse_xor ty x y) (xmm_rm_r ty (sse_xor_op ty) x y)) ;; Determine the appropriate operation to compare two vectors of the specified @@ -532,45 +1119,46 @@ ;; then it runs the risk of comparing NaN against NaN and not actually producing ;; an all-ones mask. By using integer comparision operations we're guaranteeed ;; that everything is equal to itself. -(decl vector_all_ones (Type) Reg) +(decl vector_all_ones (Type) Xmm) (rule (vector_all_ones ty) - (let ((wr WritableReg (temp_writable_reg ty)) - (r Reg (writable_reg_to_reg wr)) + (let ((wr WritableXmm (temp_writable_xmm)) + (r Xmm (writable_xmm_to_xmm wr)) (_ Unit (emit (MInst.XmmRmR (sse_cmp_op $I32X4) r - (RegMem.Reg r) + (xmm_to_xmm_mem r) wr)))) r)) ;; Helper for creating an SSE register holding an `i64x2` from two `i64` values. -(decl make_i64x2_from_lanes (RegMem RegMem) Reg) +(decl make_i64x2_from_lanes (GprMem GprMem) Xmm) (rule (make_i64x2_from_lanes lo hi) - (let ((dst_w WritableReg (temp_writable_reg $I64X2)) - (dst_r Reg (writable_reg_to_reg dst_w)) - (_0 Unit (emit (MInst.XmmUninitializedValue dst_w))) + (let ((dst_xmm_w WritableXmm (temp_writable_xmm)) + (dst_reg_w WritableReg (writable_xmm_to_reg dst_xmm_w)) + (dst_xmm_r Xmm (writable_xmm_to_xmm dst_xmm_w)) + (dst_reg_r Reg (xmm_to_reg dst_xmm_r)) + (_0 Unit (emit (MInst.XmmUninitializedValue dst_xmm_w))) (_1 Unit (emit (MInst.XmmRmRImm (SseOpcode.Pinsrd) - dst_r - lo - dst_w + dst_reg_r + (gpr_mem_to_reg_mem lo) + dst_reg_w 0 (OperandSize.Size64)))) (_2 Unit (emit (MInst.XmmRmRImm (SseOpcode.Pinsrd) - dst_r - hi - dst_w + dst_reg_r + (gpr_mem_to_reg_mem hi) + dst_reg_w 1 (OperandSize.Size64))))) - dst_r)) + dst_xmm_r)) ;; Move a `RegMemImm.Reg` operand to an XMM register, if necessary. -(decl reg_mem_imm_to_xmm (RegMemImm) RegMemImm) -(rule (reg_mem_imm_to_xmm rmi @ (RegMemImm.Mem _)) rmi) -(rule (reg_mem_imm_to_xmm rmi @ (RegMemImm.Imm _)) rmi) -(rule (reg_mem_imm_to_xmm (RegMemImm.Reg r)) - (RegMemImm.Reg (gpr_to_xmm $I8X16 - (SseOpcode.Movd) - (RegMem.Reg r) - (OperandSize.Size32)))) +(decl mov_rmi_to_xmm (RegMemImm) XmmMemImm) +(rule (mov_rmi_to_xmm rmi @ (RegMemImm.Mem _)) (xmm_mem_imm_new rmi)) +(rule (mov_rmi_to_xmm rmi @ (RegMemImm.Imm _)) (xmm_mem_imm_new rmi)) +(rule (mov_rmi_to_xmm (RegMemImm.Reg r)) + (xmm_to_xmm_mem_imm (gpr_to_xmm (SseOpcode.Movd) + (reg_to_gpr_mem r) + (OperandSize.Size32)))) ;;;; Helpers for Emitting Loads ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -588,24 +1176,24 @@ (writable_reg_to_reg dst))) (rule (x64_load $F32 addr _ext_kind) - (xmm_unary_rm_r (SseOpcode.Movss) - (synthetic_amode_to_reg_mem addr))) + (xmm_to_reg (xmm_unary_rm_r (SseOpcode.Movss) + (xmm_mem_new (synthetic_amode_to_reg_mem addr))))) (rule (x64_load $F64 addr _ext_kind) - (xmm_unary_rm_r (SseOpcode.Movsd) - (synthetic_amode_to_reg_mem addr))) + (xmm_to_reg (xmm_unary_rm_r (SseOpcode.Movsd) + (xmm_mem_new (synthetic_amode_to_reg_mem addr))))) (rule (x64_load $F32X4 addr _ext_kind) - (xmm_unary_rm_r (SseOpcode.Movups) - (synthetic_amode_to_reg_mem addr))) + (xmm_to_reg (xmm_unary_rm_r (SseOpcode.Movups) + (xmm_mem_new (synthetic_amode_to_reg_mem addr))))) (rule (x64_load $F64X2 addr _ext_kind) - (xmm_unary_rm_r (SseOpcode.Movupd) - (synthetic_amode_to_reg_mem addr))) + (xmm_to_reg (xmm_unary_rm_r (SseOpcode.Movupd) + (xmm_mem_new (synthetic_amode_to_reg_mem addr))))) (rule (x64_load (multi_lane _bits _lanes) addr _ext_kind) - (xmm_unary_rm_r (SseOpcode.Movdqu) - (synthetic_amode_to_reg_mem addr))) + (xmm_to_reg (xmm_unary_rm_r (SseOpcode.Movdqu) + (xmm_mem_new (synthetic_amode_to_reg_mem addr))))) ;;;; Instruction Constructors ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; @@ -728,11 +1316,15 @@ ;; `f32` immediates. (rule (imm $F32 bits) - (gpr_to_xmm $F32 (SseOpcode.Movd) (RegMem.Reg (imm $I32 bits)) (OperandSize.Size32))) + (xmm_to_reg (gpr_to_xmm (SseOpcode.Movd) + (gpr_mem_new (RegMem.Reg (imm $I32 bits))) + (OperandSize.Size32)))) ;; `f64` immediates. (rule (imm $F64 bits) - (gpr_to_xmm $F64 (SseOpcode.Movq) (RegMem.Reg (imm $I64 bits)) (OperandSize.Size64))) + (xmm_to_reg (gpr_to_xmm (SseOpcode.Movq) + (gpr_mem_new (RegMem.Reg (imm $I64 bits))) + (OperandSize.Size64)))) (decl nonzero_u64_fits_in_u32 (u64) u64) (extern extractor nonzero_u64_fits_in_u32 nonzero_u64_fits_in_u32) @@ -759,35 +1351,35 @@ ;; Special case for zero immediates with vector types, they turn into an xor ;; specific to the vector type. (rule (imm ty @ (multi_lane _bits _lanes) 0) - (let ((wr WritableReg (temp_writable_reg ty)) - (r Reg (writable_reg_to_reg wr)) + (let ((wr WritableXmm (temp_writable_xmm)) + (r Xmm (writable_xmm_to_xmm wr)) (_ Unit (emit (MInst.XmmRmR (sse_xor_op ty) r - (RegMem.Reg r) + (xmm_to_xmm_mem r) wr)))) - r)) + (xmm_to_reg r))) ;; Special case for `f32` zero immediates to use `xorps`. (rule (imm $F32 0) - (let ((wr WritableReg (temp_writable_reg $F32)) - (r Reg (writable_reg_to_reg wr)) + (let ((wr WritableXmm (temp_writable_xmm)) + (r Xmm (writable_xmm_to_xmm wr)) (_ Unit (emit (MInst.XmmRmR (SseOpcode.Xorps) r - (RegMem.Reg r) + (xmm_to_xmm_mem r) wr)))) - r)) + (xmm_to_reg r))) ;; TODO: use cmpeqps for all 1s ;; Special case for `f64` zero immediates to use `xorpd`. (rule (imm $F64 0) - (let ((wr WritableReg (temp_writable_reg $F64)) - (r Reg (writable_reg_to_reg wr)) + (let ((wr WritableXmm (temp_writable_xmm)) + (r Xmm (writable_xmm_to_xmm wr)) (_ Unit (emit (MInst.XmmRmR (SseOpcode.Xorpd) r - (RegMem.Reg r) + (xmm_to_xmm_mem r) wr)))) - r)) + (xmm_to_reg r))) ;; TODO: use cmpeqpd for all 1s @@ -870,199 +1462,199 @@ (writable_reg_to_reg dst))) ;; Helper for creating `MInst.XmmRmR` instructions. -(decl xmm_rm_r (Type SseOpcode Reg RegMem) Reg) +(decl xmm_rm_r (Type SseOpcode Xmm XmmMem) Xmm) (rule (xmm_rm_r ty op src1 src2) - (let ((dst WritableReg (temp_writable_reg ty)) + (let ((dst WritableXmm (temp_writable_xmm)) (_ Unit (emit (MInst.XmmRmR op src1 src2 dst)))) - (writable_reg_to_reg dst))) + (writable_xmm_to_xmm dst))) ;; Helper for creating `paddb` instructions. -(decl paddb (Reg RegMem) Reg) +(decl paddb (Xmm XmmMem) Xmm) (rule (paddb src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Paddb) src1 src2)) ;; Helper for creating `paddw` instructions. -(decl paddw (Reg RegMem) Reg) +(decl paddw (Xmm XmmMem) Xmm) (rule (paddw src1 src2) (xmm_rm_r $I16X8 (SseOpcode.Paddw) src1 src2)) ;; Helper for creating `paddd` instructions. -(decl paddd (Reg RegMem) Reg) +(decl paddd (Xmm XmmMem) Xmm) (rule (paddd src1 src2) (xmm_rm_r $I32X4 (SseOpcode.Paddd) src1 src2)) ;; Helper for creating `paddq` instructions. -(decl paddq (Reg RegMem) Reg) +(decl paddq (Xmm XmmMem) Xmm) (rule (paddq src1 src2) (xmm_rm_r $I64X2 (SseOpcode.Paddq) src1 src2)) ;; Helper for creating `paddsb` instructions. -(decl paddsb (Reg RegMem) Reg) +(decl paddsb (Xmm XmmMem) Xmm) (rule (paddsb src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Paddsb) src1 src2)) ;; Helper for creating `paddsw` instructions. -(decl paddsw (Reg RegMem) Reg) +(decl paddsw (Xmm XmmMem) Xmm) (rule (paddsw src1 src2) (xmm_rm_r $I16X8 (SseOpcode.Paddsw) src1 src2)) ;; Helper for creating `paddusb` instructions. -(decl paddusb (Reg RegMem) Reg) +(decl paddusb (Xmm XmmMem) Xmm) (rule (paddusb src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Paddusb) src1 src2)) ;; Helper for creating `paddusw` instructions. -(decl paddusw (Reg RegMem) Reg) +(decl paddusw (Xmm XmmMem) Xmm) (rule (paddusw src1 src2) (xmm_rm_r $I16X8 (SseOpcode.Paddusw) src1 src2)) ;; Helper for creating `psubb` instructions. -(decl psubb (Reg RegMem) Reg) +(decl psubb (Xmm XmmMem) Xmm) (rule (psubb src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Psubb) src1 src2)) ;; Helper for creating `psubw` instructions. -(decl psubw (Reg RegMem) Reg) +(decl psubw (Xmm XmmMem) Xmm) (rule (psubw src1 src2) (xmm_rm_r $I16X8 (SseOpcode.Psubw) src1 src2)) ;; Helper for creating `psubd` instructions. -(decl psubd (Reg RegMem) Reg) +(decl psubd (Xmm XmmMem) Xmm) (rule (psubd src1 src2) (xmm_rm_r $I32X4 (SseOpcode.Psubd) src1 src2)) ;; Helper for creating `psubq` instructions. -(decl psubq (Reg RegMem) Reg) +(decl psubq (Xmm XmmMem) Xmm) (rule (psubq src1 src2) (xmm_rm_r $I64X2 (SseOpcode.Psubq) src1 src2)) ;; Helper for creating `psubsb` instructions. -(decl psubsb (Reg RegMem) Reg) +(decl psubsb (Xmm XmmMem) Xmm) (rule (psubsb src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Psubsb) src1 src2)) ;; Helper for creating `psubsw` instructions. -(decl psubsw (Reg RegMem) Reg) +(decl psubsw (Xmm XmmMem) Xmm) (rule (psubsw src1 src2) (xmm_rm_r $I16X8 (SseOpcode.Psubsw) src1 src2)) ;; Helper for creating `psubusb` instructions. -(decl psubusb (Reg RegMem) Reg) +(decl psubusb (Xmm XmmMem) Xmm) (rule (psubusb src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Psubusb) src1 src2)) ;; Helper for creating `psubusw` instructions. -(decl psubusw (Reg RegMem) Reg) +(decl psubusw (Xmm XmmMem) Xmm) (rule (psubusw src1 src2) (xmm_rm_r $I16X8 (SseOpcode.Psubusw) src1 src2)) ;; Helper for creating `pavgb` instructions. -(decl pavgb (Reg RegMem) Reg) +(decl pavgb (Xmm XmmMem) Xmm) (rule (pavgb src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Pavgb) src1 src2)) ;; Helper for creating `pavgw` instructions. -(decl pavgw (Reg RegMem) Reg) +(decl pavgw (Xmm XmmMem) Xmm) (rule (pavgw src1 src2) (xmm_rm_r $I16X8 (SseOpcode.Pavgw) src1 src2)) ;; Helper for creating `pand` instructions. -(decl pand (Reg RegMem) Reg) +(decl pand (Xmm XmmMem) Xmm) (rule (pand src1 src2) (xmm_rm_r $F32X4 (SseOpcode.Pand) src1 src2)) ;; Helper for creating `andps` instructions. -(decl andps (Reg RegMem) Reg) +(decl andps (Xmm XmmMem) Xmm) (rule (andps src1 src2) (xmm_rm_r $F32X4 (SseOpcode.Andps) src1 src2)) ;; Helper for creating `andpd` instructions. -(decl andpd (Reg RegMem) Reg) +(decl andpd (Xmm XmmMem) Xmm) (rule (andpd src1 src2) (xmm_rm_r $F64X2 (SseOpcode.Andpd) src1 src2)) ;; Helper for creating `por` instructions. -(decl por (Reg RegMem) Reg) +(decl por (Xmm XmmMem) Xmm) (rule (por src1 src2) (xmm_rm_r $F32X4 (SseOpcode.Por) src1 src2)) ;; Helper for creating `orps` instructions. -(decl orps (Reg RegMem) Reg) +(decl orps (Xmm XmmMem) Xmm) (rule (orps src1 src2) (xmm_rm_r $F32X4 (SseOpcode.Orps) src1 src2)) ;; Helper for creating `orpd` instructions. -(decl orpd (Reg RegMem) Reg) +(decl orpd (Xmm XmmMem) Xmm) (rule (orpd src1 src2) (xmm_rm_r $F64X2 (SseOpcode.Orpd) src1 src2)) ;; Helper for creating `pxor` instructions. -(decl pxor (Reg RegMem) Reg) +(decl pxor (Xmm XmmMem) Xmm) (rule (pxor src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Pxor) src1 src2)) ;; Helper for creating `xorps` instructions. -(decl xorps (Reg RegMem) Reg) +(decl xorps (Xmm XmmMem) Xmm) (rule (xorps src1 src2) (xmm_rm_r $F32X4 (SseOpcode.Xorps) src1 src2)) ;; Helper for creating `xorpd` instructions. -(decl xorpd (Reg RegMem) Reg) +(decl xorpd (Xmm XmmMem) Xmm) (rule (xorpd src1 src2) (xmm_rm_r $F64X2 (SseOpcode.Xorpd) src1 src2)) ;; Helper for creating `pmullw` instructions. -(decl pmullw (Reg RegMem) Reg) +(decl pmullw (Xmm XmmMem) Xmm) (rule (pmullw src1 src2) (xmm_rm_r $I16X8 (SseOpcode.Pmullw) src1 src2)) ;; Helper for creating `pmulld` instructions. -(decl pmulld (Reg RegMem) Reg) +(decl pmulld (Xmm XmmMem) Xmm) (rule (pmulld src1 src2) (xmm_rm_r $I16X8 (SseOpcode.Pmulld) src1 src2)) ;; Helper for creating `pmulhw` instructions. -(decl pmulhw (Reg RegMem) Reg) +(decl pmulhw (Xmm XmmMem) Xmm) (rule (pmulhw src1 src2) (xmm_rm_r $I16X8 (SseOpcode.Pmulhw) src1 src2)) ;; Helper for creating `pmulhuw` instructions. -(decl pmulhuw (Reg RegMem) Reg) +(decl pmulhuw (Xmm XmmMem) Xmm) (rule (pmulhuw src1 src2) (xmm_rm_r $I16X8 (SseOpcode.Pmulhuw) src1 src2)) ;; Helper for creating `pmuldq` instructions. -(decl pmuldq (Reg RegMem) Reg) +(decl pmuldq (Xmm XmmMem) Xmm) (rule (pmuldq src1 src2) (xmm_rm_r $I16X8 (SseOpcode.Pmuldq) src1 src2)) ;; Helper for creating `pmuludq` instructions. -(decl pmuludq (Reg RegMem) Reg) +(decl pmuludq (Xmm XmmMem) Xmm) (rule (pmuludq src1 src2) (xmm_rm_r $I64X2 (SseOpcode.Pmuludq) src1 src2)) ;; Helper for creating `punpckhwd` instructions. -(decl punpckhwd (Reg RegMem) Reg) +(decl punpckhwd (Xmm XmmMem) Xmm) (rule (punpckhwd src1 src2) (xmm_rm_r $I16X8 (SseOpcode.Punpckhwd) src1 src2)) ;; Helper for creating `punpcklwd` instructions. -(decl punpcklwd (Reg RegMem) Reg) +(decl punpcklwd (Xmm XmmMem) Xmm) (rule (punpcklwd src1 src2) (xmm_rm_r $I16X8 (SseOpcode.Punpcklwd) src1 src2)) ;; Helper for creating `andnps` instructions. -(decl andnps (Reg RegMem) Reg) +(decl andnps (Xmm XmmMem) Xmm) (rule (andnps src1 src2) (xmm_rm_r $F32X4 (SseOpcode.Andnps) src1 src2)) ;; Helper for creating `andnpd` instructions. -(decl andnpd (Reg RegMem) Reg) +(decl andnpd (Xmm XmmMem) Xmm) (rule (andnpd src1 src2) (xmm_rm_r $F64X2 (SseOpcode.Andnpd) src1 src2)) ;; Helper for creating `pandn` instructions. -(decl pandn (Reg RegMem) Reg) +(decl pandn (Xmm XmmMem) Xmm) (rule (pandn src1 src2) (xmm_rm_r $F64X2 (SseOpcode.Pandn) src1 src2)) @@ -1077,367 +1669,387 @@ (rule (sse_mov_op (multi_lane _bits _lanes)) (SseOpcode.Movdqa)) ;; Helper for creating `blendvp{d,s}` and `pblendvb` instructions. -(decl sse_blend (Type RegMem RegMem Reg) Reg) +(decl sse_blend (Type XmmMem XmmMem Xmm) Xmm) (rule (sse_blend ty mask src1 src2) ;; Move the mask into `xmm0`, as blend instructions implicitly operate on ;; that register. (This kind of thing would normally happen inside of ;; `Inst::mov_mitosis`, but has to happen here, where we still have the ;; mask register, because the mask is implicit and doesn't appear in the ;; `Inst` itself.) - (let ((mask2 WritableReg (xmm0)) - (_ Unit (emit (MInst.XmmUnaryRmR (sse_mov_op ty) mask mask2)))) + (let ((mask2 WritableXmm (xmm0)) + (_ Unit (emit (MInst.XmmUnaryRmR (sse_mov_op ty) + mask + mask2)))) (xmm_rm_r ty (sse_blend_op ty) src2 src1))) ;; Helper for creating `blendvpd` instructions. -(decl blendvpd (Reg RegMem Reg) Reg) +(decl blendvpd (Xmm XmmMem Xmm) Xmm) (rule (blendvpd src1 src2 mask) ;; Move the mask into `xmm0`, as `blendvpd` implicitly operates on that ;; register. (This kind of thing would normally happen inside of ;; `Inst::mov_mitosis`, but has to happen here, where we still have the ;; mask register, because the mask is implicit and doesn't appear in the ;; `Inst` itself.) - (let ((mask2 WritableReg (xmm0)) - (_ Unit (emit (MInst.XmmUnaryRmR (SseOpcode.Movapd) (RegMem.Reg mask) mask2)))) + (let ((mask2 WritableXmm (xmm0)) + (_ Unit (emit (MInst.XmmUnaryRmR (SseOpcode.Movapd) + (xmm_to_xmm_mem mask) + mask2)))) (xmm_rm_r $F64X2 (SseOpcode.Blendvpd) src1 src2))) ;; Helper for creating `movsd` instructions. -(decl movsd (Reg RegMem) Reg) +(decl movsd (Xmm XmmMem) Xmm) (rule (movsd src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Movsd) src1 src2)) ;; Helper for creating `movlhps` instructions. -(decl movlhps (Reg RegMem) Reg) +(decl movlhps (Xmm XmmMem) Xmm) (rule (movlhps src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Movlhps) src1 src2)) ;; Helper for creating `pmaxsb` instructions. -(decl pmaxsb (Reg RegMem) Reg) +(decl pmaxsb (Xmm XmmMem) Xmm) (rule (pmaxsb src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Pmaxsb) src1 src2)) ;; Helper for creating `pmaxsw` instructions. -(decl pmaxsw (Reg RegMem) Reg) +(decl pmaxsw (Xmm XmmMem) Xmm) (rule (pmaxsw src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Pmaxsw) src1 src2)) ;; Helper for creating `pmaxsd` instructions. -(decl pmaxsd (Reg RegMem) Reg) +(decl pmaxsd (Xmm XmmMem) Xmm) (rule (pmaxsd src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Pmaxsd) src1 src2)) ;; Helper for creating `pminsb` instructions. -(decl pminsb (Reg RegMem) Reg) +(decl pminsb (Xmm XmmMem) Xmm) (rule (pminsb src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Pminsb) src1 src2)) ;; Helper for creating `pminsw` instructions. -(decl pminsw (Reg RegMem) Reg) +(decl pminsw (Xmm XmmMem) Xmm) (rule (pminsw src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Pminsw) src1 src2)) ;; Helper for creating `pminsd` instructions. -(decl pminsd (Reg RegMem) Reg) +(decl pminsd (Xmm XmmMem) Xmm) (rule (pminsd src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Pminsd) src1 src2)) ;; Helper for creating `pmaxub` instructions. -(decl pmaxub (Reg RegMem) Reg) +(decl pmaxub (Xmm XmmMem) Xmm) (rule (pmaxub src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Pmaxub) src1 src2)) ;; Helper for creating `pmaxuw` instructions. -(decl pmaxuw (Reg RegMem) Reg) +(decl pmaxuw (Xmm XmmMem) Xmm) (rule (pmaxuw src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Pmaxuw) src1 src2)) ;; Helper for creating `pmaxud` instructions. -(decl pmaxud (Reg RegMem) Reg) +(decl pmaxud (Xmm XmmMem) Xmm) (rule (pmaxud src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Pmaxud) src1 src2)) ;; Helper for creating `pminub` instructions. -(decl pminub (Reg RegMem) Reg) +(decl pminub (Xmm XmmMem) Xmm) (rule (pminub src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Pminub) src1 src2)) ;; Helper for creating `pminuw` instructions. -(decl pminuw (Reg RegMem) Reg) +(decl pminuw (Xmm XmmMem) Xmm) (rule (pminuw src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Pminuw) src1 src2)) ;; Helper for creating `pminud` instructions. -(decl pminud (Reg RegMem) Reg) +(decl pminud (Xmm XmmMem) Xmm) (rule (pminud src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Pminud) src1 src2)) ;; Helper for creating `punpcklbw` instructions. -(decl punpcklbw (Reg RegMem) Reg) +(decl punpcklbw (Xmm XmmMem) Xmm) (rule (punpcklbw src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Punpcklbw) src1 src2)) ;; Helper for creating `punpckhbw` instructions. -(decl punpckhbw (Reg RegMem) Reg) +(decl punpckhbw (Xmm XmmMem) Xmm) (rule (punpckhbw src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Punpckhbw) src1 src2)) ;; Helper for creating `packsswb` instructions. -(decl packsswb (Reg RegMem) Reg) +(decl packsswb (Xmm XmmMem) Xmm) (rule (packsswb src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Packsswb) src1 src2)) ;; Helper for creating `MInst.XmmRmRImm` instructions. -(decl xmm_rm_r_imm (SseOpcode Reg RegMem u8 OperandSize) Reg) +(decl xmm_rm_r_imm (SseOpcode Reg RegMem u8 OperandSize) Xmm) (rule (xmm_rm_r_imm op src1 src2 imm size) - (let ((dst WritableReg (temp_writable_reg $I8X16)) + (let ((dst WritableXmm (temp_writable_xmm)) (_ Unit (emit (MInst.XmmRmRImm op src1 src2 - dst + (writable_xmm_to_reg dst) imm size)))) - (writable_reg_to_reg dst))) + (writable_xmm_to_xmm dst))) ;; Helper for creating `palignr` instructions. -(decl palignr (Reg RegMem u8 OperandSize) Reg) +(decl palignr (Xmm XmmMem u8 OperandSize) Xmm) (rule (palignr src1 src2 imm size) (xmm_rm_r_imm (SseOpcode.Palignr) - src1 - src2 + (xmm_to_reg src1) + (xmm_mem_to_reg_mem src2) imm size)) +;; Helper for creating `cmpps` instructions. +(decl cmpps (Xmm XmmMem FcmpImm) Xmm) +(rule (cmpps src1 src2 imm) + (xmm_rm_r_imm (SseOpcode.Cmpps) + (xmm_to_reg src1) + (xmm_mem_to_reg_mem src2) + (encode_fcmp_imm imm) + (OperandSize.Size32))) + +;; Helper for creating `pinsrb` instructions. +(decl pinsrb (Xmm GprMem u8) Xmm) +(rule (pinsrb src1 src2 lane) + (xmm_rm_r_imm (SseOpcode.Pinsrb) + (xmm_to_reg src1) + (gpr_mem_to_reg_mem src2) + lane + (OperandSize.Size32))) + +;; Helper for creating `pinsrw` instructions. +(decl pinsrw (Xmm GprMem u8) Xmm) +(rule (pinsrw src1 src2 lane) + (xmm_rm_r_imm (SseOpcode.Pinsrw) + (xmm_to_reg src1) + (gpr_mem_to_reg_mem src2) + lane + (OperandSize.Size32))) + +;; Helper for creating `pinsrd` instructions. +(decl pinsrd (Xmm GprMem u8 OperandSize) Xmm) +(rule (pinsrd src1 src2 lane size) + (xmm_rm_r_imm (SseOpcode.Pinsrd) + (xmm_to_reg src1) + (gpr_mem_to_reg_mem src2) + lane + size)) + +;; Helper for creating `insertps` instructions. +(decl insertps (Xmm XmmMem u8) Xmm) +(rule (insertps src1 src2 lane) + (xmm_rm_r_imm (SseOpcode.Insertps) + (xmm_to_reg src1) + (xmm_mem_to_reg_mem src2) + lane + (OperandSize.Size32))) + ;; Helper for creating `pshufd` instructions. -(decl pshufd (RegMem u8 OperandSize) Reg) +(decl pshufd (XmmMem u8 OperandSize) Xmm) (rule (pshufd src imm size) - (let ((w_dst WritableReg (temp_writable_reg $I8X16)) - (dst Reg (writable_reg_to_reg w_dst)) + (let ((w_dst WritableXmm (temp_writable_xmm)) + (dst Xmm (writable_xmm_to_xmm w_dst)) (_ Unit (emit (MInst.XmmRmRImm (SseOpcode.Pshufd) - dst - src - w_dst + (xmm_to_reg dst) + (xmm_mem_to_reg_mem src) + (writable_xmm_to_reg w_dst) imm size)))) dst)) ;; Helper for creating `MInst.XmmUnaryRmR` instructions. -(decl xmm_unary_rm_r (SseOpcode RegMem) Reg) +(decl xmm_unary_rm_r (SseOpcode XmmMem) Xmm) (rule (xmm_unary_rm_r op src) - (let ((dst WritableReg (temp_writable_reg $I8X16)) + (let ((dst WritableXmm (temp_writable_xmm)) (_ Unit (emit (MInst.XmmUnaryRmR op src dst)))) - (writable_reg_to_reg dst))) + (writable_xmm_to_xmm dst))) ;; Helper for creating `pmovsxbw` instructions. -(decl pmovsxbw (RegMem) Reg) +(decl pmovsxbw (XmmMem) Xmm) (rule (pmovsxbw src) (xmm_unary_rm_r (SseOpcode.Pmovsxbw) src)) ;; Helper for creating `pmovzxbw` instructions. -(decl pmovzxbw (RegMem) Reg) +(decl pmovzxbw (XmmMem) Xmm) (rule (pmovzxbw src) (xmm_unary_rm_r (SseOpcode.Pmovzxbw) src)) ;; Helper for creating `pabsb` instructions. -(decl pabsb (RegMem) Reg) +(decl pabsb (XmmMem) Xmm) (rule (pabsb src) (xmm_unary_rm_r (SseOpcode.Pabsb) src)) ;; Helper for creating `pabsw` instructions. -(decl pabsw (RegMem) Reg) +(decl pabsw (XmmMem) Xmm) (rule (pabsw src) (xmm_unary_rm_r (SseOpcode.Pabsw) src)) ;; Helper for creating `pabsd` instructions. -(decl pabsd (RegMem) Reg) +(decl pabsd (XmmMem) Xmm) (rule (pabsd src) (xmm_unary_rm_r (SseOpcode.Pabsd) src)) ;; Helper for creating `MInst.XmmUnaryRmREvex` instructions. -(decl xmm_unary_rm_r_evex (Avx512Opcode RegMem) Reg) +(decl xmm_unary_rm_r_evex (Avx512Opcode XmmMem) Xmm) (rule (xmm_unary_rm_r_evex op src) - (let ((dst WritableReg (temp_writable_reg $I8X16)) + (let ((dst WritableXmm (temp_writable_xmm)) (_ Unit (emit (MInst.XmmUnaryRmREvex op src dst)))) - (writable_reg_to_reg dst))) + (writable_xmm_to_xmm dst))) ;; Helper for creating `vpabsq` instructions. -(decl vpabsq (RegMem) Reg) +(decl vpabsq (XmmMem) Xmm) (rule (vpabsq src) (xmm_unary_rm_r_evex (Avx512Opcode.Vpabsq) src)) ;; Helper for creating `MInst.XmmRmREvex` instructions. -(decl xmm_rm_r_evex (Avx512Opcode RegMem Reg) Reg) +(decl xmm_rm_r_evex (Avx512Opcode XmmMem Xmm) Xmm) (rule (xmm_rm_r_evex op src1 src2) - (let ((dst WritableReg (temp_writable_reg $I8X16)) + (let ((dst WritableXmm (temp_writable_xmm)) (_ Unit (emit (MInst.XmmRmREvex op src1 src2 dst)))) - (writable_reg_to_reg dst))) + (writable_xmm_to_xmm dst))) ;; Helper for creating `vpmullq` instructions. ;; ;; Requires AVX-512 vl and dq. -(decl vpmullq (RegMem Reg) Reg) +(decl vpmullq (XmmMem Xmm) Xmm) (rule (vpmullq src1 src2) (xmm_rm_r_evex (Avx512Opcode.Vpmullq) src1 src2)) -;; Helper for creating `MInst.XmmRmiReg` instructions. -(decl xmm_rmi_reg (SseOpcode Reg RegMemImm) Reg) -(rule (xmm_rmi_reg op src1 src2) - (let ((dst WritableReg (temp_writable_reg $I8X16)) +;; Helper for creating `MInst.MulHi` instructions. +;; +;; Returns the (lo, hi) register halves of the multiplication. +(decl mul_hi (Type bool Reg RegMem) ValueRegs) +(rule (mul_hi ty signed src1 src2) + (let ((dst_lo WritableReg (temp_writable_reg ty)) + (dst_hi WritableReg (temp_writable_reg ty)) + (size OperandSize (operand_size_of_type_32_64 ty)) + (_ Unit (emit (MInst.MulHi size + signed + src1 + src2 + dst_lo + dst_hi)))) + (value_regs (writable_reg_to_reg dst_lo) + (writable_reg_to_reg dst_hi)))) + +;; Helper for creating `mul` instructions that return both the lower and +;; (unsigned) higher halves of the result. +(decl mulhi_u (Type Reg RegMem) ValueRegs) +(rule (mulhi_u ty src1 src2) + (mul_hi ty $false src1 src2)) + +;; Helper for creating `MInst.XmmRmiXmm` instructions. +(decl xmm_rmi_xmm (SseOpcode Xmm XmmMemImm) Xmm) +(rule (xmm_rmi_xmm op src1 src2) + (let ((dst WritableXmm (temp_writable_xmm)) (_ Unit (emit (MInst.XmmRmiReg op src1 src2 dst)))) - (writable_reg_to_reg dst))) + (writable_xmm_to_xmm dst))) ;; Helper for creating `psllw` instructions. -(decl psllw (Reg RegMemImm) Reg) +(decl psllw (Xmm XmmMemImm) Xmm) (rule (psllw src1 src2) - (xmm_rmi_reg (SseOpcode.Psllw) src1 src2)) + (xmm_rmi_xmm (SseOpcode.Psllw) src1 src2)) ;; Helper for creating `pslld` instructions. -(decl pslld (Reg RegMemImm) Reg) +(decl pslld (Xmm XmmMemImm) Xmm) (rule (pslld src1 src2) - (xmm_rmi_reg (SseOpcode.Pslld) src1 src2)) + (xmm_rmi_xmm (SseOpcode.Pslld) src1 src2)) ;; Helper for creating `psllq` instructions. -(decl psllq (Reg RegMemImm) Reg) +(decl psllq (Xmm XmmMemImm) Xmm) (rule (psllq src1 src2) - (xmm_rmi_reg (SseOpcode.Psllq) src1 src2)) + (xmm_rmi_xmm (SseOpcode.Psllq) src1 src2)) ;; Helper for creating `psrlw` instructions. -(decl psrlw (Reg RegMemImm) Reg) +(decl psrlw (Xmm XmmMemImm) Xmm) (rule (psrlw src1 src2) - (xmm_rmi_reg (SseOpcode.Psrlw) src1 src2)) + (xmm_rmi_xmm (SseOpcode.Psrlw) src1 src2)) ;; Helper for creating `psrld` instructions. -(decl psrld (Reg RegMemImm) Reg) +(decl psrld (Xmm XmmMemImm) Xmm) (rule (psrld src1 src2) - (xmm_rmi_reg (SseOpcode.Psrld) src1 src2)) + (xmm_rmi_xmm (SseOpcode.Psrld) src1 src2)) ;; Helper for creating `psrlq` instructions. -(decl psrlq (Reg RegMemImm) Reg) +(decl psrlq (Xmm XmmMemImm) Xmm) (rule (psrlq src1 src2) - (xmm_rmi_reg (SseOpcode.Psrlq) src1 src2)) + (xmm_rmi_xmm (SseOpcode.Psrlq) src1 src2)) ;; Helper for creating `psraw` instructions. -(decl psraw (Reg RegMemImm) Reg) +(decl psraw (Xmm XmmMemImm) Xmm) (rule (psraw src1 src2) - (xmm_rmi_reg (SseOpcode.Psraw) src1 src2)) + (xmm_rmi_xmm (SseOpcode.Psraw) src1 src2)) ;; Helper for creating `psrad` instructions. -(decl psrad (Reg RegMemImm) Reg) +(decl psrad (Xmm XmmMemImm) Xmm) (rule (psrad src1 src2) - (xmm_rmi_reg (SseOpcode.Psrad) src1 src2)) - -;; Helper for creating `MInst.MulHi` instructions. -;; -;; Returns the (lo, hi) register halves of the multiplication. -(decl mul_hi (Type bool Reg RegMem) ValueRegs) -(rule (mul_hi ty signed src1 src2) - (let ((dst_lo WritableReg (temp_writable_reg ty)) - (dst_hi WritableReg (temp_writable_reg ty)) - (size OperandSize (operand_size_of_type_32_64 ty)) - (_ Unit (emit (MInst.MulHi size - signed - src1 - src2 - dst_lo - dst_hi)))) - (value_regs (writable_reg_to_reg dst_lo) - (writable_reg_to_reg dst_hi)))) + (xmm_rmi_xmm (SseOpcode.Psrad) src1 src2)) -;; Helper for creating `mul` instructions that return both the lower and -;; (unsigned) higher halves of the result. -(decl mulhi_u (Type Reg RegMem) ValueRegs) -(rule (mulhi_u ty src1 src2) - (mul_hi ty $false src1 src2)) - -;; Helper for creating `cmpps` instructions. -(decl cmpps (Reg RegMem FcmpImm) Reg) -(rule (cmpps src1 src2 imm) - (xmm_rm_r_imm (SseOpcode.Cmpps) - src1 - src2 - (encode_fcmp_imm imm) - (OperandSize.Size32))) +;; Helper for creating `pextrd` instructions. +(decl pextrd (Type Xmm u8) Gpr) +(rule (pextrd ty src lane) + (let ((w_dst WritableGpr (temp_writable_gpr)) + (r_dst Gpr (writable_gpr_to_gpr w_dst)) + (_ Unit (emit (MInst.XmmRmRImm (SseOpcode.Pextrd) + (gpr_to_reg r_dst) + (RegMem.Reg (xmm_to_reg src)) + (writable_gpr_to_reg w_dst) + lane + (operand_size_of_type_32_64 (lane_type ty)))))) + r_dst)) ;; Helper for creating `cmppd` instructions. ;; ;; Note that `Size32` is intentional despite this being used for 64-bit ;; operations, since this presumably induces the correct encoding of the ;; instruction. -(decl cmppd (Reg RegMem FcmpImm) Reg) +(decl cmppd (Xmm XmmMem FcmpImm) Xmm) (rule (cmppd src1 src2 imm) (xmm_rm_r_imm (SseOpcode.Cmppd) - src1 - src2 + (xmm_to_reg src1) + (xmm_mem_to_reg_mem src2) (encode_fcmp_imm imm) (OperandSize.Size32))) ;; Helper for creating `MInst.GprToXmm` instructions. -(decl gpr_to_xmm (Type SseOpcode RegMem OperandSize) Reg) -(rule (gpr_to_xmm ty op src size) - (let ((dst WritableReg (temp_writable_reg ty)) - (_ Unit (emit (MInst.GprToXmm op src dst size)))) - (writable_reg_to_reg dst))) - -;; Helper for creating `pinsrb` instructions. -(decl pinsrb (Reg RegMem u8) Reg) -(rule (pinsrb src1 src2 lane) - (xmm_rm_r_imm (SseOpcode.Pinsrb) src1 src2 lane (OperandSize.Size32))) - -;; Helper for creating `pinsrw` instructions. -(decl pinsrw (Reg RegMem u8) Reg) -(rule (pinsrw src1 src2 lane) - (xmm_rm_r_imm (SseOpcode.Pinsrw) src1 src2 lane (OperandSize.Size32))) - -;; Helper for creating `pinsrd` instructions. -(decl pinsrd (Reg RegMem u8 OperandSize) Reg) -(rule (pinsrd src1 src2 lane size) - (xmm_rm_r_imm (SseOpcode.Pinsrd) src1 src2 lane size)) - -;; Helper for creating `insertps` instructions. -(decl insertps (Reg RegMem u8) Reg) -(rule (insertps src1 src2 lane) - (xmm_rm_r_imm (SseOpcode.Insertps) src1 src2 lane (OperandSize.Size32))) - -;; Helper for creating `pextrd` instructions. -(decl pextrd (Type Reg u8) Reg) -(rule (pextrd ty src lane) - (let ((w_dst WritableReg (temp_writable_reg ty)) - (r_dst Reg (writable_reg_to_reg w_dst)) - (_ Unit (emit (MInst.XmmRmRImm (SseOpcode.Pextrd) - r_dst - (RegMem.Reg src) - w_dst - lane - (operand_size_of_type_32_64 (lane_type ty)))))) - r_dst)) +(decl gpr_to_xmm (SseOpcode GprMem OperandSize) Xmm) +(rule (gpr_to_xmm op src size) + (let ((dst WritableXmm (temp_writable_xmm)) + (_ Unit (emit (MInst.GprToXmm op (gpr_mem_to_reg_mem src) dst size)))) + (writable_xmm_to_xmm dst))) ;; Helper for creating `not` instructions. -(decl not (Type Reg) Reg) +(decl not (Type Gpr) Gpr) (rule (not ty src) - (let ((dst WritableReg (temp_writable_reg ty)) + (let ((dst WritableGpr (temp_writable_gpr)) (size OperandSize (operand_size_of_type_32_64 ty)) (_ Unit (emit (MInst.Not size src dst)))) - (writable_reg_to_reg dst))) + (writable_gpr_to_gpr dst))) ;; Helper for creating `neg` instructions. -(decl neg (Type Reg) Reg) +(decl neg (Type Gpr) Gpr) (rule (neg ty src) - (let ((dst WritableReg (temp_writable_reg ty)) + (let ((dst WritableGpr (temp_writable_gpr)) (size OperandSize (operand_size_of_type_32_64 ty)) (_ Unit (emit (MInst.Neg size src dst)))) - (writable_reg_to_reg dst))) + (writable_gpr_to_gpr dst))) -(decl lea (SyntheticAmode) Reg) +(decl lea (SyntheticAmode) Gpr) (rule (lea addr) - (let ((dst WritableReg (temp_writable_reg $I64)) + (let ((dst WritableGpr (temp_writable_gpr)) (_ Unit (emit (MInst.LoadEffectiveAddress addr dst)))) - (writable_reg_to_reg dst))) + (writable_gpr_to_gpr dst))) ;; Helper for creating `ud2` instructions. (decl ud2 (TrapCode) SideEffectNoResult) diff --git a/cranelift/codegen/src/isa/x64/inst/args.rs b/cranelift/codegen/src/isa/x64/inst/args.rs index 9b9cf2fd0ca9..e27dc3130ec6 100644 --- a/cranelift/codegen/src/isa/x64/inst/args.rs +++ b/cranelift/codegen/src/isa/x64/inst/args.rs @@ -13,6 +13,309 @@ use smallvec::{smallvec, SmallVec}; use std::fmt; use std::string::String; +/// An extenstion trait for converting `Writable{Xmm,Gpr}` to `Writable`. +pub trait ToWritableReg { + fn to_writable_reg(&self) -> Writable; +} + +/// An extension trait for converting `Writable` to `Writable{Xmm,Gpr}`. +pub trait FromWritableReg: Sized { + fn from_writable_reg(w: Writable) -> Option; +} + +/// An extension trait for mapping register uses on `{Xmm,Gpr}`. +pub trait MapUseExt { + fn map_use(&mut self, mapper: &RM) + where + RM: RegMapper; +} + +/// An extension trait for mapping register mods and defs on +/// `Writable{Xmm,Gpr}`. +pub trait MapDefModExt { + fn map_def(&mut self, mapper: &RM) + where + RM: RegMapper; + + fn map_mod(&mut self, mapper: &RM) + where + RM: RegMapper; +} + +/// A macro for defining a newtype of `Reg` that enforces some invariant about +/// the wrapped `Reg` (such as that it is of a particular register class). +macro_rules! newtype_of_reg { + ( + $newtype_reg:ident, + $newtype_writable_reg:ident, + $newtype_reg_mem:ident, + $newtype_reg_mem_imm:ident, + |$check_reg:ident| $check:expr + ) => { + /// A newtype wrapper around `Reg`. + #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] + pub struct $newtype_reg(Reg); + + impl PartialEq for $newtype_reg { + fn eq(&self, other: &Reg) -> bool { + self.0 == *other + } + } + + impl From<$newtype_reg> for Reg { + fn from(r: $newtype_reg) -> Self { + r.0 + } + } + + impl PrettyPrint for $newtype_reg { + fn show_rru(&self, mb_rru: Option<&RealRegUniverse>) -> String { + self.0.show_rru(mb_rru) + } + } + + impl $newtype_reg { + /// Create this newtype from the given register, or return `None` if the register + /// is not a valid instance of this newtype. + pub fn new($check_reg: Reg) -> Option { + if $check { + Some(Self($check_reg)) + } else { + None + } + } + + /// Get this newtype's underlying `Reg`. + pub fn to_reg(self) -> Reg { + self.0 + } + } + + // Convenience impl so that people working with this newtype can use it + // "just like" a plain `Reg`. + // + // NB: We cannot implement `DerefMut` because that would let people do + // nasty stuff like `*my_gpr.deref_mut() = some_xmm_reg`, breaking the + // invariants that `Gpr` provides. + impl std::ops::Deref for $newtype_reg { + type Target = Reg; + + fn deref(&self) -> &Reg { + &self.0 + } + } + + impl MapUseExt for $newtype_reg { + fn map_use(&mut self, mapper: &RM) + where + RM: RegMapper, + { + let mut reg = self.0; + mapper.map_use(&mut reg); + debug_assert!({ + let $check_reg = reg; + $check + }); + *self = $newtype_reg(reg); + } + } + + pub type $newtype_writable_reg = Writable<$newtype_reg>; + + impl ToWritableReg for $newtype_writable_reg { + fn to_writable_reg(&self) -> Writable { + Writable::from_reg(self.to_reg().to_reg()) + } + } + + impl FromWritableReg for $newtype_writable_reg { + fn from_writable_reg(w: Writable) -> Option { + Some(Writable::from_reg($newtype_reg::new(w.to_reg())?)) + } + } + + impl MapDefModExt for $newtype_writable_reg { + fn map_def(&mut self, mapper: &RM) + where + RM: RegMapper, + { + let mut reg = self.to_writable_reg(); + mapper.map_def(&mut reg); + debug_assert!({ + let $check_reg = reg.to_reg(); + $check + }); + *self = Writable::from_reg($newtype_reg(reg.to_reg())); + } + + fn map_mod(&mut self, mapper: &RM) + where + RM: RegMapper, + { + let mut reg = self.to_writable_reg(); + mapper.map_mod(&mut reg); + debug_assert!({ + let $check_reg = reg.to_reg(); + $check + }); + *self = Writable::from_reg($newtype_reg(reg.to_reg())); + } + } + + /// A newtype wrapper around `RegMem` for general-purpose registers. + #[derive(Clone, Debug)] + pub struct $newtype_reg_mem(RegMem); + + impl From<$newtype_reg_mem> for RegMem { + fn from(rm: $newtype_reg_mem) -> Self { + rm.0 + } + } + + impl From<$newtype_reg> for $newtype_reg_mem { + fn from(r: $newtype_reg) -> Self { + $newtype_reg_mem(RegMem::reg(r.into())) + } + } + + impl $newtype_reg_mem { + /// Construct a `RegMem` newtype from the given `RegMem`, or return + /// `None` if the `RegMem` is not a valid instance of this `RegMem` + /// newtype. + pub fn new(rm: RegMem) -> Option { + match rm { + RegMem::Mem { addr: _ } => Some(Self(rm)), + RegMem::Reg { reg: $check_reg } if $check => Some(Self(rm)), + RegMem::Reg { reg: _ } => None, + } + } + + /// Convert this newtype into its underlying `RegMem`. + pub fn to_reg_mem(self) -> RegMem { + self.0 + } + + #[allow(dead_code)] // Used by some newtypes and not others. + pub fn map_uses(&mut self, mapper: &RM) + where + RM: RegMapper, + { + self.0.map_uses(mapper); + debug_assert!(match self.0 { + RegMem::Reg { reg: $check_reg } => $check, + _ => true, + }); + } + + #[allow(dead_code)] // Used by some newtypes and not others. + pub fn map_as_def(&mut self, mapper: &RM) + where + RM: RegMapper, + { + self.0.map_as_def(mapper); + debug_assert!(match self.0 { + RegMem::Reg { reg: $check_reg } => $check, + _ => true, + }); + } + } + + impl PrettyPrint for $newtype_reg_mem { + fn show_rru(&self, mb_rru: Option<&RealRegUniverse>) -> String { + self.0.show_rru(mb_rru) + } + } + + impl PrettyPrintSized for $newtype_reg_mem { + fn show_rru_sized(&self, mb_rru: Option<&RealRegUniverse>, size: u8) -> String { + self.0.show_rru_sized(mb_rru, size) + } + } + + /// A newtype wrapper around `RegMemImm`. + #[derive(Clone, Debug)] + pub struct $newtype_reg_mem_imm(RegMemImm); + + impl From<$newtype_reg_mem_imm> for RegMemImm { + fn from(rmi: $newtype_reg_mem_imm) -> RegMemImm { + rmi.0 + } + } + + impl From<$newtype_reg> for $newtype_reg_mem_imm { + fn from(r: $newtype_reg) -> Self { + $newtype_reg_mem_imm(RegMemImm::reg(r.into())) + } + } + + impl $newtype_reg_mem_imm { + /// Construct this newtype from the given `RegMemImm`, or return + /// `None` if the `RegMemImm` is not a valid instance of this + /// newtype. + pub fn new(rmi: RegMemImm) -> Option { + match rmi { + RegMemImm::Imm { .. } => Some(Self(rmi)), + RegMemImm::Mem { addr: _ } => Some(Self(rmi)), + RegMemImm::Reg { reg: $check_reg } if $check => Some(Self(rmi)), + RegMemImm::Reg { reg: _ } => None, + } + } + + /// Convert this newtype into its underlying `RegMemImm`. + #[allow(dead_code)] // Used by some newtypes and not others. + pub fn to_reg_mem_imm(self) -> RegMemImm { + self.0 + } + + #[allow(dead_code)] // Used by some newtypes and not others. + pub fn map_uses(&mut self, mapper: &RM) + where + RM: RegMapper, + { + self.0.map_uses(mapper); + debug_assert!(match self.0 { + RegMemImm::Reg { reg: $check_reg } => $check, + _ => true, + }); + } + + #[allow(dead_code)] // Used by some newtypes and not others. + pub fn map_as_def(&mut self, mapper: &RM) + where + RM: RegMapper, + { + self.0.map_as_def(mapper); + debug_assert!(match self.0 { + RegMemImm::Reg { reg: $check_reg } => $check, + _ => true, + }); + } + } + + impl PrettyPrint for $newtype_reg_mem_imm { + fn show_rru(&self, mb_rru: Option<&RealRegUniverse>) -> String { + self.0.show_rru(mb_rru) + } + } + + impl PrettyPrintSized for $newtype_reg_mem_imm { + fn show_rru_sized(&self, mb_rru: Option<&RealRegUniverse>, size: u8) -> String { + self.0.show_rru_sized(mb_rru, size) + } + } + }; +} + +// Define a newtype of `Reg` for general-purpose registers. +newtype_of_reg!(Gpr, WritableGpr, GprMem, GprMemImm, |reg| { + reg.get_class() == RegClass::I64 +}); + +// Define a newtype of `Reg` for XMM registers. +newtype_of_reg!(Xmm, WritableXmm, XmmMem, XmmMemImm, |reg| { + reg.get_class() == RegClass::V128 +}); + /// A possible addressing mode (amode) that can be used in instructions. /// These denote a 64-bit value only. #[derive(Clone, Debug)] @@ -27,8 +330,8 @@ pub enum Amode { /// sign-extend-32-to-64(Immediate) + Register1 + (Register2 << Shift) ImmRegRegShift { simm32: u32, - base: Reg, - index: Reg, + base: Gpr, + index: Gpr, shift: u8, /* 0 .. 3 only */ flags: MemFlags, }, @@ -48,7 +351,7 @@ impl Amode { } } - pub(crate) fn imm_reg_reg_shift(simm32: u32, base: Reg, index: Reg, shift: u8) -> Self { + pub(crate) fn imm_reg_reg_shift(simm32: u32, base: Gpr, index: Gpr, shift: u8) -> Self { debug_assert!(base.get_class() == RegClass::I64); debug_assert!(index.get_class() == RegClass::I64); debug_assert!(shift <= 3); @@ -96,8 +399,8 @@ impl Amode { collector.add_use(*base); } Amode::ImmRegRegShift { base, index, .. } => { - collector.add_use(*base); - collector.add_use(*index); + collector.add_use(base.to_reg()); + collector.add_use(index.to_reg()); } Amode::RipRelative { .. } => { // RIP isn't involved in regalloc. @@ -225,7 +528,7 @@ impl PrettyPrint for SyntheticAmode { /// denote an 8, 16, 32 or 64 bit value. For the Immediate form, in the 8- and 16-bit case, only /// the lower 8 or 16 bits of `simm32` is relevant. In the 64-bit case, the value denoted by /// `simm32` is its sign-extension out to 64 bits. -#[derive(Clone)] +#[derive(Clone, Debug)] pub enum RegMemImm { Reg { reg: Reg }, Mem { addr: SyntheticAmode }, diff --git a/cranelift/codegen/src/isa/x64/inst/emit.rs b/cranelift/codegen/src/isa/x64/inst/emit.rs index c4dc3f3d7a5d..bda2ddc0a241 100644 --- a/cranelift/codegen/src/isa/x64/inst/emit.rs +++ b/cranelift/codegen/src/isa/x64/inst/emit.rs @@ -298,14 +298,14 @@ pub(crate) fn emit( Popcnt => (0x0fb8, 2), }; - match src { + match src.clone().into() { RegMem::Reg { reg: src } => emit_std_reg_reg( sink, prefix, opcode, num_opcodes, - dst.to_reg(), - *src, + dst.to_reg().to_reg(), + src, rex_flags, ), RegMem::Mem { addr: src } => { @@ -317,7 +317,7 @@ pub(crate) fn emit( prefix, opcode, num_opcodes, - dst.to_reg(), + dst.to_reg().to_reg(), &amode, rex_flags, ); @@ -327,7 +327,7 @@ pub(crate) fn emit( Inst::Not { size, src, dst } => { debug_assert_eq!(*src, dst.to_reg()); - let rex_flags = RexFlags::from((*size, dst.to_reg())); + let rex_flags = RexFlags::from((*size, dst.to_writable_reg().to_reg())); let (opcode, prefix) = match size { OperandSize::Size8 => (0xF6, LegacyPrefixes::None), OperandSize::Size16 => (0xF7, LegacyPrefixes::_66), @@ -342,7 +342,7 @@ pub(crate) fn emit( Inst::Neg { size, src, dst } => { debug_assert_eq!(*src, dst.to_reg()); - let rex_flags = RexFlags::from((*size, dst.to_reg())); + let rex_flags = RexFlags::from((*size, dst.to_writable_reg().to_reg())); let (opcode, prefix) = match size { OperandSize::Size8 => (0xF6, LegacyPrefixes::None), OperandSize::Size16 => (0xF7, LegacyPrefixes::_66), @@ -728,7 +728,7 @@ pub(crate) fn emit( LegacyPrefixes::None, 0x8D, 1, - dst.to_reg(), + dst.to_reg().to_reg(), &amode, RexFlags::set_w(), ); @@ -884,6 +884,7 @@ pub(crate) fn emit( debug_assert_eq!(*src1, dst.to_reg()); let rex = RexFlags::clear_w(); let prefix = LegacyPrefixes::_66; + let src2 = src2.clone().to_reg_mem_imm(); if let RegMemImm::Imm { simm32 } = src2 { let (opcode_bytes, reg_digit) = match opcode { SseOpcode::Psllw => (0x0F71, 6), @@ -898,7 +899,7 @@ pub(crate) fn emit( }; let dst_enc = reg_enc(dst.to_reg()); emit_std_enc_enc(sink, prefix, opcode_bytes, 2, reg_digit, dst_enc, rex); - let imm = (*simm32) + let imm = (simm32) .try_into() .expect("the immediate must be convertible to a u8"); sink.put1(imm); @@ -917,7 +918,15 @@ pub(crate) fn emit( match src2 { RegMemImm::Reg { reg } => { - emit_std_reg_reg(sink, prefix, opcode_bytes, 2, dst.to_reg(), *reg, rex); + emit_std_reg_reg( + sink, + prefix, + opcode_bytes, + 2, + dst.to_reg().to_reg(), + reg, + rex, + ); } RegMemImm::Mem { addr } => { let addr = &addr.finalize(state, sink); @@ -928,7 +937,7 @@ pub(crate) fn emit( prefix, opcode_bytes, 2, - dst.to_reg(), + dst.to_reg().to_reg(), addr, rex, ); @@ -1335,7 +1344,12 @@ pub(crate) fn emit( // might be negative; use a sign-extension. let inst = Inst::movsx_rm_r( ExtMode::LQ, - RegMem::mem(Amode::imm_reg_reg_shift(0, tmp1.to_reg(), tmp2.to_reg(), 2)), + RegMem::mem(Amode::imm_reg_reg_shift( + 0, + Gpr::new(tmp1.to_reg()).unwrap(), + Gpr::new(tmp2.to_reg()).unwrap(), + 2, + )), *tmp2, ); inst.emit(sink, info, state); @@ -1424,15 +1438,15 @@ pub(crate) fn emit( _ => unimplemented!("Opcode {:?} not implemented", op), }; - match src_e { + match src_e.clone().to_reg_mem() { RegMem::Reg { reg: reg_e } => { emit_std_reg_reg( sink, prefix, opcode, num_opcodes, - reg_g.to_reg(), - *reg_e, + reg_g.to_reg().to_reg(), + reg_e, rex, ); } @@ -1445,7 +1459,7 @@ pub(crate) fn emit( prefix, opcode, num_opcodes, - reg_g.to_reg(), + reg_g.to_reg().to_reg(), addr, rex, ); @@ -1460,7 +1474,7 @@ pub(crate) fn emit( Avx512Opcode::Vpopcntb => (LegacyPrefixes::_66, OpcodeMap::_0F38, false, 0x54), _ => unimplemented!("Opcode {:?} not implemented", op), }; - match src { + match src.clone().to_reg_mem() { RegMem::Reg { reg: src } => EvexInstruction::new() .length(EvexVectorLength::V128) .prefix(prefix) @@ -1587,9 +1601,17 @@ pub(crate) fn emit( _ => unimplemented!("Opcode {:?} not implemented", op), }; - match src_e { + match src_e.clone().to_reg_mem() { RegMem::Reg { reg: reg_e } => { - emit_std_reg_reg(sink, prefix, opcode, length, reg_g.to_reg(), *reg_e, rex); + emit_std_reg_reg( + sink, + prefix, + opcode, + length, + reg_g.to_reg().to_reg(), + reg_e, + rex, + ); } RegMem::Mem { addr } => { let addr = &addr.finalize(state, sink); @@ -1600,7 +1622,7 @@ pub(crate) fn emit( prefix, opcode, length, - reg_g.to_reg(), + reg_g.to_reg().to_reg(), addr, rex, ); @@ -1619,7 +1641,7 @@ pub(crate) fn emit( Avx512Opcode::Vpmullq => (true, 0x40), _ => unimplemented!("Opcode {:?} not implemented", op), }; - match src1 { + match src1.clone().to_reg_mem() { RegMem::Reg { reg: src } => EvexInstruction::new() .length(EvexVectorLength::V128) .prefix(LegacyPrefixes::_66) @@ -1845,9 +1867,9 @@ pub(crate) fn emit( }; let rex = RexFlags::from(*dst_size); let (src, dst) = if dst_first { - (dst.to_reg(), *src) + (dst.to_reg().to_reg(), src.to_reg()) } else { - (*src, dst.to_reg()) + (src.to_reg(), dst.to_reg().to_reg()) }; emit_std_reg_reg(sink, prefix, opcode, 2, src, dst, rex); @@ -1870,7 +1892,15 @@ pub(crate) fn emit( let rex = RexFlags::from(*src_size); match src_e { RegMem::Reg { reg: reg_e } => { - emit_std_reg_reg(sink, prefix, opcode, 2, reg_g.to_reg(), *reg_e, rex); + emit_std_reg_reg( + sink, + prefix, + opcode, + 2, + reg_g.to_reg().to_reg(), + *reg_e, + rex, + ); } RegMem::Mem { addr } => { let addr = &addr.finalize(state, sink); @@ -1881,7 +1911,7 @@ pub(crate) fn emit( prefix, opcode, 2, - reg_g.to_reg(), + reg_g.to_reg().to_reg(), addr, rex, ); @@ -1950,7 +1980,11 @@ pub(crate) fn emit( // If x seen as a signed int64 is not negative, a signed-conversion will do the right // thing. // TODO use tst src, src here. - let inst = Inst::cmp_rmi_r(OperandSize::Size64, RegMemImm::imm(0), src.to_reg()); + let inst = Inst::cmp_rmi_r( + OperandSize::Size64, + RegMemImm::imm(0), + src.to_reg().to_reg(), + ); inst.emit(sink, info, state); one_way_jmp(sink, CC::L, handle_negative); @@ -1961,8 +1995,8 @@ pub(crate) fn emit( sink, info, state, - src.to_reg(), - *dst, + src.to_reg().to_reg(), + dst.to_writable_reg(), *dst_size == OperandSize::Size64, ); @@ -1973,7 +2007,11 @@ pub(crate) fn emit( // Divide x by two to get it in range for the signed conversion, keep the LSB, and // scale it back up on the FP side. - let inst = Inst::gen_move(*tmp_gpr1, src.to_reg(), types::I64); + let inst = Inst::gen_move( + tmp_gpr1.to_writable_reg(), + src.to_reg().to_reg(), + types::I64, + ); inst.emit(sink, info, state); // tmp_gpr1 := src >> 1 @@ -1981,26 +2019,30 @@ pub(crate) fn emit( OperandSize::Size64, ShiftKind::ShiftRightLogical, Some(1), - *tmp_gpr1, + tmp_gpr1.to_writable_reg(), ); inst.emit(sink, info, state); - let inst = Inst::gen_move(*tmp_gpr2, src.to_reg(), types::I64); + let inst = Inst::gen_move( + tmp_gpr2.to_writable_reg(), + src.to_reg().to_reg(), + types::I64, + ); inst.emit(sink, info, state); let inst = Inst::alu_rmi_r( OperandSize::Size64, AluRmiROpcode::And, RegMemImm::imm(1), - *tmp_gpr2, + tmp_gpr2.to_writable_reg(), ); inst.emit(sink, info, state); let inst = Inst::alu_rmi_r( OperandSize::Size64, AluRmiROpcode::Or, - RegMemImm::reg(tmp_gpr1.to_reg()), - *tmp_gpr2, + RegMemImm::reg(tmp_gpr1.to_reg().to_reg()), + tmp_gpr2.to_writable_reg(), ); inst.emit(sink, info, state); @@ -2008,8 +2050,8 @@ pub(crate) fn emit( sink, info, state, - tmp_gpr2.to_reg(), - *dst, + tmp_gpr2.to_reg().to_reg(), + dst.to_writable_reg(), *dst_size == OperandSize::Size64, ); @@ -2018,7 +2060,11 @@ pub(crate) fn emit( } else { SseOpcode::Addss }; - let inst = Inst::xmm_rm_r(add_op, RegMem::reg(dst.to_reg()), *dst); + let inst = Inst::xmm_rm_r( + add_op, + RegMem::reg(dst.to_reg().to_reg()), + dst.to_writable_reg(), + ); inst.emit(sink, info, state); sink.bind_label(done); @@ -2091,18 +2137,18 @@ pub(crate) fn emit( let not_nan = sink.get_label(); // The truncation. - let inst = Inst::xmm_to_gpr(trunc_op, src, *dst, *dst_size); + let inst = Inst::xmm_to_gpr(trunc_op, src.to_reg(), dst.to_writable_reg(), *dst_size); inst.emit(sink, info, state); // Compare against 1, in case of overflow the dst operand was INT_MIN. - let inst = Inst::cmp_rmi_r(*dst_size, RegMemImm::imm(1), dst.to_reg()); + let inst = Inst::cmp_rmi_r(*dst_size, RegMemImm::imm(1), dst.to_reg().to_reg()); inst.emit(sink, info, state); one_way_jmp(sink, CC::NO, done); // no overflow => done // Check for NaN. - let inst = Inst::xmm_cmp_rm_r(cmp_op, RegMem::reg(src), src); + let inst = Inst::xmm_cmp_rm_r(cmp_op, RegMem::reg(src.to_reg()), src.to_reg()); inst.emit(sink, info, state); one_way_jmp(sink, CC::NP, not_nan); // go to not_nan if not a NaN @@ -2112,8 +2158,8 @@ pub(crate) fn emit( let inst = Inst::alu_rmi_r( *dst_size, AluRmiROpcode::Xor, - RegMemImm::reg(dst.to_reg()), - *dst, + RegMemImm::reg(dst.to_reg().to_reg()), + dst.to_writable_reg(), ); inst.emit(sink, info, state); @@ -2125,11 +2171,18 @@ pub(crate) fn emit( // If the input was positive, saturate to INT_MAX. // Zero out tmp_xmm. - let inst = - Inst::xmm_rm_r(SseOpcode::Xorpd, RegMem::reg(tmp_xmm.to_reg()), *tmp_xmm); + let inst = Inst::xmm_rm_r( + SseOpcode::Xorpd, + RegMem::reg(tmp_xmm.to_reg().to_reg()), + tmp_xmm.to_writable_reg(), + ); inst.emit(sink, info, state); - let inst = Inst::xmm_cmp_rm_r(cmp_op, RegMem::reg(src), tmp_xmm.to_reg()); + let inst = Inst::xmm_cmp_rm_r( + cmp_op, + RegMem::reg(src.to_reg()), + tmp_xmm.to_reg().to_reg(), + ); inst.emit(sink, info, state); // Jump if >= to done. @@ -2137,10 +2190,14 @@ pub(crate) fn emit( // Otherwise, put INT_MAX. if *dst_size == OperandSize::Size64 { - let inst = Inst::imm(OperandSize::Size64, 0x7fffffffffffffff, *dst); + let inst = Inst::imm( + OperandSize::Size64, + 0x7fffffffffffffff, + dst.to_writable_reg(), + ); inst.emit(sink, info, state); } else { - let inst = Inst::imm(OperandSize::Size32, 0x7fffffff, *dst); + let inst = Inst::imm(OperandSize::Size32, 0x7fffffff, dst.to_writable_reg()); inst.emit(sink, info, state); } } else { @@ -2162,7 +2219,8 @@ pub(crate) fn emit( match *src_size { OperandSize::Size32 => { let cst = Ieee32::pow2(output_bits - 1).neg().bits(); - let inst = Inst::imm(OperandSize::Size32, cst as u64, *tmp_gpr); + let inst = + Inst::imm(OperandSize::Size32, cst as u64, tmp_gpr.to_writable_reg()); inst.emit(sink, info, state); } OperandSize::Size64 => { @@ -2174,17 +2232,26 @@ pub(crate) fn emit( } else { Ieee64::pow2(output_bits - 1).neg() }; - let inst = Inst::imm(OperandSize::Size64, cst.bits(), *tmp_gpr); + let inst = + Inst::imm(OperandSize::Size64, cst.bits(), tmp_gpr.to_writable_reg()); inst.emit(sink, info, state); } _ => unreachable!(), } - let inst = - Inst::gpr_to_xmm(cast_op, RegMem::reg(tmp_gpr.to_reg()), *src_size, *tmp_xmm); + let inst = Inst::gpr_to_xmm( + cast_op, + RegMem::reg(tmp_gpr.to_reg().to_reg()), + *src_size, + tmp_xmm.to_writable_reg(), + ); inst.emit(sink, info, state); - let inst = Inst::xmm_cmp_rm_r(cmp_op, RegMem::reg(tmp_xmm.to_reg()), src); + let inst = Inst::xmm_cmp_rm_r( + cmp_op, + RegMem::reg(tmp_xmm.to_reg().to_reg()), + src.to_reg(), + ); inst.emit(sink, info, state); // jump over trap if src >= or > threshold @@ -2198,11 +2265,18 @@ pub(crate) fn emit( sink.bind_label(check_positive); // Zero out the tmp_xmm register. - let inst = - Inst::xmm_rm_r(SseOpcode::Xorpd, RegMem::reg(tmp_xmm.to_reg()), *tmp_xmm); + let inst = Inst::xmm_rm_r( + SseOpcode::Xorpd, + RegMem::reg(tmp_xmm.to_reg().to_reg()), + tmp_xmm.to_writable_reg(), + ); inst.emit(sink, info, state); - let inst = Inst::xmm_cmp_rm_r(cmp_op, RegMem::reg(src), tmp_xmm.to_reg()); + let inst = Inst::xmm_cmp_rm_r( + cmp_op, + RegMem::reg(src.to_reg()), + tmp_xmm.to_reg().to_reg(), + ); inst.emit(sink, info, state); one_way_jmp(sink, CC::NB, done); // jump over trap if 0 >= src @@ -2282,14 +2356,22 @@ pub(crate) fn emit( _ => unreachable!(), }; - let inst = Inst::imm(*src_size, cst, *tmp_gpr); + let inst = Inst::imm(*src_size, cst, tmp_gpr.to_writable_reg()); inst.emit(sink, info, state); - let inst = - Inst::gpr_to_xmm(cast_op, RegMem::reg(tmp_gpr.to_reg()), *src_size, *tmp_xmm); + let inst = Inst::gpr_to_xmm( + cast_op, + RegMem::reg(tmp_gpr.to_reg().to_reg()), + *src_size, + tmp_xmm.to_writable_reg(), + ); inst.emit(sink, info, state); - let inst = Inst::xmm_cmp_rm_r(cmp_op, RegMem::reg(tmp_xmm.to_reg()), src.to_reg()); + let inst = Inst::xmm_cmp_rm_r( + cmp_op, + RegMem::reg(tmp_xmm.to_reg().to_reg()), + src.to_reg().to_reg(), + ); inst.emit(sink, info, state); let handle_large = sink.get_label(); @@ -2303,8 +2385,8 @@ pub(crate) fn emit( let inst = Inst::alu_rmi_r( *dst_size, AluRmiROpcode::Xor, - RegMemImm::reg(dst.to_reg()), - *dst, + RegMemImm::reg(dst.to_reg().to_reg()), + dst.to_writable_reg(), ); inst.emit(sink, info, state); @@ -2321,10 +2403,15 @@ pub(crate) fn emit( // Actual truncation for small inputs: if the result is not positive, then we had an // overflow. - let inst = Inst::xmm_to_gpr(trunc_op, src.to_reg(), *dst, *dst_size); + let inst = Inst::xmm_to_gpr( + trunc_op, + src.to_reg().to_reg(), + dst.to_writable_reg(), + *dst_size, + ); inst.emit(sink, info, state); - let inst = Inst::cmp_rmi_r(*dst_size, RegMemImm::imm(0), dst.to_reg()); + let inst = Inst::cmp_rmi_r(*dst_size, RegMemImm::imm(0), dst.to_reg().to_reg()); inst.emit(sink, info, state); one_way_jmp(sink, CC::NL, done); // if dst >= 0, jump to done @@ -2335,8 +2422,8 @@ pub(crate) fn emit( let inst = Inst::alu_rmi_r( *dst_size, AluRmiROpcode::Xor, - RegMemImm::reg(dst.to_reg()), - *dst, + RegMemImm::reg(dst.to_reg().to_reg()), + dst.to_writable_reg(), ); inst.emit(sink, info, state); @@ -2352,13 +2439,22 @@ pub(crate) fn emit( sink.bind_label(handle_large); - let inst = Inst::xmm_rm_r(sub_op, RegMem::reg(tmp_xmm.to_reg()), *src); + let inst = Inst::xmm_rm_r( + sub_op, + RegMem::reg(tmp_xmm.to_reg().to_reg()), + src.to_writable_reg(), + ); inst.emit(sink, info, state); - let inst = Inst::xmm_to_gpr(trunc_op, src.to_reg(), *dst, *dst_size); + let inst = Inst::xmm_to_gpr( + trunc_op, + src.to_reg().to_reg(), + dst.to_writable_reg(), + *dst_size, + ); inst.emit(sink, info, state); - let inst = Inst::cmp_rmi_r(*dst_size, RegMemImm::imm(0), dst.to_reg()); + let inst = Inst::cmp_rmi_r(*dst_size, RegMemImm::imm(0), dst.to_reg().to_reg()); inst.emit(sink, info, state); let next_is_large = sink.get_label(); @@ -2374,7 +2470,7 @@ pub(crate) fn emit( } else { u32::max_value() as u64 }, - *dst, + dst.to_writable_reg(), ); inst.emit(sink, info, state); @@ -2388,14 +2484,14 @@ pub(crate) fn emit( sink.bind_label(next_is_large); if *dst_size == OperandSize::Size64 { - let inst = Inst::imm(OperandSize::Size64, 1 << 63, *tmp_gpr); + let inst = Inst::imm(OperandSize::Size64, 1 << 63, tmp_gpr.to_writable_reg()); inst.emit(sink, info, state); let inst = Inst::alu_rmi_r( OperandSize::Size64, AluRmiROpcode::Add, - RegMemImm::reg(tmp_gpr.to_reg()), - *dst, + RegMemImm::reg(tmp_gpr.to_reg().to_reg()), + dst.to_writable_reg(), ); inst.emit(sink, info, state); } else { @@ -2403,7 +2499,7 @@ pub(crate) fn emit( OperandSize::Size32, AluRmiROpcode::Add, RegMemImm::imm(1 << 31), - *dst, + dst.to_writable_reg(), ); inst.emit(sink, info, state); } diff --git a/cranelift/codegen/src/isa/x64/inst/mod.rs b/cranelift/codegen/src/isa/x64/inst/mod.rs index 6f00370ea09d..bd7c039b9436 100644 --- a/cranelift/codegen/src/isa/x64/inst/mod.rs +++ b/cranelift/codegen/src/isa/x64/inst/mod.rs @@ -2,13 +2,11 @@ use crate::binemit::{Addend, CodeOffset, Reloc, StackMap}; use crate::ir::{types, ExternalName, Opcode, SourceLoc, TrapCode, Type, ValueLabel}; -use crate::isa::unwind::UnwindInst; use crate::isa::x64::abi::X64ABIMachineSpec; use crate::isa::x64::settings as x64_settings; use crate::isa::CallConv; use crate::machinst::*; use crate::{settings, CodegenError, CodegenResult}; -use alloc::boxed::Box; use alloc::vec::Vec; use regalloc::{ PrettyPrint, PrettyPrintSized, RealRegUniverse, Reg, RegClass, RegUsageCollector, SpillSlot, @@ -31,510 +29,8 @@ use regs::show_ireg_sized; //============================================================================= // Instructions (top level): definition -// Don't build these directly. Instead use the Inst:: functions to create them. - -/// Instructions. -#[derive(Clone)] -pub enum Inst { - /// Nops of various sizes, including zero. - Nop { len: u8 }, - - // ===================================== - // Integer instructions. - /// Integer arithmetic/bit-twiddling: (add sub and or xor mul adc? sbb?) (32 64) (reg addr imm) reg - AluRmiR { - size: OperandSize, // 4 or 8 - op: AluRmiROpcode, - src1: Reg, - src2: RegMemImm, - dst: Writable, - }, - - /// Instructions on GPR that only read src and defines dst (dst is not modified): bsr, etc. - UnaryRmR { - size: OperandSize, // 2, 4 or 8 - op: UnaryRmROpcode, - src: RegMem, - dst: Writable, - }, - - /// Bitwise not - Not { - size: OperandSize, // 1, 2, 4 or 8 - src: Reg, - dst: Writable, - }, - - /// Integer negation - Neg { - size: OperandSize, // 1, 2, 4 or 8 - src: Reg, - dst: Writable, - }, - - /// Integer quotient and remainder: (div idiv) $rax $rdx (reg addr) - Div { - size: OperandSize, // 1, 2, 4 or 8 - signed: bool, - divisor: RegMem, - dividend: Reg, - dst_quotient: Writable, - dst_remainder: Writable, - }, - - /// The high bits (RDX) of a (un)signed multiply: RDX:RAX := RAX * rhs. - MulHi { - size: OperandSize, // 2, 4, or 8 - signed: bool, - src1: Reg, - src2: RegMem, - dst_lo: Writable, - dst_hi: Writable, - }, - - /// A synthetic sequence to implement the right inline checks for remainder and division, - /// assuming the dividend is in %rax. - /// - /// Puts the result back into %rax if is_div, %rdx if !is_div, to mimic what the div - /// instruction does. - /// - /// The generated code sequence is described in the emit's function match arm for this - /// instruction. - /// - /// Note: %rdx is marked as modified by this instruction, to avoid an early clobber problem - /// with the temporary and divisor registers. Make sure to zero %rdx right before this - /// instruction, or you might run into regalloc failures where %rdx is live before its first - /// def! - CheckedDivOrRemSeq { - kind: DivOrRemKind, - size: OperandSize, - dividend: Reg, - /// The divisor operand. Note it's marked as modified so that it gets assigned a register - /// different from the temporary. - divisor: Writable, - dst_quotient: Writable, - dst_remainder: Writable, - tmp: Option>, - }, - - /// Do a sign-extend based on the sign of the value in rax into rdx: (cwd cdq cqo) - /// or al into ah: (cbw) - SignExtendData { - size: OperandSize, // 1, 2, 4 or 8 - src: Reg, - dst: Writable, - }, - - /// Constant materialization: (imm32 imm64) reg. - /// - /// Either: movl $imm32, %reg32 or movabsq $imm64, %reg32. - Imm { - dst_size: OperandSize, // 4 or 8 - simm64: u64, - dst: Writable, - }, - - /// GPR to GPR move: mov (64 32) reg reg. - MovRR { - size: OperandSize, // 4 or 8 - src: Reg, - dst: Writable, - }, - - /// Zero-extended loads, except for 64 bits: movz (bl bq wl wq lq) addr reg. - /// Note that the lq variant doesn't really exist since the default zero-extend rule makes it - /// unnecessary. For that case we emit the equivalent "movl AM, reg32". - MovzxRmR { - ext_mode: ExtMode, - src: RegMem, - dst: Writable, - }, - - /// A plain 64-bit integer load, since MovZX_RM_R can't represent that. - Mov64MR { - src: SyntheticAmode, - dst: Writable, - }, - - /// Loads the memory address of addr into dst. - LoadEffectiveAddress { - addr: SyntheticAmode, - dst: Writable, - }, - - /// Sign-extended loads and moves: movs (bl bq wl wq lq) addr reg. - MovsxRmR { - ext_mode: ExtMode, - src: RegMem, - dst: Writable, - }, - - /// Integer stores: mov (b w l q) reg addr. - MovRM { - size: OperandSize, // 1, 2, 4 or 8. - src: Reg, - dst: SyntheticAmode, - }, - - /// Arithmetic shifts: (shl shr sar) (b w l q) imm reg. - ShiftR { - size: OperandSize, // 1, 2, 4 or 8 - kind: ShiftKind, - src: Reg, - /// shift count: Some(0 .. #bits-in-type - 1), or None to mean "%cl". - num_bits: Imm8Reg, - dst: Writable, - }, - - /// Arithmetic SIMD shifts. - XmmRmiReg { - opcode: SseOpcode, - src1: Reg, - src2: RegMemImm, - dst: Writable, - }, - - /// Integer comparisons/tests: cmp or test (b w l q) (reg addr imm) reg. - CmpRmiR { - size: OperandSize, // 1, 2, 4 or 8 - opcode: CmpOpcode, - src: RegMemImm, - dst: Reg, - }, - - /// Materializes the requested condition code in the destination reg. - Setcc { cc: CC, dst: Writable }, - - /// Integer conditional move. - /// Overwrites the destination register. - Cmove { - size: OperandSize, // 2, 4, or 8 - cc: CC, - consequent: RegMem, - alternative: Reg, - dst: Writable, - }, - - // ===================================== - // Stack manipulation. - /// pushq (reg addr imm) - Push64 { src: RegMemImm }, - - /// popq reg - Pop64 { dst: Writable }, - - // ===================================== - // Floating-point operations. - /// XMM (scalar or vector) binary op: (add sub and or xor mul adc? sbb?) (32 64) (reg addr) reg - XmmRmR { - op: SseOpcode, - src1: Reg, - src2: RegMem, - dst: Writable, - }, - - XmmRmREvex { - op: Avx512Opcode, - src1: RegMem, - src2: Reg, - dst: Writable, - }, - - /// XMM (scalar or vector) unary op: mov between XMM registers (32 64) (reg addr) reg, sqrt, - /// etc. - /// - /// This differs from XMM_RM_R in that the dst register of XmmUnaryRmR is not used in the - /// computation of the instruction dst value and so does not have to be a previously valid - /// value. This is characteristic of mov instructions. - XmmUnaryRmR { - op: SseOpcode, - src: RegMem, - dst: Writable, - }, - - XmmUnaryRmREvex { - op: Avx512Opcode, - src: RegMem, - dst: Writable, - }, - - /// XMM (scalar or vector) unary op (from xmm to reg/mem): stores, movd, movq - XmmMovRM { - op: SseOpcode, - src: Reg, - dst: SyntheticAmode, - }, - - /// XMM (vector) unary op (to move a constant value into an xmm register): movups - XmmLoadConst { - src: VCodeConstant, - dst: Writable, - ty: Type, - }, - - /// XMM (scalar) unary op (from xmm to integer reg): movd, movq, cvtts{s,d}2si - XmmToGpr { - op: SseOpcode, - src: Reg, - dst: Writable, - dst_size: OperandSize, - }, - - /// XMM (scalar) unary op (from integer to float reg): movd, movq, cvtsi2s{s,d} - GprToXmm { - op: SseOpcode, - src: RegMem, - dst: Writable, - src_size: OperandSize, - }, - - /// Converts an unsigned int64 to a float32/float64. - CvtUint64ToFloatSeq { - dst_size: OperandSize, // 4 or 8 - /// A copy of the source register, fed by lowering. It is marked as modified during - /// register allocation to make sure that the temporary registers differ from the src - /// register, since both registers are live at the same time in the generated code - /// sequence. - src: Writable, - dst: Writable, - tmp_gpr1: Writable, - tmp_gpr2: Writable, - }, - - /// Converts a scalar xmm to a signed int32/int64. - CvtFloatToSintSeq { - dst_size: OperandSize, - src_size: OperandSize, - is_saturating: bool, - /// A copy of the source register, fed by lowering. It is marked as modified during - /// register allocation to make sure that the temporary xmm register differs from the src - /// register, since both registers are live at the same time in the generated code - /// sequence. - src: Writable, - dst: Writable, - tmp_gpr: Writable, - tmp_xmm: Writable, - }, - - /// Converts a scalar xmm to an unsigned int32/int64. - CvtFloatToUintSeq { - src_size: OperandSize, - dst_size: OperandSize, - is_saturating: bool, - /// A copy of the source register, fed by lowering, reused as a temporary. It is marked as - /// modified during register allocation to make sure that the temporary xmm register - /// differs from the src register, since both registers are live at the same time in the - /// generated code sequence. - src: Writable, - dst: Writable, - tmp_gpr: Writable, - tmp_xmm: Writable, - }, - - /// A sequence to compute min/max with the proper NaN semantics for xmm registers. - XmmMinMaxSeq { - size: OperandSize, - is_min: bool, - lhs: Reg, - rhs_dst: Writable, - }, - - /// XMM (scalar) conditional move. - /// Overwrites the destination register if cc is set. - XmmCmove { - size: OperandSize, // 4 or 8 - cc: CC, - src: RegMem, - dst: Writable, - }, - - /// Float comparisons/tests: cmp (b w l q) (reg addr imm) reg. - XmmCmpRmR { - op: SseOpcode, - src: RegMem, - dst: Reg, - }, - - /// A binary XMM instruction with an 8-bit immediate: e.g. cmp (ps pd) imm (reg addr) reg - XmmRmRImm { - op: SseOpcode, - src1: Reg, - src2: RegMem, - dst: Writable, - imm: u8, - size: OperandSize, // 4 or 8 - }, - - // ===================================== - // Control flow instructions. - /// Direct call: call simm32. - CallKnown { - dest: ExternalName, - uses: Vec, - defs: Vec>, - opcode: Opcode, - }, - - /// Indirect call: callq (reg mem). - CallUnknown { - dest: RegMem, - uses: Vec, - defs: Vec>, - opcode: Opcode, - }, - - /// Return. - Ret, - - /// A placeholder instruction, generating no code, meaning that a function epilogue must be - /// inserted there. - EpiloguePlaceholder, - - /// Jump to a known target: jmp simm32. - JmpKnown { dst: MachLabel }, - - /// One-way conditional branch: jcond cond target. - /// - /// This instruction is useful when we have conditional jumps depending on more than two - /// conditions, see for instance the lowering of Brz/brnz with Fcmp inputs. - /// - /// A note of caution: in contexts where the branch target is another block, this has to be the - /// same successor as the one specified in the terminator branch of the current block. - /// Otherwise, this might confuse register allocation by creating new invisible edges. - JmpIf { cc: CC, taken: MachLabel }, - - /// Two-way conditional branch: jcond cond target target. - /// Emitted as a compound sequence; the MachBuffer will shrink it as appropriate. - JmpCond { - cc: CC, - taken: MachLabel, - not_taken: MachLabel, - }, - - /// Jump-table sequence, as one compound instruction (see note in lower.rs for rationale). - /// The generated code sequence is described in the emit's function match arm for this - /// instruction. - /// See comment in lowering about the temporaries signedness. - JmpTableSeq { - idx: Reg, - tmp1: Writable, - tmp2: Writable, - default_target: MachLabel, - targets: Vec, - targets_for_term: Vec, - }, - - /// Indirect jump: jmpq (reg mem). - JmpUnknown { target: RegMem }, - - /// Traps if the condition code is set. - TrapIf { cc: CC, trap_code: TrapCode }, - - /// A debug trap. - Hlt, - - /// An instruction that will always trigger the illegal instruction exception. - Ud2 { trap_code: TrapCode }, - - /// Loads an external symbol in a register, with a relocation: - /// - /// movq $name@GOTPCREL(%rip), dst if PIC is enabled, or - /// movabsq $name, dst otherwise. - LoadExtName { - dst: Writable, - name: Box, - offset: i64, - }, - - // ===================================== - // Instructions pertaining to atomic memory accesses. - /// A standard (native) `lock cmpxchg src, (amode)`, with register conventions: - /// - /// `mem` (read) address - /// `replacement` (read) replacement value - /// %rax (modified) in: expected value, out: value that was actually at `dst` - /// %rflags is written. Do not assume anything about it after the instruction. - /// - /// The instruction "succeeded" iff the lowest `ty` bits of %rax afterwards are the same as - /// they were before. - LockCmpxchg { - ty: Type, // I8, I16, I32 or I64 - replacement: Reg, - expected: Reg, - mem: SyntheticAmode, - dst_old: Writable, - }, - - /// A synthetic instruction, based on a loop around a native `lock cmpxchg` instruction. - /// This atomically modifies a value in memory and returns the old value. The sequence - /// consists of an initial "normal" load from `dst`, followed by a loop which computes the - /// new value and tries to compare-and-swap ("CAS") it into `dst`, using the native - /// instruction `lock cmpxchg{b,w,l,q}` . The loop iterates until the CAS is successful. - /// If there is no contention, there will be only one pass through the loop body. The - /// sequence does *not* perform any explicit memory fence instructions - /// (mfence/sfence/lfence). - /// - /// Note that the transaction is atomic in the sense that, as observed by some other thread, - /// `dst` either has the initial or final value, but no other. It isn't atomic in the sense - /// of guaranteeing that no other thread writes to `dst` in between the initial load and the - /// CAS -- but that would cause the CAS to fail unless the other thread's last write before - /// the CAS wrote the same value that was already there. In other words, this - /// implementation suffers (unavoidably) from the A-B-A problem. - /// - /// This instruction sequence has fixed register uses as follows: - /// - /// %r9 (read) address - /// %r10 (read) second operand for `op` - /// %r11 (written) scratch reg; value afterwards has no meaning - /// %rax (written) the old value at %r9 - /// %rflags is written. Do not assume anything about it after the instruction. - AtomicRmwSeq { - ty: Type, // I8, I16, I32 or I64 - op: inst_common::AtomicRmwOp, - address: Reg, - operand: Reg, - temp: Writable, - dst_old: Writable, - }, - - /// A memory fence (mfence, lfence or sfence). - Fence { kind: FenceKind }, - - // ===================================== - // Meta-instructions generating no code. - /// Marker, no-op in generated code: SP "virtual offset" is adjusted. This - /// controls how MemArg::NominalSPOffset args are lowered. - VirtualSPOffsetAdj { offset: i64 }, - - /// Provides a way to tell the register allocator that the upcoming sequence of instructions - /// will overwrite `dst` so it should be considered as a `def`; use this with care. - /// - /// This is useful when we have a sequence of instructions whose register usages are nominally - /// `mod`s, but such that the combination of operations creates a result that is independent of - /// the initial register value. It's thus semantically a `def`, not a `mod`, when all the - /// instructions are taken together, so we want to ensure the register is defined (its - /// live-range starts) prior to the sequence to keep analyses happy. - /// - /// One alternative would be a compound instruction that somehow encapsulates the others and - /// reports its own `def`s/`use`s/`mod`s; this adds complexity (the instruction list is no - /// longer flat) and requires knowledge about semantics and initial-value independence anyway. - XmmUninitializedValue { dst: Writable }, - - /// A call to the `ElfTlsGetAddr` libcall. Returns address - /// of TLS symbol in rax. - ElfTlsGetAddr { symbol: ExternalName }, - - /// A Mach-O TLS symbol access. Returns address of the TLS - /// symbol in rax. - MachOTlsGetAddr { symbol: ExternalName }, - - /// A definition of a value label. - ValueLabelMarker { reg: Reg, label: ValueLabel }, - - /// An unwind pseudoinstruction describing the state of the - /// machine at this program point. - Unwind { inst: UnwindInst }, -} +// `Inst` is defined inside ISLE as `MInst`. We publicly re-export it here. +pub use super::lower::isle::generated_code::MInst as Inst; pub(crate) fn low32_will_sign_extend_to_64(x: u64) -> bool { let xs = x as i64; @@ -656,15 +152,20 @@ impl Inst { OperandSize::Size32, OperandSize::Size64 ])); - Self::UnaryRmR { size, op, src, dst } + Self::UnaryRmR { + size, + op, + src: GprMem::new(src).unwrap(), + dst: WritableGpr::from_writable_reg(dst).unwrap(), + } } pub(crate) fn not(size: OperandSize, src: Writable) -> Inst { debug_assert_eq!(src.to_reg().get_class(), RegClass::I64); Inst::Not { size, - src: src.to_reg(), - dst: src, + src: Gpr::new(src.to_reg()).unwrap(), + dst: WritableGpr::from_writable_reg(src).unwrap(), } } @@ -753,7 +254,11 @@ impl Inst { pub(crate) fn xmm_mov(op: SseOpcode, src: RegMem, dst: Writable) -> Inst { src.assert_regclass_is(RegClass::V128); debug_assert!(dst.to_reg().get_class() == RegClass::V128); - Inst::XmmUnaryRmR { op, src, dst } + Inst::XmmUnaryRmR { + op, + src: XmmMem::new(src).unwrap(), + dst: WritableXmm::from_writable_reg(dst).unwrap(), + } } pub(crate) fn xmm_load_const(src: VCodeConstant, dst: Writable, ty: Type) -> Inst { @@ -766,13 +271,21 @@ impl Inst { pub(crate) fn xmm_unary_rm_r(op: SseOpcode, src: RegMem, dst: Writable) -> Inst { src.assert_regclass_is(RegClass::V128); debug_assert!(dst.to_reg().get_class() == RegClass::V128); - Inst::XmmUnaryRmR { op, src, dst } + Inst::XmmUnaryRmR { + op, + src: XmmMem::new(src).unwrap(), + dst: WritableXmm::from_writable_reg(dst).unwrap(), + } } pub(crate) fn xmm_unary_rm_r_evex(op: Avx512Opcode, src: RegMem, dst: Writable) -> Inst { src.assert_regclass_is(RegClass::V128); debug_assert!(dst.to_reg().get_class() == RegClass::V128); - Inst::XmmUnaryRmREvex { op, src, dst } + Inst::XmmUnaryRmREvex { + op, + src: XmmMem::new(src).unwrap(), + dst: WritableXmm::from_writable_reg(dst).unwrap(), + } } pub(crate) fn xmm_rm_r(op: SseOpcode, src: RegMem, dst: Writable) -> Self { @@ -780,9 +293,9 @@ impl Inst { debug_assert!(dst.to_reg().get_class() == RegClass::V128); Inst::XmmRmR { op, - src1: dst.to_reg(), - src2: src, - dst, + src1: Xmm::new(dst.to_reg()).unwrap(), + src2: XmmMem::new(src).unwrap(), + dst: WritableXmm::from_writable_reg(dst).unwrap(), } } @@ -797,15 +310,17 @@ impl Inst { debug_assert!(dst.to_reg().get_class() == RegClass::V128); Inst::XmmRmREvex { op, - src1, - src2, - dst, + src1: XmmMem::new(src1).unwrap(), + src2: Xmm::new(src2).unwrap(), + dst: WritableXmm::from_writable_reg(dst).unwrap(), } } pub(crate) fn xmm_uninit_value(dst: Writable) -> Self { debug_assert!(dst.to_reg().get_class() == RegClass::V128); - Inst::XmmUninitializedValue { dst } + Inst::XmmUninitializedValue { + dst: WritableXmm::from_writable_reg(dst).unwrap(), + } } pub(crate) fn xmm_mov_r_m(op: SseOpcode, src: Reg, dst: impl Into) -> Inst { @@ -828,8 +343,8 @@ impl Inst { debug_assert!(dst_size.is_one_of(&[OperandSize::Size32, OperandSize::Size64])); Inst::XmmToGpr { op, - src, - dst, + src: Xmm::new(src).unwrap(), + dst: WritableGpr::from_writable_reg(dst).unwrap(), dst_size, } } @@ -846,7 +361,7 @@ impl Inst { Inst::GprToXmm { op, src, - dst, + dst: WritableXmm::from_writable_reg(dst).unwrap(), src_size, } } @@ -870,10 +385,10 @@ impl Inst { debug_assert!(tmp_gpr2.to_reg().get_class() == RegClass::I64); debug_assert!(dst.to_reg().get_class() == RegClass::V128); Inst::CvtUint64ToFloatSeq { - src, - dst, - tmp_gpr1, - tmp_gpr2, + src: WritableGpr::from_writable_reg(src).unwrap(), + dst: WritableXmm::from_writable_reg(dst).unwrap(), + tmp_gpr1: WritableGpr::from_writable_reg(tmp_gpr1).unwrap(), + tmp_gpr2: WritableGpr::from_writable_reg(tmp_gpr2).unwrap(), dst_size, } } @@ -897,10 +412,10 @@ impl Inst { src_size, dst_size, is_saturating, - src, - dst, - tmp_gpr, - tmp_xmm, + src: WritableXmm::from_writable_reg(src).unwrap(), + dst: WritableGpr::from_writable_reg(dst).unwrap(), + tmp_gpr: WritableGpr::from_writable_reg(tmp_gpr).unwrap(), + tmp_xmm: WritableXmm::from_writable_reg(tmp_xmm).unwrap(), } } @@ -923,10 +438,10 @@ impl Inst { src_size, dst_size, is_saturating, - src, - dst, - tmp_gpr, - tmp_xmm, + src: WritableXmm::from_writable_reg(src).unwrap(), + dst: WritableGpr::from_writable_reg(dst).unwrap(), + tmp_gpr: WritableGpr::from_writable_reg(tmp_gpr).unwrap(), + tmp_xmm: WritableXmm::from_writable_reg(tmp_xmm).unwrap(), } } @@ -976,9 +491,9 @@ impl Inst { debug_assert!(dst.to_reg().get_class() == RegClass::V128); Inst::XmmRmiReg { opcode, - src1: dst.to_reg(), - src2: src, - dst, + src1: Xmm::new(dst.to_reg()).unwrap(), + src2: XmmMemImm::new(src).unwrap(), + dst: WritableXmm::from_writable_reg(dst).unwrap(), } } @@ -1018,7 +533,7 @@ impl Inst { debug_assert!(dst.to_reg().get_class() == RegClass::I64); Inst::LoadEffectiveAddress { addr: addr.into(), - dst, + dst: WritableGpr::from_writable_reg(dst).unwrap(), } } @@ -1270,7 +785,7 @@ impl Inst { } Self::XmmRmR { op, src2, dst, .. } => { - src2.to_reg() == Some(dst.to_reg()) + src2.clone().to_reg_mem().to_reg() == Some(dst.to_reg().to_reg()) && (*op == SseOpcode::Xorps || *op == SseOpcode::Xorpd || *op == SseOpcode::Pxor @@ -1331,7 +846,11 @@ impl Inst { Inst::XmmRmiReg { src1, dst, .. } => { if *src1 != dst.to_reg() { debug_assert!(src1.is_virtual()); - insts.push(Self::gen_move(*dst, *src1, types::I8X16)); + insts.push(Self::gen_move( + dst.to_writable_reg(), + src1.to_reg(), + types::I8X16, + )); *src1 = dst.to_reg(); } insts.push(self); @@ -1339,7 +858,11 @@ impl Inst { Inst::XmmRmR { src1, dst, .. } => { if *src1 != dst.to_reg() { debug_assert!(src1.is_virtual()); - insts.push(Self::gen_move(*dst, *src1, types::I8X16)); + insts.push(Self::gen_move( + dst.to_writable_reg(), + src1.to_reg(), + types::I8X16, + )); *src1 = dst.to_reg(); } insts.push(self); @@ -1368,7 +891,11 @@ impl Inst { Inst::Not { src, dst, .. } | Inst::Neg { src, dst, .. } => { if *src != dst.to_reg() { debug_assert!(src.is_virtual()); - insts.push(Self::gen_move(*dst, *src, types::I64)); + insts.push(Self::gen_move( + dst.to_writable_reg(), + src.to_reg(), + types::I64, + )); *src = dst.to_reg(); } insts.push(self); @@ -1626,19 +1153,19 @@ impl PrettyPrint for Inst { "{} {}, {}", ljustify2(op.to_string(), suffix_bwlq(*size)), src.show_rru_sized(mb_rru, size.to_bytes()), - show_ireg_sized(dst.to_reg(), mb_rru, size.to_bytes()), + show_ireg_sized(dst.to_reg().to_reg(), mb_rru, size.to_bytes()), ), Inst::Not { size, src: _, dst } => format!( "{} {}", ljustify2("not".to_string(), suffix_bwlq(*size)), - show_ireg_sized(dst.to_reg(), mb_rru, size.to_bytes()) + show_ireg_sized(dst.to_reg().to_reg(), mb_rru, size.to_bytes()) ), Inst::Neg { size, src: _, dst } => format!( "{} {}", ljustify2("neg".to_string(), suffix_bwlq(*size)), - show_ireg_sized(dst.to_reg(), mb_rru, size.to_bytes()) + show_ireg_sized(dst.to_reg().to_reg(), mb_rru, size.to_bytes()) ), Inst::Div { @@ -1696,14 +1223,14 @@ impl PrettyPrint for Inst { "{} {}, {}", ljustify(op.to_string()), src.show_rru_sized(mb_rru, op.src_size()), - show_ireg_sized(dst.to_reg(), mb_rru, 8), + show_ireg_sized(dst.to_reg().to_reg(), mb_rru, 8), ), Inst::XmmUnaryRmREvex { op, src, dst, .. } => format!( "{} {}, {}", ljustify(op.to_string()), src.show_rru_sized(mb_rru, 8), - show_ireg_sized(dst.to_reg(), mb_rru, 8), + show_ireg_sized(dst.to_reg().to_reg(), mb_rru, 8), ), Inst::XmmMovRM { op, src, dst, .. } => format!( @@ -1717,7 +1244,7 @@ impl PrettyPrint for Inst { "{} {}, {}", ljustify(op.to_string()), src2.show_rru_sized(mb_rru, 8), - show_ireg_sized(dst.to_reg(), mb_rru, 8), + show_ireg_sized(dst.to_reg().to_reg(), mb_rru, 8), ), Inst::XmmRmREvex { @@ -1730,8 +1257,8 @@ impl PrettyPrint for Inst { "{} {}, {}, {}", ljustify(op.to_string()), src1.show_rru_sized(mb_rru, 8), - show_ireg_sized(*src2, mb_rru, 8), - show_ireg_sized(dst.to_reg(), mb_rru, 8), + show_ireg_sized(src2.to_reg(), mb_rru, 8), + show_ireg_sized(dst.to_reg().to_reg(), mb_rru, 8), ), Inst::XmmMinMaxSeq { @@ -1795,7 +1322,7 @@ impl PrettyPrint for Inst { "{} {}, {}", ljustify(op.to_string()), src.show_rru(mb_rru), - show_ireg_sized(dst.to_reg(), mb_rru, dst_size), + show_ireg_sized(dst.to_reg().to_reg(), mb_rru, dst_size), ) } @@ -1830,7 +1357,7 @@ impl PrettyPrint for Inst { "f32" } )), - show_ireg_sized(src.to_reg(), mb_rru, 8), + show_ireg_sized(src.to_reg().to_reg(), mb_rru, 8), dst.show_rru(mb_rru), ), @@ -1847,8 +1374,8 @@ impl PrettyPrint for Inst { src_size.to_bits(), dst_size.to_bits() )), - show_ireg_sized(src.to_reg(), mb_rru, 8), - show_ireg_sized(dst.to_reg(), mb_rru, dst_size.to_bytes()), + show_ireg_sized(src.to_reg().to_reg(), mb_rru, 8), + show_ireg_sized(dst.to_reg().to_reg(), mb_rru, dst_size.to_bytes()), ), Inst::CvtFloatToUintSeq { @@ -1864,8 +1391,8 @@ impl PrettyPrint for Inst { src_size.to_bits(), dst_size.to_bits() )), - show_ireg_sized(src.to_reg(), mb_rru, 8), - show_ireg_sized(dst.to_reg(), mb_rru, dst_size.to_bytes()), + show_ireg_sized(src.to_reg().to_reg(), mb_rru, 8), + show_ireg_sized(dst.to_reg().to_reg(), mb_rru, dst_size.to_bytes()), ), Inst::Imm { @@ -2174,11 +1701,11 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) { } Inst::Not { src, dst, .. } => { debug_assert_eq!(*src, dst.to_reg()); - collector.add_mod(*dst); + collector.add_mod(dst.to_writable_reg()); } Inst::Neg { src, dst, .. } => { debug_assert_eq!(*src, dst.to_reg()); - collector.add_mod(*dst); + collector.add_mod(dst.to_writable_reg()); } Inst::Div { size, @@ -2249,11 +1776,13 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) { } } } - Inst::UnaryRmR { src, dst, .. } - | Inst::XmmUnaryRmR { src, dst, .. } - | Inst::XmmUnaryRmREvex { src, dst, .. } => { - src.get_regs_as_uses(collector); - collector.add_def(*dst); + Inst::UnaryRmR { src, dst, .. } => { + src.clone().to_reg_mem().get_regs_as_uses(collector); + collector.add_def(dst.to_writable_reg()); + } + Inst::XmmUnaryRmR { src, dst, .. } | Inst::XmmUnaryRmREvex { src, dst, .. } => { + src.clone().to_reg_mem().get_regs_as_uses(collector); + collector.add_def(dst.to_writable_reg()); } Inst::XmmRmR { src1, @@ -2265,10 +1794,10 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) { debug_assert_eq!(*src1, dst.to_reg()); if inst.produces_const() { // No need to account for src, since src == dst. - collector.add_def(*dst); + collector.add_def(dst.to_writable_reg()); } else { - src2.get_regs_as_uses(collector); - collector.add_mod(*dst); + src2.clone().to_reg_mem().get_regs_as_uses(collector); + collector.add_mod(dst.to_writable_reg()); // Some instructions have an implicit use of XMM0. if *op == SseOpcode::Blendvpd || *op == SseOpcode::Blendvps @@ -2285,11 +1814,11 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) { dst, .. } => { - src1.get_regs_as_uses(collector); - collector.add_use(*src2); + src1.clone().to_reg_mem().get_regs_as_uses(collector); + collector.add_use(src2.to_reg()); match *op { - Avx512Opcode::Vpermi2b => collector.add_mod(*dst), - _ => collector.add_def(*dst), + Avx512Opcode::Vpermi2b => collector.add_mod(dst.to_writable_reg()), + _ => collector.add_def(dst.to_writable_reg()), } } Inst::XmmRmRImm { @@ -2320,7 +1849,7 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) { collector.add_mod(*dst); } } - Inst::XmmUninitializedValue { dst } => collector.add_def(*dst), + Inst::XmmUninitializedValue { dst } => collector.add_def(dst.to_writable_reg()), Inst::XmmLoadConst { dst, .. } => collector.add_def(*dst), Inst::XmmMinMaxSeq { lhs, rhs_dst, .. } => { collector.add_use(*lhs); @@ -2330,8 +1859,8 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) { src1, src2, dst, .. } => { debug_assert_eq!(*src1, dst.to_reg()); - src2.get_regs_as_uses(collector); - collector.add_mod(*dst); + src2.clone().to_reg_mem_imm().get_regs_as_uses(collector); + collector.add_mod(dst.to_writable_reg()); } Inst::XmmMovRM { src, dst, .. } => { collector.add_use(*src); @@ -2344,13 +1873,17 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) { Inst::Imm { dst, .. } => { collector.add_def(*dst); } - Inst::MovRR { src, dst, .. } | Inst::XmmToGpr { src, dst, .. } => { + Inst::MovRR { src, dst, .. } => { collector.add_use(*src); collector.add_def(*dst); } + Inst::XmmToGpr { src, dst, .. } => { + collector.add_use(src.to_reg()); + collector.add_def(dst.to_writable_reg()); + } Inst::GprToXmm { src, dst, .. } => { src.get_regs_as_uses(collector); - collector.add_def(*dst); + collector.add_def(dst.to_writable_reg()); } Inst::CvtUint64ToFloatSeq { src, @@ -2359,10 +1892,10 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) { tmp_gpr2, .. } => { - collector.add_mod(*src); - collector.add_def(*dst); - collector.add_def(*tmp_gpr1); - collector.add_def(*tmp_gpr2); + collector.add_mod(src.to_writable_reg()); + collector.add_def(dst.to_writable_reg()); + collector.add_def(tmp_gpr1.to_writable_reg()); + collector.add_def(tmp_gpr2.to_writable_reg()); } Inst::CvtFloatToSintSeq { src, @@ -2378,19 +1911,23 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) { tmp_xmm, .. } => { - collector.add_mod(*src); - collector.add_def(*dst); - collector.add_def(*tmp_gpr); - collector.add_def(*tmp_xmm); + collector.add_mod(src.to_writable_reg()); + collector.add_def(dst.to_writable_reg()); + collector.add_def(tmp_gpr.to_writable_reg()); + collector.add_def(tmp_xmm.to_writable_reg()); } Inst::MovzxRmR { src, dst, .. } => { src.get_regs_as_uses(collector); collector.add_def(*dst); } - Inst::Mov64MR { src, dst, .. } | Inst::LoadEffectiveAddress { addr: src, dst } => { + Inst::Mov64MR { src, dst, .. } => { src.get_regs_as_uses(collector); collector.add_def(*dst) } + Inst::LoadEffectiveAddress { addr: src, dst } => { + src.get_regs_as_uses(collector); + collector.add_def(dst.to_writable_reg()) + } Inst::MovsxRmR { src, dst, .. } => { src.get_regs_as_uses(collector); collector.add_def(*dst); @@ -2535,8 +2072,8 @@ impl Amode { ref mut index, .. } => { - map.map_use(base); - map.map_use(index); + base.map_use(map); + index.map_use(map); } Amode::RipRelative { .. } => { // RIP isn't involved in regalloc. @@ -2622,7 +2159,7 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { } Inst::Not { src, dst, .. } | Inst::Neg { src, dst, .. } => { debug_assert_eq!(*src, dst.to_reg()); - mapper.map_mod(dst); + dst.map_mod(mapper); *src = dst.to_reg(); } Inst::Div { divisor, .. } => divisor.map_uses(mapper), @@ -2643,14 +2180,17 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { ref mut src, ref mut dst, .. + } => { + src.map_uses(mapper); + dst.map_def(mapper); } - | Inst::UnaryRmR { + Inst::UnaryRmR { ref mut src, ref mut dst, .. } => { src.map_uses(mapper); - mapper.map_def(dst); + dst.map_def(mapper); } Inst::XmmRmRImm { ref op, @@ -2691,11 +2231,11 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { debug_assert_eq!(*src1, dst.to_reg()); if produces_const { src2.map_as_def(mapper); - mapper.map_def(dst); + dst.map_def(mapper); *src1 = dst.to_reg(); } else { src2.map_uses(mapper); - mapper.map_mod(dst); + dst.map_mod(mapper); *src1 = dst.to_reg(); } } @@ -2707,10 +2247,10 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { .. } => { src1.map_uses(mapper); - mapper.map_use(src2); + src2.map_use(mapper); match *op { - Avx512Opcode::Vpermi2b => mapper.map_mod(dst), - _ => mapper.map_def(dst), + Avx512Opcode::Vpermi2b => dst.map_mod(mapper), + _ => dst.map_def(mapper), } } Inst::XmmRmiReg { @@ -2721,11 +2261,11 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { } => { debug_assert_eq!(*src1, dst.to_reg()); src2.map_uses(mapper); - mapper.map_mod(dst); + dst.map_mod(mapper); *src1 = dst.to_reg(); } Inst::XmmUninitializedValue { ref mut dst, .. } => { - mapper.map_def(dst); + dst.map_def(mapper); } Inst::XmmLoadConst { ref mut dst, .. } => { mapper.map_def(dst); @@ -2759,14 +2299,17 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { ref mut src, ref mut dst, .. + } => { + mapper.map_use(src); + mapper.map_def(dst); } - | Inst::XmmToGpr { + Inst::XmmToGpr { ref mut src, ref mut dst, .. } => { - mapper.map_use(src); - mapper.map_def(dst); + src.map_use(mapper); + dst.map_def(mapper); } Inst::GprToXmm { ref mut src, @@ -2774,7 +2317,7 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { .. } => { src.map_uses(mapper); - mapper.map_def(dst); + dst.map_def(mapper); } Inst::CvtUint64ToFloatSeq { ref mut src, @@ -2783,10 +2326,10 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { ref mut tmp_gpr2, .. } => { - mapper.map_mod(src); - mapper.map_def(dst); - mapper.map_def(tmp_gpr1); - mapper.map_def(tmp_gpr2); + src.map_mod(mapper); + dst.map_def(mapper); + tmp_gpr1.map_def(mapper); + tmp_gpr2.map_def(mapper); } Inst::CvtFloatToSintSeq { ref mut src, @@ -2802,10 +2345,10 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { ref mut tmp_xmm, .. } => { - mapper.map_mod(src); - mapper.map_def(dst); - mapper.map_def(tmp_gpr); - mapper.map_def(tmp_xmm); + src.map_mod(mapper); + dst.map_def(mapper); + tmp_gpr.map_def(mapper); + tmp_xmm.map_def(mapper); } Inst::MovzxRmR { ref mut src, @@ -2815,10 +2358,14 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { src.map_uses(mapper); mapper.map_def(dst); } - Inst::Mov64MR { src, dst, .. } | Inst::LoadEffectiveAddress { addr: src, dst } => { + Inst::Mov64MR { src, dst, .. } => { src.map_uses(mapper); mapper.map_def(dst); } + Inst::LoadEffectiveAddress { addr: src, dst } => { + src.map_uses(mapper); + dst.map_def(mapper); + } Inst::MovsxRmR { ref mut src, ref mut dst, @@ -2989,8 +2536,8 @@ impl MachInst for Inst { || *op == SseOpcode::Movdqa || *op == SseOpcode::Movdqu => { - if let RegMem::Reg { reg } = src { - Some((*dst, *reg)) + if let RegMem::Reg { reg } = src.clone().to_reg_mem() { + Some((dst.to_writable_reg(), reg)) } else { None } diff --git a/cranelift/codegen/src/isa/x64/lower.isle b/cranelift/codegen/src/isa/x64/lower.isle index 9b881b5f691c..8de9110bbdfe 100644 --- a/cranelift/codegen/src/isa/x64/lower.isle +++ b/cranelift/codegen/src/isa/x64/lower.isle @@ -95,23 +95,23 @@ (rule (lower (has_type (multi_lane 8 16) (iadd x y))) - (value_reg (paddb (put_in_reg x) - (put_in_reg_mem y)))) + (value_xmm (paddb (put_in_xmm x) + (put_in_xmm_mem y)))) (rule (lower (has_type (multi_lane 16 8) (iadd x y))) - (value_reg (paddw (put_in_reg x) - (put_in_reg_mem y)))) + (value_xmm (paddw (put_in_xmm x) + (put_in_xmm_mem y)))) (rule (lower (has_type (multi_lane 32 4) (iadd x y))) - (value_reg (paddd (put_in_reg x) - (put_in_reg_mem y)))) + (value_xmm (paddd (put_in_xmm x) + (put_in_xmm_mem y)))) (rule (lower (has_type (multi_lane 64 2) (iadd x y))) - (value_reg (paddq (put_in_reg x) - (put_in_reg_mem y)))) + (value_xmm (paddq (put_in_xmm x) + (put_in_xmm_mem y)))) ;; `i128` (rule (lower (has_type $I128 (iadd x y))) @@ -131,25 +131,25 @@ (rule (lower (has_type (multi_lane 8 16) (sadd_sat x y))) - (value_reg (paddsb (put_in_reg x) - (put_in_reg_mem y)))) + (value_xmm (paddsb (put_in_xmm x) + (put_in_xmm_mem y)))) (rule (lower (has_type (multi_lane 16 8) (sadd_sat x y))) - (value_reg (paddsw (put_in_reg x) - (put_in_reg_mem y)))) + (value_xmm (paddsw (put_in_xmm x) + (put_in_xmm_mem y)))) ;;;; Rules for `uadd_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (rule (lower (has_type (multi_lane 8 16) (uadd_sat x y))) - (value_reg (paddusb (put_in_reg x) - (put_in_reg_mem y)))) + (value_xmm (paddusb (put_in_xmm x) + (put_in_xmm_mem y)))) (rule (lower (has_type (multi_lane 16 8) (uadd_sat x y))) - (value_reg (paddusw (put_in_reg x) - (put_in_reg_mem y)))) + (value_xmm (paddusw (put_in_xmm x) + (put_in_xmm_mem y)))) ;;;; Rules for `iadd_ifcout` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -264,23 +264,23 @@ (rule (lower (has_type (multi_lane 8 16) (isub x y))) - (value_reg (psubb (put_in_reg x) - (put_in_reg_mem y)))) + (value_xmm (psubb (put_in_xmm x) + (put_in_xmm_mem y)))) (rule (lower (has_type (multi_lane 16 8) (isub x y))) - (value_reg (psubw (put_in_reg x) - (put_in_reg_mem y)))) + (value_xmm (psubw (put_in_xmm x) + (put_in_xmm_mem y)))) (rule (lower (has_type (multi_lane 32 4) (isub x y))) - (value_reg (psubd (put_in_reg x) - (put_in_reg_mem y)))) + (value_xmm (psubd (put_in_xmm x) + (put_in_xmm_mem y)))) (rule (lower (has_type (multi_lane 64 2) (isub x y))) - (value_reg (psubq (put_in_reg x) - (put_in_reg_mem y)))) + (value_xmm (psubq (put_in_xmm x) + (put_in_xmm_mem y)))) ;; `i128` (rule (lower (has_type $I128 (isub x y))) @@ -300,25 +300,25 @@ (rule (lower (has_type (multi_lane 8 16) (ssub_sat x y))) - (value_reg (psubsb (put_in_reg x) - (put_in_reg_mem y)))) + (value_xmm (psubsb (put_in_xmm x) + (put_in_xmm_mem y)))) (rule (lower (has_type (multi_lane 16 8) (ssub_sat x y))) - (value_reg (psubsw (put_in_reg x) - (put_in_reg_mem y)))) + (value_xmm (psubsw (put_in_xmm x) + (put_in_xmm_mem y)))) ;;;; Rules for `usub_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (rule (lower (has_type (multi_lane 8 16) (usub_sat x y))) - (value_reg (psubusb (put_in_reg x) - (put_in_reg_mem y)))) + (value_xmm (psubusb (put_in_xmm x) + (put_in_xmm_mem y)))) (rule (lower (has_type (multi_lane 16 8) (usub_sat x y))) - (value_reg (psubusw (put_in_reg x) - (put_in_reg_mem y)))) + (value_xmm (psubusw (put_in_xmm x) + (put_in_xmm_mem y)))) ;;;; Rules for `band` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -360,14 +360,16 @@ ;; SSE. -(decl sse_and (Type Reg RegMem) Reg) +(decl sse_and (Type Xmm XmmMem) Xmm) (rule (sse_and $F32X4 x y) (andps x y)) (rule (sse_and $F64X2 x y) (andpd x y)) (rule (sse_and (multi_lane _bits _lanes) x y) (pand x y)) (rule (lower (has_type ty @ (multi_lane _bits _lanes) (band x y))) - (value_reg (sse_and ty (put_in_reg x) (put_in_reg_mem y)))) + (value_xmm (sse_and ty + (put_in_xmm x) + (put_in_xmm_mem y)))) ;; `{i,b}128`. @@ -432,14 +434,16 @@ ;; SSE. -(decl sse_or (Type Reg RegMem) Reg) +(decl sse_or (Type Xmm XmmMem) Xmm) (rule (sse_or $F32X4 x y) (orps x y)) (rule (sse_or $F64X2 x y) (orpd x y)) (rule (sse_or (multi_lane _bits _lanes) x y) (por x y)) (rule (lower (has_type ty @ (multi_lane _bits _lanes) (bor x y))) - (value_reg (sse_or ty (put_in_reg x) (put_in_reg_mem y)))) + (value_xmm (sse_or ty + (put_in_xmm x) + (put_in_xmm_mem y)))) ;; `{i,b}128`. @@ -507,7 +511,7 @@ ;; SSE. (rule (lower (has_type ty @ (multi_lane _bits _lanes) (bxor x y))) - (value_reg (sse_xor ty (put_in_reg x) (put_in_reg_mem y)))) + (value_xmm (sse_xor ty (put_in_xmm x) (put_in_xmm_mem y)))) ;; `{i,b}128`. @@ -578,16 +582,16 @@ ;; instructions. The basic idea, whether the amount to shift by is an immediate ;; or not, is to use a 16x8 shift and then mask off the incorrect bits to 0s. (rule (lower (has_type $I8X16 (ishl src amt))) - (let ((src_ Reg (put_in_reg src)) + (let ((src_ Xmm (put_in_xmm src)) (amt_gpr RegMemImm (put_in_reg_mem_imm amt)) - (amt_xmm RegMemImm (reg_mem_imm_to_xmm amt_gpr)) + (amt_xmm XmmMemImm (mov_rmi_to_xmm amt_gpr)) ;; Shift `src` using 16x8. Unfortunately, a 16x8 shift will only be ;; correct for half of the lanes; the others must be fixed up with ;; the mask below. - (unmasked Reg (psllw src_ amt_xmm)) + (unmasked Xmm (psllw src_ amt_xmm)) (mask_addr SyntheticAmode (ishl_i8x16_mask amt_gpr)) (mask Reg (x64_load $I8X16 mask_addr (ExtKind.None)))) - (value_reg (sse_and $I8X16 unmasked (RegMem.Reg mask))))) + (value_xmm (sse_and $I8X16 unmasked (xmm_mem_new (RegMem.Reg mask)))))) ;; Get the address of the mask to use when fixing up the lanes that weren't ;; correctly generated by the 16x8 shift. @@ -608,25 +612,28 @@ (extern constructor ishl_i8x16_mask_table ishl_i8x16_mask_table) (rule (ishl_i8x16_mask (RegMemImm.Reg amt)) (let ((mask_table SyntheticAmode (ishl_i8x16_mask_table)) - (base_mask_addr Reg (lea mask_table)) + (base_mask_addr Gpr (lea mask_table)) (mask_offset Reg (shl $I64 amt (Imm8Reg.Imm8 4)))) (amode_to_synthetic_amode (amode_imm_reg_reg_shift 0 base_mask_addr - mask_offset + (gpr_new mask_offset) 0)))) (rule (ishl_i8x16_mask (RegMemImm.Mem amt)) (ishl_i8x16_mask (RegMemImm.Reg (x64_load $I64 amt (ExtKind.None))))) ;; 16x8, 32x4, and 64x2 shifts can each use a single instruction. + (rule (lower (has_type $I16X8 (ishl src amt))) - (value_reg (psllw (put_in_reg src) - (reg_mem_imm_to_xmm (put_in_reg_mem_imm amt))))) + (value_xmm (psllw (put_in_xmm src) + (mov_rmi_to_xmm (put_in_reg_mem_imm amt))))) + (rule (lower (has_type $I32X4 (ishl src amt))) - (value_reg (pslld (put_in_reg src) - (reg_mem_imm_to_xmm (put_in_reg_mem_imm amt))))) + (value_xmm (pslld (put_in_xmm src) + (mov_rmi_to_xmm (put_in_reg_mem_imm amt))))) + (rule (lower (has_type $I64X2 (ishl src amt))) - (value_reg (psllq (put_in_reg src) - (reg_mem_imm_to_xmm (put_in_reg_mem_imm amt))))) + (value_xmm (psllq (put_in_xmm src) + (mov_rmi_to_xmm (put_in_reg_mem_imm amt))))) ;;;; Rules for `ushr` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -672,16 +679,18 @@ ;; There are no 8x16 shifts in x64. Do the same 16x8-shift-and-mask thing we do ;; with 8x16 `ishl`. (rule (lower (has_type $I8X16 (ushr src amt))) - (let ((src_ Reg (put_in_reg src)) + (let ((src_ Xmm (put_in_xmm src)) (amt_gpr RegMemImm (put_in_reg_mem_imm amt)) - (amt_xmm RegMemImm (reg_mem_imm_to_xmm amt_gpr)) + (amt_xmm XmmMemImm (mov_rmi_to_xmm amt_gpr)) ;; Shift `src` using 16x8. Unfortunately, a 16x8 shift will only be ;; correct for half of the lanes; the others must be fixed up with ;; the mask below. - (unmasked Reg (psrlw src_ amt_xmm)) + (unmasked Xmm (psrlw src_ amt_xmm)) (mask_addr SyntheticAmode (ushr_i8x16_mask amt_gpr)) (mask Reg (x64_load $I8X16 mask_addr (ExtKind.None)))) - (value_reg (sse_and $I8X16 unmasked (RegMem.Reg mask))))) + (value_xmm (sse_and $I8X16 + unmasked + (xmm_mem_new (RegMem.Reg mask)))))) ;; Get the address of the mask to use when fixing up the lanes that weren't ;; correctly generated by the 16x8 shift. @@ -702,25 +711,28 @@ (extern constructor ushr_i8x16_mask_table ushr_i8x16_mask_table) (rule (ushr_i8x16_mask (RegMemImm.Reg amt)) (let ((mask_table SyntheticAmode (ushr_i8x16_mask_table)) - (base_mask_addr Reg (lea mask_table)) + (base_mask_addr Gpr (lea mask_table)) (mask_offset Reg (shl $I64 amt (Imm8Reg.Imm8 4)))) (amode_to_synthetic_amode (amode_imm_reg_reg_shift 0 base_mask_addr - mask_offset + (gpr_new mask_offset) 0)))) (rule (ushr_i8x16_mask (RegMemImm.Mem amt)) (ushr_i8x16_mask (RegMemImm.Reg (x64_load $I64 amt (ExtKind.None))))) ;; 16x8, 32x4, and 64x2 shifts can each use a single instruction. + (rule (lower (has_type $I16X8 (ushr src amt))) - (value_reg (psrlw (put_in_reg src) - (reg_mem_imm_to_xmm (put_in_reg_mem_imm amt))))) + (value_xmm (psrlw (put_in_xmm src) + (mov_rmi_to_xmm (put_in_reg_mem_imm amt))))) + (rule (lower (has_type $I32X4 (ushr src amt))) - (value_reg (psrld (put_in_reg src) - (reg_mem_imm_to_xmm (put_in_reg_mem_imm amt))))) + (value_xmm (psrld (put_in_xmm src) + (mov_rmi_to_xmm (put_in_reg_mem_imm amt))))) + (rule (lower (has_type $I64X2 (ushr src amt))) - (value_reg (psrlq (put_in_reg src) - (reg_mem_imm_to_xmm (put_in_reg_mem_imm amt))))) + (value_xmm (psrlq (put_in_xmm src) + (mov_rmi_to_xmm (put_in_reg_mem_imm amt))))) ;;;; Rules for `sshr` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -780,33 +792,35 @@ ;; shifted_hi.i16x8 = shift each lane of `high` ;; result = [s0'', s1'', ..., s15''] (rule (lower (has_type $I8X16 (sshr src amt @ (value_type amt_ty)))) - (let ((src_ Reg (put_in_reg src)) + (let ((src_ Xmm (put_in_xmm src)) ;; In order for `packsswb` later to only use the high byte of each ;; 16x8 lane, we shift right an extra 8 bits, relying on `psraw` to ;; fill in the upper bits appropriately. - (lo Reg (punpcklbw src_ (RegMem.Reg src_))) - (hi Reg (punpckhbw src_ (RegMem.Reg src_))) - (amt_ RegMemImm (sshr_i8x16_bigger_shift amt_ty (put_in_reg_mem_imm amt))) - (shifted_lo Reg (psraw lo amt_)) - (shifted_hi Reg (psraw hi amt_))) - (value_reg (packsswb shifted_lo (RegMem.Reg shifted_hi))))) - -(decl sshr_i8x16_bigger_shift (Type RegMemImm) RegMemImm) + (lo Xmm (punpcklbw src_ (xmm_to_xmm_mem src_))) + (hi Xmm (punpckhbw src_ (xmm_to_xmm_mem src_))) + (amt_ XmmMemImm (sshr_i8x16_bigger_shift amt_ty (put_in_reg_mem_imm amt))) + (shifted_lo Xmm (psraw lo amt_)) + (shifted_hi Xmm (psraw hi amt_))) + (value_xmm (packsswb shifted_lo (xmm_to_xmm_mem shifted_hi))))) + +(decl sshr_i8x16_bigger_shift (Type RegMemImm) XmmMemImm) (rule (sshr_i8x16_bigger_shift _ty (RegMemImm.Imm i)) - (RegMemImm.Imm (u32_add i 8))) + (xmm_mem_imm_new (RegMemImm.Imm (u32_add i 8)))) (rule (sshr_i8x16_bigger_shift ty (RegMemImm.Reg r)) - (reg_mem_imm_to_xmm (RegMemImm.Reg (add ty r (RegMemImm.Imm 8))))) + (mov_rmi_to_xmm (RegMemImm.Reg (add ty r (RegMemImm.Imm 8))))) (rule (sshr_i8x16_bigger_shift ty rmi @ (RegMemImm.Mem _m)) - (reg_mem_imm_to_xmm (RegMemImm.Reg (add ty (imm ty 8) rmi)))) + (mov_rmi_to_xmm (RegMemImm.Reg (add ty (imm ty 8) rmi)))) ;; `sshr.{i16x8,i32x4}` can be a simple `psra{w,d}`, we just have to make sure ;; that if the shift amount is in a register, it is in an XMM register. + (rule (lower (has_type $I16X8 (sshr src amt))) - (value_reg (psraw (put_in_reg src) - (reg_mem_imm_to_xmm (put_in_reg_mem_imm amt))))) + (value_xmm (psraw (put_in_xmm src) + (mov_rmi_to_xmm (put_in_reg_mem_imm amt))))) + (rule (lower (has_type $I32X4 (sshr src amt))) - (value_reg (psrad (put_in_reg src) - (reg_mem_imm_to_xmm (put_in_reg_mem_imm amt))))) + (value_xmm (psrad (put_in_xmm src) + (mov_rmi_to_xmm (put_in_reg_mem_imm amt))))) ;; The `sshr.i64x2` CLIF instruction has no single x86 instruction in the older ;; feature sets. Newer ones like AVX512VL + AVX512F include `vpsraq`, a 128-bit @@ -817,14 +831,15 @@ ;; ;; (TODO: when EVEX support is available, add an alternate lowering here). (rule (lower (has_type $I64X2 (sshr src amt))) - (let ((src_ Reg (put_in_reg src)) - (lo Reg (pextrd $I64 src_ 0)) - (hi Reg (pextrd $I64 src_ 1)) + (let ((src_ Xmm (put_in_xmm src)) + (lo Gpr (pextrd $I64 src_ 0)) + (hi Gpr (pextrd $I64 src_ 1)) (amt_ Imm8Reg (put_masked_in_imm8_reg amt $I64)) - (shifted_lo Reg (sar $I64 lo amt_)) - (shifted_hi Reg (sar $I64 hi amt_))) - (value_reg (make_i64x2_from_lanes (RegMem.Reg shifted_lo) - (RegMem.Reg shifted_hi))))) + (shifted_lo Reg (sar $I64 (gpr_to_reg lo) amt_)) + (shifted_hi Reg (sar $I64 (gpr_to_reg hi) amt_))) + (value_xmm (make_i64x2_from_lanes (gpr_mem_new (RegMem.Reg shifted_lo)) + (gpr_mem_new (RegMem.Reg shifted_hi)))))) + ;;;; Rules for `rotl` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; `i16` and `i8`: we need to extend the shift amount, or mask the @@ -910,35 +925,35 @@ ;; `i64` and smaller. (rule (lower (has_type (fits_in_64 ty) (ineg x))) - (value_reg (neg ty (put_in_reg x)))) + (value_gpr (neg ty (put_in_gpr x)))) ;; SSE. (rule (lower (has_type $I8X16 (ineg x))) - (value_reg (psubb (imm $I8X16 0) - (put_in_reg_mem x)))) + (value_xmm (psubb (xmm_new (imm $I8X16 0)) + (put_in_xmm_mem x)))) (rule (lower (has_type $I16X8 (ineg x))) - (value_reg (psubw (imm $I16X8 0) - (put_in_reg_mem x)))) + (value_xmm (psubw (xmm_new (imm $I16X8 0)) + (put_in_xmm_mem x)))) (rule (lower (has_type $I32X4 (ineg x))) - (value_reg (psubd (imm $I32X4 0) - (put_in_reg_mem x)))) + (value_xmm (psubd (xmm_new (imm $I32X4 0)) + (put_in_xmm_mem x)))) (rule (lower (has_type $I64X2 (ineg x))) - (value_reg (psubq (imm $I64X2 0) - (put_in_reg_mem x)))) + (value_xmm (psubq (xmm_new (imm $I64X2 0)) + (put_in_xmm_mem x)))) ;;;; Rules for `avg_round` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (rule (lower (has_type (multi_lane 8 16) (avg_round x y))) - (value_reg (pavgb (put_in_reg x) (put_in_reg_mem y)))) + (value_xmm (pavgb (put_in_xmm x) (put_in_xmm_mem y)))) (rule (lower (has_type (multi_lane 16 8) (avg_round x y))) - (value_reg (pavgw (put_in_reg x) (put_in_reg_mem y)))) + (value_xmm (pavgw (put_in_xmm x) (put_in_xmm_mem y)))) ;;;; Rules for `imul` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -1017,10 +1032,10 @@ ;; (No i8x16 multiply.) (rule (lower (has_type (multi_lane 16 8) (imul x y))) - (value_reg (pmullw (put_in_reg x) (put_in_reg_mem y)))) + (value_xmm (pmullw (put_in_xmm x) (put_in_xmm_mem y)))) (rule (lower (has_type (multi_lane 32 4) (imul x y))) - (value_reg (pmulld (put_in_reg x) (put_in_reg_mem y)))) + (value_xmm (pmulld (put_in_xmm x) (put_in_xmm_mem y)))) ;; With AVX-512 we can implement `i64x2` multiplication with a single ;; instruction. @@ -1028,7 +1043,7 @@ (avx512dq_enabled) (multi_lane 64 2)) (imul x y))) - (value_reg (vpmullq (put_in_reg_mem x) (put_in_reg y)))) + (value_xmm (vpmullq (put_in_xmm_mem x) (put_in_xmm y)))) ;; Otherwise, for i64x2 multiplication we describe a lane A as being composed of ;; a 32-bit upper half "Ah" and a 32-bit lower half "Al". The 32-bit long hand @@ -1052,24 +1067,24 @@ ;; 32-bits when doing calculations, i.e., `Ah == A >> 32`. (rule (lower (has_type (multi_lane 64 2) (imul a b))) - (let ((a0 Reg (put_in_reg a)) - (b0 Reg (put_in_reg b)) + (let ((a0 Xmm (put_in_xmm a)) + (b0 Xmm (put_in_xmm b)) ;; a_hi = A >> 32 - (a_hi Reg (psrlq a0 (RegMemImm.Imm 32))) + (a_hi Xmm (psrlq a0 (xmm_mem_imm_new (RegMemImm.Imm 32)))) ;; ah_bl = Ah * Bl - (ah_bl Reg (pmuludq a_hi (RegMem.Reg b0))) + (ah_bl Xmm (pmuludq a_hi (xmm_to_xmm_mem b0))) ;; b_hi = B >> 32 - (b_hi Reg (psrlq b0 (RegMemImm.Imm 32))) + (b_hi Xmm (psrlq b0 (xmm_mem_imm_new (RegMemImm.Imm 32)))) ;; al_bh = Al * Bh - (al_bh Reg (pmuludq a0 (RegMem.Reg b_hi))) + (al_bh Xmm (pmuludq a0 (xmm_to_xmm_mem b_hi))) ;; aa_bb = ah_bl + al_bh - (aa_bb Reg (paddq ah_bl (RegMem.Reg al_bh))) + (aa_bb Xmm (paddq ah_bl (xmm_to_xmm_mem al_bh))) ;; aa_bb_shifted = aa_bb << 32 - (aa_bb_shifted Reg (psllq aa_bb (RegMemImm.Imm 32))) + (aa_bb_shifted Xmm (psllq aa_bb (xmm_mem_imm_new (RegMemImm.Imm 32)))) ;; al_bl = Al * Bl - (al_bl Reg (pmuludq a0 (RegMem.Reg b0)))) + (al_bl Xmm (pmuludq a0 (xmm_to_xmm_mem b0)))) ;; al_bl + aa_bb_shifted - (value_reg (paddq al_bl (RegMem.Reg aa_bb_shifted))))) + (value_xmm (paddq al_bl (xmm_to_xmm_mem aa_bb_shifted))))) ;; Special case for `i16x8.extmul_high_i8x16_s`. (rule (lower (has_type (multi_lane 16 8) @@ -1077,13 +1092,13 @@ x))) (def_inst (swiden_high (and (value_type (multi_lane 8 16)) y)))))) - (let ((x1 Reg (put_in_reg x)) - (x2 Reg (palignr x1 (RegMem.Reg x1) 8 (OperandSize.Size32))) - (x3 Reg (pmovsxbw (RegMem.Reg x2))) - (y1 Reg (put_in_reg y)) - (y2 Reg (palignr y1 (RegMem.Reg y1) 8 (OperandSize.Size32))) - (y3 Reg (pmovsxbw (RegMem.Reg y2)))) - (value_reg (pmullw x3 (RegMem.Reg y3))))) + (let ((x1 Xmm (put_in_xmm x)) + (x2 Xmm (palignr x1 (xmm_to_xmm_mem x1) 8 (OperandSize.Size32))) + (x3 Xmm (pmovsxbw (xmm_to_xmm_mem x2))) + (y1 Xmm (put_in_xmm y)) + (y2 Xmm (palignr y1 (xmm_to_xmm_mem y1) 8 (OperandSize.Size32))) + (y3 Xmm (pmovsxbw (xmm_to_xmm_mem y2)))) + (value_xmm (pmullw x3 (xmm_to_xmm_mem y3))))) ;; Special case for `i32x4.extmul_high_i16x8_s`. (rule (lower (has_type (multi_lane 32 4) @@ -1091,11 +1106,11 @@ x))) (def_inst (swiden_high (and (value_type (multi_lane 16 8)) y)))))) - (let ((x2 Reg (put_in_reg x)) - (y2 Reg (put_in_reg y)) - (lo Reg (pmullw x2 (RegMem.Reg y2))) - (hi Reg (pmulhw x2 (RegMem.Reg y2)))) - (value_reg (punpckhwd lo (RegMem.Reg hi))))) + (let ((x2 Xmm (put_in_xmm x)) + (y2 Xmm (put_in_xmm y)) + (lo Xmm (pmullw x2 (xmm_to_xmm_mem y2))) + (hi Xmm (pmulhw x2 (xmm_to_xmm_mem y2)))) + (value_xmm (punpckhwd lo (xmm_to_xmm_mem hi))))) ;; Special case for `i64x2.extmul_high_i32x4_s`. (rule (lower (has_type (multi_lane 64 2) @@ -1103,13 +1118,13 @@ x))) (def_inst (swiden_high (and (value_type (multi_lane 32 4)) y)))))) - (let ((x2 Reg (pshufd (put_in_reg_mem x) + (let ((x2 Xmm (pshufd (put_in_xmm_mem x) 0xFA (OperandSize.Size32))) - (y2 Reg (pshufd (put_in_reg_mem y) + (y2 Xmm (pshufd (put_in_xmm_mem y) 0xFA (OperandSize.Size32)))) - (value_reg (pmuldq x2 (RegMem.Reg y2))))) + (value_xmm (pmuldq x2 (xmm_to_xmm_mem y2))))) ;; Special case for `i16x8.extmul_low_i8x16_s`. (rule (lower (has_type (multi_lane 16 8) @@ -1117,9 +1132,9 @@ x))) (def_inst (swiden_low (and (value_type (multi_lane 8 16)) y)))))) - (let ((x2 Reg (pmovsxbw (put_in_reg_mem x))) - (y2 Reg (pmovsxbw (put_in_reg_mem y)))) - (value_reg (pmullw x2 (RegMem.Reg y2))))) + (let ((x2 Xmm (pmovsxbw (put_in_xmm_mem x))) + (y2 Xmm (pmovsxbw (put_in_xmm_mem y)))) + (value_xmm (pmullw x2 (xmm_to_xmm_mem y2))))) ;; Special case for `i32x4.extmul_low_i16x8_s`. (rule (lower (has_type (multi_lane 32 4) @@ -1127,11 +1142,11 @@ x))) (def_inst (swiden_low (and (value_type (multi_lane 16 8)) y)))))) - (let ((x2 Reg (put_in_reg x)) - (y2 Reg (put_in_reg y)) - (lo Reg (pmullw x2 (RegMem.Reg y2))) - (hi Reg (pmulhw x2 (RegMem.Reg y2)))) - (value_reg (punpcklwd lo (RegMem.Reg hi))))) + (let ((x2 Xmm (put_in_xmm x)) + (y2 Xmm (put_in_xmm y)) + (lo Xmm (pmullw x2 (xmm_to_xmm_mem y2))) + (hi Xmm (pmulhw x2 (xmm_to_xmm_mem y2)))) + (value_xmm (punpcklwd lo (xmm_to_xmm_mem hi))))) ;; Special case for `i64x2.extmul_low_i32x4_s`. (rule (lower (has_type (multi_lane 64 2) @@ -1139,13 +1154,13 @@ x))) (def_inst (swiden_low (and (value_type (multi_lane 32 4)) y)))))) - (let ((x2 Reg (pshufd (put_in_reg_mem x) + (let ((x2 Xmm (pshufd (put_in_xmm_mem x) 0x50 (OperandSize.Size32))) - (y2 Reg (pshufd (put_in_reg_mem y) + (y2 Xmm (pshufd (put_in_xmm_mem y) 0x50 (OperandSize.Size32)))) - (value_reg (pmuldq x2 (RegMem.Reg y2))))) + (value_xmm (pmuldq x2 (xmm_to_xmm_mem y2))))) ;; Special case for `i16x8.extmul_high_i8x16_u`. (rule (lower (has_type (multi_lane 16 8) @@ -1153,13 +1168,13 @@ x))) (def_inst (uwiden_high (and (value_type (multi_lane 8 16)) y)))))) - (let ((x1 Reg (put_in_reg x)) - (x2 Reg (palignr x1 (RegMem.Reg x1) 8 (OperandSize.Size32))) - (x3 Reg (pmovzxbw (RegMem.Reg x2))) - (y1 Reg (put_in_reg y)) - (y2 Reg (palignr y1 (RegMem.Reg y1) 8 (OperandSize.Size32))) - (y3 Reg (pmovzxbw (RegMem.Reg y2)))) - (value_reg (pmullw x3 (RegMem.Reg y3))))) + (let ((x1 Xmm (put_in_xmm x)) + (x2 Xmm (palignr x1 (xmm_to_xmm_mem x1) 8 (OperandSize.Size32))) + (x3 Xmm (pmovzxbw (xmm_to_xmm_mem x2))) + (y1 Xmm (put_in_xmm y)) + (y2 Xmm (palignr y1 (xmm_to_xmm_mem y1) 8 (OperandSize.Size32))) + (y3 Xmm (pmovzxbw (xmm_to_xmm_mem y2)))) + (value_xmm (pmullw x3 (xmm_to_xmm_mem y3))))) ;; Special case for `i32x4.extmul_high_i16x8_u`. (rule (lower (has_type (multi_lane 32 4) @@ -1167,11 +1182,11 @@ x))) (def_inst (uwiden_high (and (value_type (multi_lane 16 8)) y)))))) - (let ((x2 Reg (put_in_reg x)) - (y2 Reg (put_in_reg y)) - (lo Reg (pmullw x2 (RegMem.Reg y2))) - (hi Reg (pmulhuw x2 (RegMem.Reg y2)))) - (value_reg (punpckhwd lo (RegMem.Reg hi))))) + (let ((x2 Xmm (put_in_xmm x)) + (y2 Xmm (put_in_xmm y)) + (lo Xmm (pmullw x2 (xmm_to_xmm_mem y2))) + (hi Xmm (pmulhuw x2 (xmm_to_xmm_mem y2)))) + (value_xmm (punpckhwd lo (xmm_to_xmm_mem hi))))) ;; Special case for `i64x2.extmul_high_i32x4_u`. (rule (lower (has_type (multi_lane 64 2) @@ -1179,13 +1194,13 @@ x))) (def_inst (uwiden_high (and (value_type (multi_lane 32 4)) y)))))) - (let ((x2 Reg (pshufd (put_in_reg_mem x) + (let ((x2 Xmm (pshufd (put_in_xmm_mem x) 0xFA (OperandSize.Size32))) - (y2 Reg (pshufd (put_in_reg_mem y) + (y2 Xmm (pshufd (put_in_xmm_mem y) 0xFA (OperandSize.Size32)))) - (value_reg (pmuludq x2 (RegMem.Reg y2))))) + (value_xmm (pmuludq x2 (xmm_to_xmm_mem y2))))) ;; Special case for `i16x8.extmul_low_i8x16_u`. (rule (lower (has_type (multi_lane 16 8) @@ -1193,9 +1208,9 @@ x))) (def_inst (uwiden_low (and (value_type (multi_lane 8 16)) y)))))) - (let ((x2 Reg (pmovzxbw (put_in_reg_mem x))) - (y2 Reg (pmovzxbw (put_in_reg_mem y)))) - (value_reg (pmullw x2 (RegMem.Reg y2))))) + (let ((x2 Xmm (pmovzxbw (put_in_xmm_mem x))) + (y2 Xmm (pmovzxbw (put_in_xmm_mem y)))) + (value_xmm (pmullw x2 (xmm_to_xmm_mem y2))))) ;; Special case for `i32x4.extmul_low_i16x8_u`. (rule (lower (has_type (multi_lane 32 4) @@ -1203,11 +1218,11 @@ x))) (def_inst (uwiden_low (and (value_type (multi_lane 16 8)) y)))))) - (let ((x2 Reg (put_in_reg x)) - (y2 Reg (put_in_reg y)) - (lo Reg (pmullw x2 (RegMem.Reg y2))) - (hi Reg (pmulhuw x2 (RegMem.Reg y2)))) - (value_reg (punpcklwd lo (RegMem.Reg hi))))) + (let ((x2 Xmm (put_in_xmm x)) + (y2 Xmm (put_in_xmm y)) + (lo Xmm (pmullw x2 (xmm_to_xmm_mem y2))) + (hi Xmm (pmulhuw x2 (xmm_to_xmm_mem y2)))) + (value_xmm (punpcklwd lo (xmm_to_xmm_mem hi))))) ;; Special case for `i64x2.extmul_low_i32x4_u`. (rule (lower (has_type (multi_lane 64 2) @@ -1215,17 +1230,17 @@ x))) (def_inst (uwiden_low (and (value_type (multi_lane 32 4)) y)))))) - (let ((x2 Reg (pshufd (put_in_reg_mem x) + (let ((x2 Xmm (pshufd (put_in_xmm_mem x) 0x50 (OperandSize.Size32))) - (y2 Reg (pshufd (put_in_reg_mem y) + (y2 Xmm (pshufd (put_in_xmm_mem y) 0x50 (OperandSize.Size32)))) - (value_reg (pmuludq x2 (RegMem.Reg y2))))) + (value_xmm (pmuludq x2 (xmm_to_xmm_mem y2))))) ;;;; Rules for `band_not` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -(decl sse_and_not (Type Reg RegMem) Reg) +(decl sse_and_not (Type Xmm XmmMem) Xmm) (rule (sse_and_not $F32X4 x y) (andnps x y)) (rule (sse_and_not $F64X2 x y) (andnpd x y)) (rule (sse_and_not (multi_lane _bits _lanes) x y) (pandn x y)) @@ -1238,64 +1253,66 @@ ;; ;; pandn(x, y) = and(not(x), y) (rule (lower (has_type ty (band_not x y))) - (value_reg (sse_and_not ty - (put_in_reg y) - (put_in_reg_mem x)))) + (value_xmm (sse_and_not ty + (put_in_xmm y) + (put_in_xmm_mem x)))) ;;;; Rules for `iabs` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (rule (lower (has_type $I8X16 (iabs x))) - (value_reg (pabsb (put_in_reg_mem x)))) + (value_xmm (pabsb (put_in_xmm_mem x)))) (rule (lower (has_type $I16X8 (iabs x))) - (value_reg (pabsw (put_in_reg_mem x)))) + (value_xmm (pabsw (put_in_xmm_mem x)))) (rule (lower (has_type $I32X4 (iabs x))) - (value_reg (pabsd (put_in_reg_mem x)))) + (value_xmm (pabsd (put_in_xmm_mem x)))) ;; When AVX512 is available, we can use a single `vpabsq` instruction. (rule (lower (has_type (and (avx512vl_enabled) (avx512f_enabled) $I64X2) (iabs x))) - (value_reg (vpabsq (put_in_reg_mem x)))) + (value_xmm (vpabsq (put_in_xmm_mem x)))) -;; Otherwise, we use a separate register, `neg`, to contain the results of `0 - +;; Otherwise, we use a separate xmmister, `neg`, to contain the results of `0 - ;; x` and then blend in those results with `blendvpd` if the MSB of `neg` was ;; set to 1 (i.e. if `neg` was negative or, conversely, if `x` was originally ;; positive). (rule (lower (has_type $I64X2 (iabs x))) - (let ((rx Reg (put_in_reg x)) - (neg Reg (psubq (imm $I64X2 0) (RegMem.Reg rx)))) - (value_reg (blendvpd neg (RegMem.Reg rx) neg)))) + (let ((rx Xmm (put_in_xmm x)) + (neg Xmm (psubq (xmm_new (imm $I64X2 0)) (xmm_to_xmm_mem rx)))) + (value_xmm (blendvpd neg (xmm_to_xmm_mem rx) neg)))) ;;;; Rules for `fabs` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; Special case for `f32x4.abs`. (rule (lower (has_type $F32X4 (fabs x))) - (value_reg (andps (put_in_reg x) - (RegMem.Reg (psrld (vector_all_ones $F32X4) (RegMemImm.Imm 1)))))) + (value_xmm (andps (put_in_xmm x) + (xmm_to_xmm_mem (psrld (vector_all_ones $F32X4) + (xmm_mem_imm_new (RegMemImm.Imm 1))))))) ;; Special case for `f64x2.abs`. (rule (lower (has_type $F64X2 (fabs x))) - (value_reg (andpd (put_in_reg x) - (RegMem.Reg (psrlq (vector_all_ones $F64X2) (RegMemImm.Imm 1)))))) + (value_xmm (andpd (put_in_xmm x) + (xmm_to_xmm_mem (psrlq (vector_all_ones $F64X2) + (xmm_mem_imm_new (RegMemImm.Imm 1))))))) ;;;; Rules for `bnot` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; `i64` and smaller. (rule (lower (has_type (fits_in_64 ty) (bnot x))) - (value_reg (not ty (put_in_reg x)))) + (value_gpr (not ty (put_in_gpr x)))) ;; `i128`. (decl i128_not (Value) ValueRegs) (rule (i128_not x) (let ((x_regs ValueRegs (put_in_regs x)) - (x_lo Reg (value_regs_get x_regs 0)) - (x_hi Reg (value_regs_get x_regs 1))) - (value_regs (not $I64 x_lo) + (x_lo Gpr (gpr_new (value_regs_get x_regs 0))) + (x_hi Gpr (gpr_new (value_regs_get x_regs 1)))) + (value_gprs (not $I64 x_lo) (not $I64 x_hi)))) (rule (lower (has_type $I128 (bnot x))) @@ -1307,7 +1324,7 @@ ;; Special case for vector-types where bit-negation is an xor against an ;; all-one value (rule (lower (has_type ty @ (multi_lane _bits _lanes) (bnot x))) - (value_reg (sse_xor ty (put_in_reg x) (RegMem.Reg (vector_all_ones ty))))) + (value_xmm (sse_xor ty (put_in_xmm x) (xmm_to_xmm_mem (vector_all_ones ty))))) ;;;; Rules for `bitselect` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -1318,48 +1335,53 @@ ;; a = and if_true, condition ;; b = and_not condition, if_false ;; or b, a - (let ((cond_reg Reg (put_in_reg condition)) - (a Reg (sse_and ty (put_in_reg if_true) (RegMem.Reg cond_reg))) - (b Reg (sse_and_not ty cond_reg (put_in_reg_mem if_false)))) - (value_reg (sse_or ty b (RegMem.Reg a))))) + (let ((cond_xmm Xmm (put_in_xmm condition)) + (a Xmm (sse_and ty (put_in_xmm if_true) (xmm_to_xmm_mem cond_xmm))) + (b Xmm (sse_and_not ty cond_xmm (put_in_xmm_mem if_false)))) + (value_xmm (sse_or ty b (xmm_to_xmm_mem a))))) ;;;; Rules for `vselect` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (rule (lower (has_type ty @ (multi_lane _bits _lanes) (vselect condition if_true if_false))) - (value_reg (sse_blend ty - (put_in_reg_mem condition) - (put_in_reg_mem if_true) - (put_in_reg if_false)))) + (value_xmm (sse_blend ty + (put_in_xmm_mem condition) + (put_in_xmm_mem if_true) + (put_in_xmm if_false)))) ;;;; Rules for `insertlane` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (rule (lower (insertlane vec @ (value_type ty) val (u8_from_uimm8 idx))) - (value_reg (vec_insert_lane ty (put_in_reg vec) (put_in_reg_mem val) idx))) + (value_xmm (vec_insert_lane ty (put_in_xmm vec) (put_in_reg_mem val) idx))) ;; Helper function used below for `insertlane` but also here for other ;; lowerings. ;; ;; Note that the `Type` used here is the type of vector the insertion is ;; happening into, or the type of the first `Reg` argument. -(decl vec_insert_lane (Type Reg RegMem u8) Reg) +(decl vec_insert_lane (Type Xmm RegMem u8) Xmm) ;; i8x16.replace_lane -(rule (vec_insert_lane $I8X16 vec val idx) (pinsrb vec val idx)) +(rule (vec_insert_lane $I8X16 vec val idx) + (pinsrb vec (gpr_mem_new val) idx)) ;; i16x8.replace_lane -(rule (vec_insert_lane $I16X8 vec val idx) (pinsrw vec val idx)) +(rule (vec_insert_lane $I16X8 vec val idx) + (pinsrw vec (gpr_mem_new val) idx)) ;; i32x4.replace_lane -(rule (vec_insert_lane $I32X4 vec val idx) (pinsrd vec val idx (OperandSize.Size32))) +(rule (vec_insert_lane $I32X4 vec val idx) + (pinsrd vec (gpr_mem_new val) idx (OperandSize.Size32))) ;; i64x2.replace_lane -(rule (vec_insert_lane $I64X2 vec val idx) (pinsrd vec val idx (OperandSize.Size64))) +(rule (vec_insert_lane $I64X2 vec val idx) + (pinsrd vec (gpr_mem_new val) idx (OperandSize.Size64))) ;; f32x4.replace_lane -(rule (vec_insert_lane $F32X4 vec val idx) (insertps vec val (sse_insertps_lane_imm idx))) +(rule (vec_insert_lane $F32X4 vec val idx) + (insertps vec (xmm_mem_new val) (sse_insertps_lane_imm idx))) -;; external rust code used to calculate the immediate value to `insertps` +;; External rust code used to calculate the immediate value to `insertps`. (decl sse_insertps_lane_imm (u8) u8) (extern constructor sse_insertps_lane_imm sse_insertps_lane_imm) @@ -1378,60 +1400,63 @@ ;; load from memory into a temp register and then the second `movsd` (modeled ;; internally as `xmm_rm_r` will merge the temp register into our `vec` ;; register. -(rule (vec_insert_lane $F64X2 vec (RegMem.Reg val) 0) (movsd vec (RegMem.Reg val))) +(rule (vec_insert_lane $F64X2 vec (RegMem.Reg val) 0) + (movsd vec (xmm_mem_new (RegMem.Reg val)))) (rule (vec_insert_lane $F64X2 vec mem 0) - (movsd vec (RegMem.Reg (xmm_unary_rm_r (SseOpcode.Movsd) mem)))) + (movsd vec (xmm_to_xmm_mem (xmm_unary_rm_r (SseOpcode.Movsd) + (xmm_mem_new mem))))) ;; f64x2.replace_lane 1 ;; ;; Here the `movlhps` instruction is used specifically to specialize moving ;; into the second lane where unlike above cases we're not using the lane ;; immediate as an immediate to the instruction itself. -(rule (vec_insert_lane $F64X2 vec val 1) (movlhps vec val)) +(rule (vec_insert_lane $F64X2 vec val 1) + (movlhps vec (xmm_mem_new val))) ;;;; Rules for `imax` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (rule (lower (has_type $I8X16 (imax x y))) - (value_reg (pmaxsb (put_in_reg x) (put_in_reg_mem y)))) + (value_xmm (pmaxsb (put_in_xmm x) (put_in_xmm_mem y)))) (rule (lower (has_type $I16X8 (imax x y))) - (value_reg (pmaxsw (put_in_reg x) (put_in_reg_mem y)))) + (value_xmm (pmaxsw (put_in_xmm x) (put_in_xmm_mem y)))) (rule (lower (has_type $I32X4 (imax x y))) - (value_reg (pmaxsd (put_in_reg x) (put_in_reg_mem y)))) + (value_xmm (pmaxsd (put_in_xmm x) (put_in_xmm_mem y)))) ;;;; Rules for `imin` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (rule (lower (has_type $I8X16 (imin x y))) - (value_reg (pminsb (put_in_reg x) (put_in_reg_mem y)))) + (value_xmm (pminsb (put_in_xmm x) (put_in_xmm_mem y)))) (rule (lower (has_type $I16X8 (imin x y))) - (value_reg (pminsw (put_in_reg x) (put_in_reg_mem y)))) + (value_xmm (pminsw (put_in_xmm x) (put_in_xmm_mem y)))) (rule (lower (has_type $I32X4 (imin x y))) - (value_reg (pminsd (put_in_reg x) (put_in_reg_mem y)))) + (value_xmm (pminsd (put_in_xmm x) (put_in_xmm_mem y)))) ;;;; Rules for `umax` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (rule (lower (has_type $I8X16 (umax x y))) - (value_reg (pmaxub (put_in_reg x) (put_in_reg_mem y)))) + (value_xmm (pmaxub (put_in_xmm x) (put_in_xmm_mem y)))) (rule (lower (has_type $I16X8 (umax x y))) - (value_reg (pmaxuw (put_in_reg x) (put_in_reg_mem y)))) + (value_xmm (pmaxuw (put_in_xmm x) (put_in_xmm_mem y)))) (rule (lower (has_type $I32X4 (umax x y))) - (value_reg (pmaxud (put_in_reg x) (put_in_reg_mem y)))) + (value_xmm (pmaxud (put_in_xmm x) (put_in_xmm_mem y)))) ;;;; Rules for `umin` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (rule (lower (has_type $I8X16 (umin x y))) - (value_reg (pminub (put_in_reg x) (put_in_reg_mem y)))) + (value_xmm (pminub (put_in_xmm x) (put_in_xmm_mem y)))) (rule (lower (has_type $I16X8 (umin x y))) - (value_reg (pminuw (put_in_reg x) (put_in_reg_mem y)))) + (value_xmm (pminuw (put_in_xmm x) (put_in_xmm_mem y)))) (rule (lower (has_type $I32X4 (umin x y))) - (value_reg (pminud (put_in_reg x) (put_in_reg_mem y)))) + (value_xmm (pminud (put_in_xmm x) (put_in_xmm_mem y)))) ;;;; Rules for `trap` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; diff --git a/cranelift/codegen/src/isa/x64/lower.rs b/cranelift/codegen/src/isa/x64/lower.rs index d5503416de9f..3e74dddda93d 100644 --- a/cranelift/codegen/src/isa/x64/lower.rs +++ b/cranelift/codegen/src/isa/x64/lower.rs @@ -1,7 +1,7 @@ //! Lowering rules for X64. // ISLE integration glue. -mod isle; +pub(super) mod isle; use crate::data_value::DataValue; use crate::ir::{ @@ -1057,7 +1057,13 @@ fn lower_to_amode>(ctx: &mut C, spec: InsnInput, offset: i ) }; - return Amode::imm_reg_reg_shift(offset as u32, base, index, shift).with_flags(flags); + return Amode::imm_reg_reg_shift( + offset as u32, + Gpr::new(base).unwrap(), + Gpr::new(index).unwrap(), + shift, + ) + .with_flags(flags); } let input = put_input_in_reg(ctx, spec); @@ -3950,7 +3956,13 @@ fn lower_insn_to_regs>( let index = put_input_in_reg(ctx, inputs[1]); let shift = 0; let flags = ctx.memflags(insn).expect("load should have memflags"); - Amode::imm_reg_reg_shift(offset as u32, base, index, shift).with_flags(flags) + Amode::imm_reg_reg_shift( + offset as u32, + Gpr::new(base).unwrap(), + Gpr::new(index).unwrap(), + shift, + ) + .with_flags(flags) } _ => unreachable!(), }; @@ -4054,7 +4066,13 @@ fn lower_insn_to_regs>( let index = put_input_in_reg(ctx, inputs[2]); let shift = 0; let flags = ctx.memflags(insn).expect("store should have memflags"); - Amode::imm_reg_reg_shift(offset as u32, base, index, shift).with_flags(flags) + Amode::imm_reg_reg_shift( + offset as u32, + Gpr::new(base).unwrap(), + Gpr::new(index).unwrap(), + shift, + ) + .with_flags(flags) } _ => unreachable!(), diff --git a/cranelift/codegen/src/isa/x64/lower/isle.rs b/cranelift/codegen/src/isa/x64/lower/isle.rs index 613b6832d792..87a5ea9f6ece 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle.rs +++ b/cranelift/codegen/src/isa/x64/lower/isle.rs @@ -1,26 +1,28 @@ //! ISLE integration glue code for x64 lowering. // Pull in the ISLE generated code. -mod generated_code; +pub(crate) mod generated_code; +use generated_code::MInst; +use regalloc::Writable; // Types that the generated ISLE code uses via `use super::*`. -use super::{ - is_mergeable_load, lower_to_amode, AluRmiROpcode, Inst as MInst, OperandSize, Reg, RegMemImm, -}; +use super::{is_mergeable_load, lower_to_amode, Reg}; use crate::{ - ir::{immediates::*, types::*, Inst, InstructionData, Opcode, TrapCode, Value, ValueList}, - isa::x64::{ - inst::{ - args::{ - Amode, Avx512Opcode, CmpOpcode, ExtKind, ExtMode, FcmpImm, Imm8Reg, RegMem, - ShiftKind, SseOpcode, SyntheticAmode, CC, - }, - regs, x64_map_regs, + ir::{ + immediates::*, types::*, Inst, InstructionData, Opcode, TrapCode, Value, ValueLabel, + ValueList, + }, + isa::{ + settings::Flags, + unwind::UnwindInst, + x64::{ + inst::{args::*, regs, x64_map_regs}, + settings::Flags as IsaFlags, }, - settings::Flags as IsaFlags, }, - machinst::{isle::*, InsnInput, InsnOutput, LowerCtx, VCodeConstantData}, - settings::Flags, + machinst::{ + isle::*, AtomicRmwOp, InsnInput, InsnOutput, LowerCtx, VCodeConstant, VCodeConstantData, + }, }; use std::convert::TryFrom; @@ -252,8 +254,8 @@ where } #[inline] - fn xmm0(&mut self) -> WritableReg { - WritableReg::from_reg(regs::xmm0()) + fn xmm0(&mut self) -> WritableXmm { + WritableXmm::from_reg(Xmm::new(regs::xmm0()).unwrap()) } #[inline] @@ -262,7 +264,7 @@ where } #[inline] - fn amode_imm_reg_reg_shift(&mut self, simm32: u32, base: Reg, index: Reg, shift: u8) -> Amode { + fn amode_imm_reg_reg_shift(&mut self, simm32: u32, base: Gpr, index: Gpr, shift: u8) -> Amode { Amode::imm_reg_reg_shift(simm32, base, index, shift) } @@ -271,6 +273,16 @@ where amode.clone().into() } + #[inline] + fn writable_gpr_to_reg(&mut self, r: WritableGpr) -> WritableReg { + r.to_writable_reg() + } + + #[inline] + fn writable_xmm_to_reg(&mut self, r: WritableXmm) -> WritableReg { + r.to_writable_reg() + } + fn ishl_i8x16_mask_for_const(&mut self, amt: u32) -> SyntheticAmode { // When the shift amount is known, we can statically (i.e. at compile // time) determine the mask to use and only emit that. @@ -306,6 +318,96 @@ where .use_constant(VCodeConstantData::WellKnown(&I8X16_USHR_MASKS)); SyntheticAmode::ConstantOffset(mask_table) } + + #[inline] + fn writable_reg_to_xmm(&mut self, r: WritableReg) -> WritableXmm { + Writable::from_reg(Xmm::new(r.to_reg()).unwrap()) + } + + #[inline] + fn writable_xmm_to_xmm(&mut self, r: WritableXmm) -> Xmm { + r.to_reg() + } + + #[inline] + fn writable_gpr_to_gpr(&mut self, r: WritableGpr) -> Gpr { + r.to_reg() + } + + #[inline] + fn gpr_to_reg(&mut self, r: Gpr) -> Reg { + r.into() + } + + #[inline] + fn xmm_to_reg(&mut self, r: Xmm) -> Reg { + r.into() + } + + #[inline] + fn xmm_to_xmm_mem_imm(&mut self, r: Xmm) -> XmmMemImm { + r.into() + } + + #[inline] + fn temp_writable_gpr(&mut self) -> WritableGpr { + Writable::from_reg(Gpr::new(self.temp_writable_reg(I64).to_reg()).unwrap()) + } + + #[inline] + fn temp_writable_xmm(&mut self) -> WritableXmm { + Writable::from_reg(Xmm::new(self.temp_writable_reg(I8X16).to_reg()).unwrap()) + } + + #[inline] + fn xmm_mem_new(&mut self, rm: &RegMem) -> XmmMem { + XmmMem::new(rm.clone()).unwrap() + } + + #[inline] + fn gpr_mem_imm_new(&mut self, rmi: &RegMemImm) -> GprMemImm { + GprMemImm::new(rmi.clone()).unwrap() + } + + #[inline] + fn xmm_mem_imm_new(&mut self, rmi: &RegMemImm) -> XmmMemImm { + XmmMemImm::new(rmi.clone()).unwrap() + } + + #[inline] + fn xmm_to_xmm_mem(&mut self, r: Xmm) -> XmmMem { + r.into() + } + + #[inline] + fn xmm_mem_to_reg_mem(&mut self, xm: &XmmMem) -> RegMem { + xm.clone().into() + } + + #[inline] + fn gpr_mem_to_reg_mem(&mut self, gm: &GprMem) -> RegMem { + gm.clone().into() + } + + #[inline] + fn xmm_new(&mut self, r: Reg) -> Xmm { + Xmm::new(r).unwrap() + } + + #[inline] + fn gpr_new(&mut self, r: Reg) -> Gpr { + Gpr::new(r).unwrap() + } + + #[inline] + fn gpr_mem_new(&mut self, rm: &RegMem) -> GprMem { + GprMem::new(rm.clone()).unwrap() + } + + #[inline] + fn reg_to_gpr_mem(&mut self, r: Reg) -> GprMem { + GprMem::new(RegMem::reg(r)).unwrap() + } } // Since x64 doesn't have 8x16 shifts and we must use a 16x8 shift instead, we diff --git a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest index 98720addfbd4..2c8fb4b29805 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest +++ b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest @@ -1,4 +1,4 @@ src/clif.isle 9ea75a6f790b5c03 -src/prelude.isle 2bfcafbef6b29358 -src/isa/x64/inst.isle bbb6a3d201200cc8 -src/isa/x64/lower.isle 82db7f7d47ac7809 +src/prelude.isle 6aaf8ce0f5a5c2ec +src/isa/x64/inst.isle 2f76eb1f9ecf0c5e +src/isa/x64/lower.isle 144c33c4e64a17a7 diff --git a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs index 89e9f0f9a27e..31aaea3e4844 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs +++ b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs @@ -70,11 +70,31 @@ pub trait Context { fn put_in_reg_mem_imm(&mut self, arg0: Value) -> RegMemImm; fn put_in_reg_mem(&mut self, arg0: Value) -> RegMem; fn synthetic_amode_to_reg_mem(&mut self, arg0: &SyntheticAmode) -> RegMem; - fn amode_imm_reg_reg_shift(&mut self, arg0: u32, arg1: Reg, arg2: Reg, arg3: u8) -> Amode; + fn amode_imm_reg_reg_shift(&mut self, arg0: u32, arg1: Gpr, arg2: Gpr, arg3: u8) -> Amode; fn amode_to_synthetic_amode(&mut self, arg0: &Amode) -> SyntheticAmode; fn put_masked_in_imm8_reg(&mut self, arg0: Value, arg1: Type) -> Imm8Reg; fn encode_fcmp_imm(&mut self, arg0: &FcmpImm) -> u8; - fn xmm0(&mut self) -> WritableReg; + fn writable_gpr_to_reg(&mut self, arg0: WritableGpr) -> WritableReg; + fn writable_xmm_to_reg(&mut self, arg0: WritableXmm) -> WritableReg; + fn writable_reg_to_xmm(&mut self, arg0: WritableReg) -> WritableXmm; + fn writable_xmm_to_xmm(&mut self, arg0: WritableXmm) -> Xmm; + fn writable_gpr_to_gpr(&mut self, arg0: WritableGpr) -> Gpr; + fn gpr_to_reg(&mut self, arg0: Gpr) -> Reg; + fn xmm_to_reg(&mut self, arg0: Xmm) -> Reg; + fn xmm_to_xmm_mem_imm(&mut self, arg0: Xmm) -> XmmMemImm; + fn temp_writable_gpr(&mut self) -> WritableGpr; + fn temp_writable_xmm(&mut self) -> WritableXmm; + fn xmm_mem_new(&mut self, arg0: &RegMem) -> XmmMem; + fn gpr_mem_imm_new(&mut self, arg0: &RegMemImm) -> GprMemImm; + fn xmm_mem_imm_new(&mut self, arg0: &RegMemImm) -> XmmMemImm; + fn xmm_to_xmm_mem(&mut self, arg0: Xmm) -> XmmMem; + fn xmm_mem_to_reg_mem(&mut self, arg0: &XmmMem) -> RegMem; + fn gpr_mem_to_reg_mem(&mut self, arg0: &GprMem) -> RegMem; + fn xmm_new(&mut self, arg0: Reg) -> Xmm; + fn gpr_new(&mut self, arg0: Reg) -> Gpr; + fn gpr_mem_new(&mut self, arg0: &RegMem) -> GprMem; + fn reg_to_gpr_mem(&mut self, arg0: Reg) -> GprMem; + fn xmm0(&mut self) -> WritableXmm; fn avx512vl_enabled(&mut self, arg0: Type) -> Option<()>; fn avx512dq_enabled(&mut self, arg0: Type) -> Option<()>; fn avx512f_enabled(&mut self, arg0: Type) -> Option<()>; @@ -93,26 +113,337 @@ pub trait Context { fn sse_insertps_lane_imm(&mut self, arg0: u8) -> u8; } -/// Internal type SideEffectNoResult: defined at src/prelude.isle line 295. +/// Internal type SideEffectNoResult: defined at src/prelude.isle line 307. #[derive(Clone, Debug)] pub enum SideEffectNoResult { Inst { inst: MInst }, } -/// Internal type ProducesFlags: defined at src/prelude.isle line 314. +/// Internal type ProducesFlags: defined at src/prelude.isle line 326. #[derive(Clone, Debug)] pub enum ProducesFlags { ProducesFlags { inst: MInst, result: Reg }, } -/// Internal type ConsumesFlags: defined at src/prelude.isle line 317. +/// Internal type ConsumesFlags: defined at src/prelude.isle line 329. #[derive(Clone, Debug)] pub enum ConsumesFlags { ConsumesFlags { inst: MInst, result: Reg }, } -/// Internal type ExtendKind: defined at src/isa/x64/inst.isle line 464. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type MInst: defined at src/isa/x64/inst.isle line 8. +#[derive(Clone)] +pub enum MInst { + Nop { + len: u8, + }, + AluRmiR { + size: OperandSize, + op: AluRmiROpcode, + src1: Reg, + src2: RegMemImm, + dst: WritableReg, + }, + UnaryRmR { + size: OperandSize, + op: UnaryRmROpcode, + src: GprMem, + dst: WritableGpr, + }, + Not { + size: OperandSize, + src: Gpr, + dst: WritableGpr, + }, + Neg { + size: OperandSize, + src: Gpr, + dst: WritableGpr, + }, + Div { + size: OperandSize, + signed: bool, + divisor: RegMem, + dividend: Reg, + dst_quotient: WritableReg, + dst_remainder: WritableReg, + }, + MulHi { + size: OperandSize, + signed: bool, + src1: Reg, + src2: RegMem, + dst_lo: WritableReg, + dst_hi: WritableReg, + }, + CheckedDivOrRemSeq { + kind: DivOrRemKind, + size: OperandSize, + dividend: Reg, + divisor: WritableReg, + dst_quotient: WritableReg, + dst_remainder: WritableReg, + tmp: OptionWritableReg, + }, + SignExtendData { + size: OperandSize, + src: Reg, + dst: WritableReg, + }, + Imm { + dst_size: OperandSize, + simm64: u64, + dst: WritableReg, + }, + MovRR { + size: OperandSize, + src: Reg, + dst: WritableReg, + }, + MovzxRmR { + ext_mode: ExtMode, + src: RegMem, + dst: WritableReg, + }, + Mov64MR { + src: SyntheticAmode, + dst: WritableReg, + }, + LoadEffectiveAddress { + addr: SyntheticAmode, + dst: WritableGpr, + }, + MovsxRmR { + ext_mode: ExtMode, + src: RegMem, + dst: WritableReg, + }, + MovRM { + size: OperandSize, + src: Reg, + dst: SyntheticAmode, + }, + ShiftR { + size: OperandSize, + kind: ShiftKind, + src: Reg, + num_bits: Imm8Reg, + dst: WritableReg, + }, + XmmRmiReg { + opcode: SseOpcode, + src1: Xmm, + src2: XmmMemImm, + dst: WritableXmm, + }, + CmpRmiR { + size: OperandSize, + opcode: CmpOpcode, + src: RegMemImm, + dst: Reg, + }, + Setcc { + cc: CC, + dst: WritableReg, + }, + Cmove { + size: OperandSize, + cc: CC, + consequent: RegMem, + alternative: Reg, + dst: WritableReg, + }, + Push64 { + src: RegMemImm, + }, + Pop64 { + dst: WritableReg, + }, + XmmRmR { + op: SseOpcode, + src1: Xmm, + src2: XmmMem, + dst: WritableXmm, + }, + XmmRmREvex { + op: Avx512Opcode, + src1: XmmMem, + src2: Xmm, + dst: WritableXmm, + }, + XmmUnaryRmR { + op: SseOpcode, + src: XmmMem, + dst: WritableXmm, + }, + XmmUnaryRmREvex { + op: Avx512Opcode, + src: XmmMem, + dst: WritableXmm, + }, + XmmMovRM { + op: SseOpcode, + src: Reg, + dst: SyntheticAmode, + }, + XmmLoadConst { + src: VCodeConstant, + dst: WritableReg, + ty: Type, + }, + XmmToGpr { + op: SseOpcode, + src: Xmm, + dst: WritableGpr, + dst_size: OperandSize, + }, + GprToXmm { + op: SseOpcode, + src: RegMem, + dst: WritableXmm, + src_size: OperandSize, + }, + CvtUint64ToFloatSeq { + dst_size: OperandSize, + src: WritableGpr, + dst: WritableXmm, + tmp_gpr1: WritableGpr, + tmp_gpr2: WritableGpr, + }, + CvtFloatToSintSeq { + dst_size: OperandSize, + src_size: OperandSize, + is_saturating: bool, + src: WritableXmm, + dst: WritableGpr, + tmp_gpr: WritableGpr, + tmp_xmm: WritableXmm, + }, + CvtFloatToUintSeq { + dst_size: OperandSize, + src_size: OperandSize, + is_saturating: bool, + src: WritableXmm, + dst: WritableGpr, + tmp_gpr: WritableGpr, + tmp_xmm: WritableXmm, + }, + XmmMinMaxSeq { + size: OperandSize, + is_min: bool, + lhs: Reg, + rhs_dst: WritableReg, + }, + XmmCmove { + size: OperandSize, + cc: CC, + src: RegMem, + dst: WritableReg, + }, + XmmCmpRmR { + op: SseOpcode, + src: RegMem, + dst: Reg, + }, + XmmRmRImm { + op: SseOpcode, + src1: Reg, + src2: RegMem, + dst: WritableReg, + imm: u8, + size: OperandSize, + }, + CallKnown { + dest: ExternalName, + uses: VecReg, + defs: VecWritableReg, + opcode: Opcode, + }, + CallUnknown { + dest: RegMem, + uses: VecReg, + defs: VecWritableReg, + opcode: Opcode, + }, + Ret, + EpiloguePlaceholder, + JmpKnown { + dst: MachLabel, + }, + JmpIf { + cc: CC, + taken: MachLabel, + }, + JmpCond { + cc: CC, + taken: MachLabel, + not_taken: MachLabel, + }, + JmpTableSeq { + idx: Reg, + tmp1: WritableReg, + tmp2: WritableReg, + default_target: MachLabel, + targets: VecMachLabel, + targets_for_term: VecMachLabel, + }, + JmpUnknown { + target: RegMem, + }, + TrapIf { + cc: CC, + trap_code: TrapCode, + }, + Hlt, + Ud2 { + trap_code: TrapCode, + }, + LoadExtName { + dst: WritableReg, + name: BoxExternalName, + offset: i64, + }, + LockCmpxchg { + ty: Type, + replacement: Reg, + expected: Reg, + mem: SyntheticAmode, + dst_old: WritableReg, + }, + AtomicRmwSeq { + ty: Type, + op: AtomicRmwOp, + address: Reg, + operand: Reg, + temp: WritableReg, + dst_old: WritableReg, + }, + Fence { + kind: FenceKind, + }, + VirtualSPOffsetAdj { + offset: i64, + }, + XmmUninitializedValue { + dst: WritableXmm, + }, + ElfTlsGetAddr { + symbol: ExternalName, + }, + MachOTlsGetAddr { + symbol: ExternalName, + }, + ValueLabelMarker { + reg: Reg, + label: ValueLabel, + }, + Unwind { + inst: UnwindInst, + }, +} + +/// Internal type ExtendKind: defined at src/isa/x64/inst.isle line 1051. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum ExtendKind { Sign, Zero, @@ -121,7 +452,7 @@ pub enum ExtendKind { // Generated as internal constructor for term temp_reg. pub fn constructor_temp_reg(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; - // Rule at src/prelude.isle line 70. + // Rule at src/prelude.isle line 73. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); return Some(expr1_0); @@ -130,7 +461,7 @@ pub fn constructor_temp_reg(ctx: &mut C, arg0: Type) -> Option // Generated as internal constructor for term lo_reg. pub fn constructor_lo_reg(ctx: &mut C, arg0: Value) -> Option { let pattern0_0 = arg0; - // Rule at src/prelude.isle line 105. + // Rule at src/prelude.isle line 108. let expr0_0 = C::put_in_regs(ctx, pattern0_0); let expr1_0: usize = 0; let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); @@ -147,7 +478,7 @@ pub fn constructor_value_regs_none( inst: ref pattern1_0, } = pattern0_0 { - // Rule at src/prelude.isle line 300. + // Rule at src/prelude.isle line 312. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::value_regs_invalid(ctx); return Some(expr1_0); @@ -165,7 +496,7 @@ pub fn constructor_safepoint( inst: ref pattern1_0, } = pattern0_0 { - // Rule at src/prelude.isle line 306. + // Rule at src/prelude.isle line 318. let expr0_0 = C::emit_safepoint(ctx, &pattern1_0); let expr1_0 = C::value_regs_invalid(ctx); return Some(expr1_0); @@ -191,7 +522,7 @@ pub fn constructor_with_flags( result: pattern3_1, } = pattern2_0 { - // Rule at src/prelude.isle line 327. + // Rule at src/prelude.isle line 339. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern3_0); let expr2_0 = C::value_regs(ctx, pattern1_1, pattern3_1); @@ -219,7 +550,7 @@ pub fn constructor_with_flags_1( result: pattern3_1, } = pattern2_0 { - // Rule at src/prelude.isle line 335. + // Rule at src/prelude.isle line 347. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern3_0); return Some(pattern3_1); @@ -253,7 +584,7 @@ pub fn constructor_with_flags_2( result: pattern5_1, } = pattern4_0 { - // Rule at src/prelude.isle line 345. + // Rule at src/prelude.isle line 357. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern5_0); let expr2_0 = C::emit(ctx, &pattern3_0); @@ -270,22 +601,22 @@ pub fn constructor_operand_size_bits(ctx: &mut C, arg0: &OperandSize let pattern0_0 = arg0; match pattern0_0 { &OperandSize::Size8 => { - // Rule at src/isa/x64/inst.isle line 98. + // Rule at src/isa/x64/inst.isle line 509. let expr0_0: u16 = 8; return Some(expr0_0); } &OperandSize::Size16 => { - // Rule at src/isa/x64/inst.isle line 99. + // Rule at src/isa/x64/inst.isle line 510. let expr0_0: u16 = 16; return Some(expr0_0); } &OperandSize::Size32 => { - // Rule at src/isa/x64/inst.isle line 100. + // Rule at src/isa/x64/inst.isle line 511. let expr0_0: u16 = 32; return Some(expr0_0); } &OperandSize::Size64 => { - // Rule at src/isa/x64/inst.isle line 101. + // Rule at src/isa/x64/inst.isle line 512. let expr0_0: u16 = 64; return Some(expr0_0); } @@ -294,6 +625,89 @@ pub fn constructor_operand_size_bits(ctx: &mut C, arg0: &OperandSize return None; } +// Generated as internal constructor for term put_in_gpr. +pub fn constructor_put_in_gpr(ctx: &mut C, arg0: Value) -> Option { + let pattern0_0 = arg0; + // Rule at src/isa/x64/inst.isle line 934. + let expr0_0 = C::put_in_reg(ctx, pattern0_0); + let expr1_0 = C::gpr_new(ctx, expr0_0); + return Some(expr1_0); +} + +// Generated as internal constructor for term put_in_gpr_mem. +pub fn constructor_put_in_gpr_mem(ctx: &mut C, arg0: Value) -> Option { + let pattern0_0 = arg0; + // Rule at src/isa/x64/inst.isle line 941. + let expr0_0 = C::put_in_reg_mem(ctx, pattern0_0); + let expr1_0 = C::gpr_mem_new(ctx, &expr0_0); + return Some(expr1_0); +} + +// Generated as internal constructor for term put_in_gpr_mem_imm. +pub fn constructor_put_in_gpr_mem_imm(ctx: &mut C, arg0: Value) -> Option { + let pattern0_0 = arg0; + // Rule at src/isa/x64/inst.isle line 948. + let expr0_0 = C::put_in_reg_mem_imm(ctx, pattern0_0); + let expr1_0 = C::gpr_mem_imm_new(ctx, &expr0_0); + return Some(expr1_0); +} + +// Generated as internal constructor for term put_in_xmm. +pub fn constructor_put_in_xmm(ctx: &mut C, arg0: Value) -> Option { + let pattern0_0 = arg0; + // Rule at src/isa/x64/inst.isle line 955. + let expr0_0 = C::put_in_reg(ctx, pattern0_0); + let expr1_0 = C::xmm_new(ctx, expr0_0); + return Some(expr1_0); +} + +// Generated as internal constructor for term put_in_xmm_mem. +pub fn constructor_put_in_xmm_mem(ctx: &mut C, arg0: Value) -> Option { + let pattern0_0 = arg0; + // Rule at src/isa/x64/inst.isle line 962. + let expr0_0 = C::put_in_reg_mem(ctx, pattern0_0); + let expr1_0 = C::xmm_mem_new(ctx, &expr0_0); + return Some(expr1_0); +} + +// Generated as internal constructor for term put_in_xmm_mem_imm. +pub fn constructor_put_in_xmm_mem_imm(ctx: &mut C, arg0: Value) -> Option { + let pattern0_0 = arg0; + // Rule at src/isa/x64/inst.isle line 969. + let expr0_0 = C::put_in_reg_mem_imm(ctx, pattern0_0); + let expr1_0 = C::xmm_mem_imm_new(ctx, &expr0_0); + return Some(expr1_0); +} + +// Generated as internal constructor for term value_gpr. +pub fn constructor_value_gpr(ctx: &mut C, arg0: Gpr) -> Option { + let pattern0_0 = arg0; + // Rule at src/isa/x64/inst.isle line 974. + let expr0_0 = C::gpr_to_reg(ctx, pattern0_0); + let expr1_0 = C::value_reg(ctx, expr0_0); + return Some(expr1_0); +} + +// Generated as internal constructor for term value_gprs. +pub fn constructor_value_gprs(ctx: &mut C, arg0: Gpr, arg1: Gpr) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 979. + let expr0_0 = C::gpr_to_reg(ctx, pattern0_0); + let expr1_0 = C::gpr_to_reg(ctx, pattern1_0); + let expr2_0 = C::value_regs(ctx, expr0_0, expr1_0); + return Some(expr2_0); +} + +// Generated as internal constructor for term value_xmm. +pub fn constructor_value_xmm(ctx: &mut C, arg0: Xmm) -> Option { + let pattern0_0 = arg0; + // Rule at src/isa/x64/inst.isle line 984. + let expr0_0 = C::xmm_to_reg(ctx, pattern0_0); + let expr1_0 = C::value_reg(ctx, expr0_0); + return Some(expr1_0); +} + // Generated as internal constructor for term extend_to_reg. pub fn constructor_extend_to_reg( ctx: &mut C, @@ -306,12 +720,12 @@ pub fn constructor_extend_to_reg( let pattern2_0 = arg1; if pattern2_0 == pattern1_0 { let pattern4_0 = arg2; - // Rule at src/isa/x64/inst.isle line 476. + // Rule at src/isa/x64/inst.isle line 1063. let expr0_0 = C::put_in_reg(ctx, pattern0_0); return Some(expr0_0); } let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 479. + // Rule at src/isa/x64/inst.isle line 1066. let expr0_0 = C::ty_bits_u16(ctx, pattern1_0); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern2_0); let expr2_0 = constructor_operand_size_bits(ctx, &expr1_0)?; @@ -335,7 +749,7 @@ pub fn constructor_extend( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/inst.isle line 499. + // Rule at src/isa/x64/inst.isle line 1086. let expr0_0 = constructor_movsx(ctx, pattern2_0, pattern3_0, pattern4_0)?; return Some(expr0_0); } @@ -343,7 +757,7 @@ pub fn constructor_extend( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/inst.isle line 495. + // Rule at src/isa/x64/inst.isle line 1082. let expr0_0 = constructor_movzx(ctx, pattern2_0, pattern3_0, pattern4_0)?; return Some(expr0_0); } @@ -356,17 +770,17 @@ pub fn constructor_extend( pub fn constructor_sse_xor_op(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32X4 { - // Rule at src/isa/x64/inst.isle line 506. + // Rule at src/isa/x64/inst.isle line 1093. let expr0_0 = SseOpcode::Xorps; return Some(expr0_0); } if pattern0_0 == F64X2 { - // Rule at src/isa/x64/inst.isle line 507. + // Rule at src/isa/x64/inst.isle line 1094. let expr0_0 = SseOpcode::Xorpd; return Some(expr0_0); } if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { - // Rule at src/isa/x64/inst.isle line 508. + // Rule at src/isa/x64/inst.isle line 1095. let expr0_0 = SseOpcode::Pxor; return Some(expr0_0); } @@ -377,13 +791,13 @@ pub fn constructor_sse_xor_op(ctx: &mut C, arg0: Type) -> Option( ctx: &mut C, arg0: Type, - arg1: Reg, - arg2: &RegMem, -) -> Option { + arg1: Xmm, + arg2: &XmmMem, +) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 512. + // Rule at src/isa/x64/inst.isle line 1099. let expr0_0 = constructor_sse_xor_op(ctx, pattern0_0)?; let expr1_0 = constructor_xmm_rm_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -393,40 +807,40 @@ pub fn constructor_sse_xor( pub fn constructor_sse_cmp_op(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32X4 { - // Rule at src/isa/x64/inst.isle line 521. + // Rule at src/isa/x64/inst.isle line 1108. let expr0_0 = SseOpcode::Cmpps; return Some(expr0_0); } if pattern0_0 == F64X2 { - // Rule at src/isa/x64/inst.isle line 522. + // Rule at src/isa/x64/inst.isle line 1109. let expr0_0 = SseOpcode::Cmppd; return Some(expr0_0); } if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { if pattern1_0 == 8 { if pattern1_1 == 16 { - // Rule at src/isa/x64/inst.isle line 517. + // Rule at src/isa/x64/inst.isle line 1104. let expr0_0 = SseOpcode::Pcmpeqb; return Some(expr0_0); } } if pattern1_0 == 16 { if pattern1_1 == 8 { - // Rule at src/isa/x64/inst.isle line 518. + // Rule at src/isa/x64/inst.isle line 1105. let expr0_0 = SseOpcode::Pcmpeqw; return Some(expr0_0); } } if pattern1_0 == 32 { if pattern1_1 == 4 { - // Rule at src/isa/x64/inst.isle line 519. + // Rule at src/isa/x64/inst.isle line 1106. let expr0_0 = SseOpcode::Pcmpeqd; return Some(expr0_0); } } if pattern1_0 == 64 { if pattern1_1 == 2 { - // Rule at src/isa/x64/inst.isle line 520. + // Rule at src/isa/x64/inst.isle line 1107. let expr0_0 = SseOpcode::Pcmpeqq; return Some(expr0_0); } @@ -436,14 +850,14 @@ pub fn constructor_sse_cmp_op(ctx: &mut C, arg0: Type) -> Option(ctx: &mut C, arg0: Type) -> Option { +pub fn constructor_vector_all_ones(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 536. - let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); - let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); + // Rule at src/isa/x64/inst.isle line 1123. + let expr0_0 = C::temp_writable_xmm(ctx); + let expr1_0 = C::writable_xmm_to_xmm(ctx, expr0_0); let expr2_0: Type = I32X4; let expr3_0 = constructor_sse_cmp_op(ctx, expr2_0)?; - let expr4_0 = RegMem::Reg { reg: expr1_0 }; + let expr4_0 = C::xmm_to_xmm_mem(ctx, expr1_0); let expr5_0 = MInst::XmmRmR { op: expr3_0, src1: expr1_0, @@ -457,70 +871,71 @@ pub fn constructor_vector_all_ones(ctx: &mut C, arg0: Type) -> Optio // Generated as internal constructor for term make_i64x2_from_lanes. pub fn constructor_make_i64x2_from_lanes( ctx: &mut C, - arg0: &RegMem, - arg1: &RegMem, -) -> Option { + arg0: &GprMem, + arg1: &GprMem, +) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 547. - let expr0_0: Type = I64X2; - let expr1_0 = C::temp_writable_reg(ctx, expr0_0); - let expr2_0 = C::writable_reg_to_reg(ctx, expr1_0); - let expr3_0 = MInst::XmmUninitializedValue { dst: expr1_0 }; - let expr4_0 = C::emit(ctx, &expr3_0); - let expr5_0 = SseOpcode::Pinsrd; - let expr6_0: u8 = 0; - let expr7_0 = OperandSize::Size64; - let expr8_0 = MInst::XmmRmRImm { - op: expr5_0, - src1: expr2_0, - src2: pattern0_0.clone(), + // Rule at src/isa/x64/inst.isle line 1134. + let expr0_0 = C::temp_writable_xmm(ctx); + let expr1_0 = C::writable_xmm_to_reg(ctx, expr0_0); + let expr2_0 = C::writable_xmm_to_xmm(ctx, expr0_0); + let expr3_0 = C::xmm_to_reg(ctx, expr2_0); + let expr4_0 = MInst::XmmUninitializedValue { dst: expr0_0 }; + let expr5_0 = C::emit(ctx, &expr4_0); + let expr6_0 = SseOpcode::Pinsrd; + let expr7_0 = C::gpr_mem_to_reg_mem(ctx, pattern0_0); + let expr8_0: u8 = 0; + let expr9_0 = OperandSize::Size64; + let expr10_0 = MInst::XmmRmRImm { + op: expr6_0, + src1: expr3_0, + src2: expr7_0, dst: expr1_0, - imm: expr6_0, - size: expr7_0, + imm: expr8_0, + size: expr9_0, }; - let expr9_0 = C::emit(ctx, &expr8_0); - let expr10_0 = SseOpcode::Pinsrd; - let expr11_0: u8 = 1; - let expr12_0 = OperandSize::Size64; - let expr13_0 = MInst::XmmRmRImm { - op: expr10_0, - src1: expr2_0, - src2: pattern1_0.clone(), + let expr11_0 = C::emit(ctx, &expr10_0); + let expr12_0 = SseOpcode::Pinsrd; + let expr13_0 = C::gpr_mem_to_reg_mem(ctx, pattern1_0); + let expr14_0: u8 = 1; + let expr15_0 = OperandSize::Size64; + let expr16_0 = MInst::XmmRmRImm { + op: expr12_0, + src1: expr3_0, + src2: expr13_0, dst: expr1_0, - imm: expr11_0, - size: expr12_0, + imm: expr14_0, + size: expr15_0, }; - let expr14_0 = C::emit(ctx, &expr13_0); + let expr17_0 = C::emit(ctx, &expr16_0); return Some(expr2_0); } -// Generated as internal constructor for term reg_mem_imm_to_xmm. -pub fn constructor_reg_mem_imm_to_xmm( - ctx: &mut C, - arg0: &RegMemImm, -) -> Option { +// Generated as internal constructor for term mov_rmi_to_xmm. +pub fn constructor_mov_rmi_to_xmm(ctx: &mut C, arg0: &RegMemImm) -> Option { let pattern0_0 = arg0; match pattern0_0 { &RegMemImm::Imm { simm32: pattern1_0 } => { - // Rule at src/isa/x64/inst.isle line 568. - return Some(pattern0_0.clone()); + // Rule at src/isa/x64/inst.isle line 1157. + let expr0_0 = C::xmm_mem_imm_new(ctx, pattern0_0); + return Some(expr0_0); } &RegMemImm::Reg { reg: pattern1_0 } => { - // Rule at src/isa/x64/inst.isle line 569. - let expr0_0: Type = I8X16; - let expr1_0 = SseOpcode::Movd; - let expr2_0 = RegMem::Reg { reg: pattern1_0 }; - let expr3_0 = OperandSize::Size32; - let expr4_0 = constructor_gpr_to_xmm(ctx, expr0_0, &expr1_0, &expr2_0, &expr3_0)?; - let expr5_0 = RegMemImm::Reg { reg: expr4_0 }; - return Some(expr5_0); + // Rule at src/isa/x64/inst.isle line 1158. + let expr0_0 = SseOpcode::Movd; + let expr1_0 = C::reg_to_gpr_mem(ctx, pattern1_0); + let expr2_0 = OperandSize::Size32; + let expr3_0 = constructor_gpr_to_xmm(ctx, &expr0_0, &expr1_0, &expr2_0)?; + let expr4_0 = C::xmm_to_xmm_mem_imm(ctx, expr3_0); + return Some(expr4_0); } &RegMemImm::Mem { addr: ref pattern1_0, } => { - // Rule at src/isa/x64/inst.isle line 567. - return Some(pattern0_0.clone()); + // Rule at src/isa/x64/inst.isle line 1156. + let expr0_0 = C::xmm_mem_imm_new(ctx, pattern0_0); + return Some(expr0_0); } _ => {} } @@ -538,7 +953,7 @@ pub fn constructor_x64_load( if pattern0_0 == I64 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 585. + // Rule at src/isa/x64/inst.isle line 1173. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::Mov64MR { @@ -552,53 +967,63 @@ pub fn constructor_x64_load( if pattern0_0 == F32 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 590. + // Rule at src/isa/x64/inst.isle line 1178. let expr0_0 = SseOpcode::Movss; let expr1_0 = C::synthetic_amode_to_reg_mem(ctx, pattern2_0); - let expr2_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, &expr1_0)?; - return Some(expr2_0); + let expr2_0 = C::xmm_mem_new(ctx, &expr1_0); + let expr3_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, &expr2_0)?; + let expr4_0 = C::xmm_to_reg(ctx, expr3_0); + return Some(expr4_0); } if pattern0_0 == F64 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 594. + // Rule at src/isa/x64/inst.isle line 1182. let expr0_0 = SseOpcode::Movsd; let expr1_0 = C::synthetic_amode_to_reg_mem(ctx, pattern2_0); - let expr2_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, &expr1_0)?; - return Some(expr2_0); + let expr2_0 = C::xmm_mem_new(ctx, &expr1_0); + let expr3_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, &expr2_0)?; + let expr4_0 = C::xmm_to_reg(ctx, expr3_0); + return Some(expr4_0); } if pattern0_0 == F32X4 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 598. + // Rule at src/isa/x64/inst.isle line 1186. let expr0_0 = SseOpcode::Movups; let expr1_0 = C::synthetic_amode_to_reg_mem(ctx, pattern2_0); - let expr2_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, &expr1_0)?; - return Some(expr2_0); + let expr2_0 = C::xmm_mem_new(ctx, &expr1_0); + let expr3_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, &expr2_0)?; + let expr4_0 = C::xmm_to_reg(ctx, expr3_0); + return Some(expr4_0); } if pattern0_0 == F64X2 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 602. + // Rule at src/isa/x64/inst.isle line 1190. let expr0_0 = SseOpcode::Movupd; let expr1_0 = C::synthetic_amode_to_reg_mem(ctx, pattern2_0); - let expr2_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, &expr1_0)?; - return Some(expr2_0); + let expr2_0 = C::xmm_mem_new(ctx, &expr1_0); + let expr3_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, &expr2_0)?; + let expr4_0 = C::xmm_to_reg(ctx, expr3_0); + return Some(expr4_0); } if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 606. + // Rule at src/isa/x64/inst.isle line 1194. let expr0_0 = SseOpcode::Movdqu; let expr1_0 = C::synthetic_amode_to_reg_mem(ctx, pattern2_0); - let expr2_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, &expr1_0)?; - return Some(expr2_0); + let expr2_0 = C::xmm_mem_new(ctx, &expr1_0); + let expr3_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, &expr2_0)?; + let expr4_0 = C::xmm_to_reg(ctx, expr3_0); + return Some(expr4_0); } if let Some(pattern1_0) = C::fits_in_32(ctx, pattern0_0) { let pattern2_0 = arg1; let pattern3_0 = arg2; if let &ExtKind::SignExtend = pattern3_0 { - // Rule at src/isa/x64/inst.isle line 580. + // Rule at src/isa/x64/inst.isle line 1168. let expr0_0 = C::ty_bytes(ctx, pattern1_0); let expr1_0: u16 = 8; let expr2_0 = C::ext_mode(ctx, expr0_0, expr1_0); @@ -622,7 +1047,7 @@ pub fn constructor_alu_rmi_r( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 618. + // Rule at src/isa/x64/inst.isle line 1206. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = MInst::AluRmiR { @@ -647,7 +1072,7 @@ pub fn constructor_add( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 626. + // Rule at src/isa/x64/inst.isle line 1214. let expr0_0 = AluRmiROpcode::Add; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -663,7 +1088,7 @@ pub fn constructor_add_with_flags( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 634. + // Rule at src/isa/x64/inst.isle line 1222. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = AluRmiROpcode::Add; @@ -692,7 +1117,7 @@ pub fn constructor_adc( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 645. + // Rule at src/isa/x64/inst.isle line 1233. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = AluRmiROpcode::Adc; @@ -721,7 +1146,7 @@ pub fn constructor_sub( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 656. + // Rule at src/isa/x64/inst.isle line 1244. let expr0_0 = AluRmiROpcode::Sub; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -737,7 +1162,7 @@ pub fn constructor_sub_with_flags( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 664. + // Rule at src/isa/x64/inst.isle line 1252. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = AluRmiROpcode::Sub; @@ -766,7 +1191,7 @@ pub fn constructor_sbb( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 675. + // Rule at src/isa/x64/inst.isle line 1263. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = AluRmiROpcode::Sbb; @@ -795,7 +1220,7 @@ pub fn constructor_mul( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 686. + // Rule at src/isa/x64/inst.isle line 1274. let expr0_0 = AluRmiROpcode::Mul; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -811,7 +1236,7 @@ pub fn constructor_x64_and( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 697. + // Rule at src/isa/x64/inst.isle line 1285. let expr0_0 = AluRmiROpcode::And; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -827,7 +1252,7 @@ pub fn constructor_or( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 705. + // Rule at src/isa/x64/inst.isle line 1293. let expr0_0 = AluRmiROpcode::Or; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -843,7 +1268,7 @@ pub fn constructor_xor( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 713. + // Rule at src/isa/x64/inst.isle line 1301. let expr0_0 = AluRmiROpcode::Xor; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -855,7 +1280,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option if pattern0_0 == I64 { let pattern2_0 = arg1; if let Some(pattern3_0) = C::nonzero_u64_fits_in_u32(ctx, pattern2_0) { - // Rule at src/isa/x64/inst.isle line 742. + // Rule at src/isa/x64/inst.isle line 1334. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = OperandSize::Size32; @@ -872,67 +1297,69 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option if pattern0_0 == F32 { let pattern2_0 = arg1; if pattern2_0 == 0 { - // Rule at src/isa/x64/inst.isle line 771. - let expr0_0: Type = F32; - let expr1_0 = C::temp_writable_reg(ctx, expr0_0); - let expr2_0 = C::writable_reg_to_reg(ctx, expr1_0); - let expr3_0 = SseOpcode::Xorps; - let expr4_0 = RegMem::Reg { reg: expr2_0 }; - let expr5_0 = MInst::XmmRmR { - op: expr3_0, - src1: expr2_0, - src2: expr4_0, - dst: expr1_0, + // Rule at src/isa/x64/inst.isle line 1363. + let expr0_0 = C::temp_writable_xmm(ctx); + let expr1_0 = C::writable_xmm_to_xmm(ctx, expr0_0); + let expr2_0 = SseOpcode::Xorps; + let expr3_0 = C::xmm_to_xmm_mem(ctx, expr1_0); + let expr4_0 = MInst::XmmRmR { + op: expr2_0, + src1: expr1_0, + src2: expr3_0, + dst: expr0_0, }; - let expr6_0 = C::emit(ctx, &expr5_0); - return Some(expr2_0); + let expr5_0 = C::emit(ctx, &expr4_0); + let expr6_0 = C::xmm_to_reg(ctx, expr1_0); + return Some(expr6_0); } - // Rule at src/isa/x64/inst.isle line 730. - let expr0_0: Type = F32; - let expr1_0 = SseOpcode::Movd; - let expr2_0: Type = I32; - let expr3_0 = constructor_imm(ctx, expr2_0, pattern2_0)?; - let expr4_0 = RegMem::Reg { reg: expr3_0 }; + // Rule at src/isa/x64/inst.isle line 1318. + let expr0_0 = SseOpcode::Movd; + let expr1_0: Type = I32; + let expr2_0 = constructor_imm(ctx, expr1_0, pattern2_0)?; + let expr3_0 = RegMem::Reg { reg: expr2_0 }; + let expr4_0 = C::gpr_mem_new(ctx, &expr3_0); let expr5_0 = OperandSize::Size32; - let expr6_0 = constructor_gpr_to_xmm(ctx, expr0_0, &expr1_0, &expr4_0, &expr5_0)?; - return Some(expr6_0); + let expr6_0 = constructor_gpr_to_xmm(ctx, &expr0_0, &expr4_0, &expr5_0)?; + let expr7_0 = C::xmm_to_reg(ctx, expr6_0); + return Some(expr7_0); } if pattern0_0 == F64 { let pattern2_0 = arg1; if pattern2_0 == 0 { - // Rule at src/isa/x64/inst.isle line 783. - let expr0_0: Type = F64; - let expr1_0 = C::temp_writable_reg(ctx, expr0_0); - let expr2_0 = C::writable_reg_to_reg(ctx, expr1_0); - let expr3_0 = SseOpcode::Xorpd; - let expr4_0 = RegMem::Reg { reg: expr2_0 }; - let expr5_0 = MInst::XmmRmR { - op: expr3_0, - src1: expr2_0, - src2: expr4_0, - dst: expr1_0, + // Rule at src/isa/x64/inst.isle line 1375. + let expr0_0 = C::temp_writable_xmm(ctx); + let expr1_0 = C::writable_xmm_to_xmm(ctx, expr0_0); + let expr2_0 = SseOpcode::Xorpd; + let expr3_0 = C::xmm_to_xmm_mem(ctx, expr1_0); + let expr4_0 = MInst::XmmRmR { + op: expr2_0, + src1: expr1_0, + src2: expr3_0, + dst: expr0_0, }; - let expr6_0 = C::emit(ctx, &expr5_0); - return Some(expr2_0); + let expr5_0 = C::emit(ctx, &expr4_0); + let expr6_0 = C::xmm_to_reg(ctx, expr1_0); + return Some(expr6_0); } - // Rule at src/isa/x64/inst.isle line 734. - let expr0_0: Type = F64; - let expr1_0 = SseOpcode::Movq; - let expr2_0: Type = I64; - let expr3_0 = constructor_imm(ctx, expr2_0, pattern2_0)?; - let expr4_0 = RegMem::Reg { reg: expr3_0 }; + // Rule at src/isa/x64/inst.isle line 1324. + let expr0_0 = SseOpcode::Movq; + let expr1_0: Type = I64; + let expr2_0 = constructor_imm(ctx, expr1_0, pattern2_0)?; + let expr3_0 = RegMem::Reg { reg: expr2_0 }; + let expr4_0 = C::gpr_mem_new(ctx, &expr3_0); let expr5_0 = OperandSize::Size64; - let expr6_0 = constructor_gpr_to_xmm(ctx, expr0_0, &expr1_0, &expr4_0, &expr5_0)?; - return Some(expr6_0); + let expr6_0 = constructor_gpr_to_xmm(ctx, &expr0_0, &expr4_0, &expr5_0)?; + let expr7_0 = C::xmm_to_reg(ctx, expr6_0); + return Some(expr7_0); } if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { let pattern2_0 = arg1; if pattern2_0 == 0 { - // Rule at src/isa/x64/inst.isle line 761. - let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); - let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); + // Rule at src/isa/x64/inst.isle line 1353. + let expr0_0 = C::temp_writable_xmm(ctx); + let expr1_0 = C::writable_xmm_to_xmm(ctx, expr0_0); let expr2_0 = constructor_sse_xor_op(ctx, pattern0_0)?; - let expr3_0 = RegMem::Reg { reg: expr1_0 }; + let expr3_0 = C::xmm_to_xmm_mem(ctx, expr1_0); let expr4_0 = MInst::XmmRmR { op: expr2_0, src1: expr1_0, @@ -940,13 +1367,14 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option dst: expr0_0, }; let expr5_0 = C::emit(ctx, &expr4_0); - return Some(expr1_0); + let expr6_0 = C::xmm_to_reg(ctx, expr1_0); + return Some(expr6_0); } } if let Some(pattern1_0) = C::fits_in_64(ctx, pattern0_0) { let pattern2_0 = arg1; if pattern2_0 == 0 { - // Rule at src/isa/x64/inst.isle line 748. + // Rule at src/isa/x64/inst.isle line 1340. let expr0_0 = C::temp_writable_reg(ctx, pattern1_0); let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); let expr2_0 = C::operand_size_of_type_32_64(ctx, pattern1_0); @@ -962,7 +1390,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option let expr6_0 = C::emit(ctx, &expr5_0); return Some(expr1_0); } - // Rule at src/isa/x64/inst.isle line 723. + // Rule at src/isa/x64/inst.isle line 1311. let expr0_0 = C::temp_writable_reg(ctx, pattern1_0); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern1_0); let expr2_0 = MInst::Imm { @@ -989,7 +1417,7 @@ pub fn constructor_shift_r( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 796. + // Rule at src/isa/x64/inst.isle line 1388. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::raw_operand_size_of_type(ctx, pattern0_0); let expr2_0 = MInst::ShiftR { @@ -1014,7 +1442,7 @@ pub fn constructor_x64_rotl( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 807. + // Rule at src/isa/x64/inst.isle line 1399. let expr0_0 = ShiftKind::RotateLeft; let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -1030,7 +1458,7 @@ pub fn constructor_x64_rotr( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 813. + // Rule at src/isa/x64/inst.isle line 1405. let expr0_0 = ShiftKind::RotateRight; let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -1046,7 +1474,7 @@ pub fn constructor_shl( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 818. + // Rule at src/isa/x64/inst.isle line 1410. let expr0_0 = ShiftKind::ShiftLeft; let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -1062,7 +1490,7 @@ pub fn constructor_shr( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 823. + // Rule at src/isa/x64/inst.isle line 1415. let expr0_0 = ShiftKind::ShiftRightLogical; let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -1078,7 +1506,7 @@ pub fn constructor_sar( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 828. + // Rule at src/isa/x64/inst.isle line 1420. let expr0_0 = ShiftKind::ShiftRightArithmetic; let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -1096,7 +1524,7 @@ pub fn constructor_cmp_rmi_r( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 833. + // Rule at src/isa/x64/inst.isle line 1425. let expr0_0 = MInst::CmpRmiR { size: pattern0_0.clone(), opcode: pattern1_0.clone(), @@ -1121,7 +1549,7 @@ pub fn constructor_cmp( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 842. + // Rule at src/isa/x64/inst.isle line 1434. let expr0_0 = CmpOpcode::Cmp; let expr1_0 = constructor_cmp_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -1137,7 +1565,7 @@ pub fn constructor_test( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 847. + // Rule at src/isa/x64/inst.isle line 1439. let expr0_0 = CmpOpcode::Test; let expr1_0 = constructor_cmp_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -1155,7 +1583,7 @@ pub fn constructor_cmove( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 852. + // Rule at src/isa/x64/inst.isle line 1444. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = MInst::Cmove { @@ -1183,7 +1611,7 @@ pub fn constructor_movzx( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 860. + // Rule at src/isa/x64/inst.isle line 1452. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::MovzxRmR { ext_mode: pattern1_0.clone(), @@ -1205,7 +1633,7 @@ pub fn constructor_movsx( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 867. + // Rule at src/isa/x64/inst.isle line 1459. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::MovsxRmR { ext_mode: pattern1_0.clone(), @@ -1222,15 +1650,15 @@ pub fn constructor_xmm_rm_r( ctx: &mut C, arg0: Type, arg1: &SseOpcode, - arg2: Reg, - arg3: &RegMem, -) -> Option { + arg2: Xmm, + arg3: &XmmMem, +) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 874. - let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); + // Rule at src/isa/x64/inst.isle line 1466. + let expr0_0 = C::temp_writable_xmm(ctx); let expr1_0 = MInst::XmmRmR { op: pattern1_0.clone(), src1: pattern2_0, @@ -1238,15 +1666,15 @@ pub fn constructor_xmm_rm_r( dst: expr0_0, }; let expr2_0 = C::emit(ctx, &expr1_0); - let expr3_0 = C::writable_reg_to_reg(ctx, expr0_0); + let expr3_0 = C::writable_xmm_to_xmm(ctx, expr0_0); return Some(expr3_0); } // Generated as internal constructor for term paddb. -pub fn constructor_paddb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_paddb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 881. + // Rule at src/isa/x64/inst.isle line 1473. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Paddb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1254,10 +1682,10 @@ pub fn constructor_paddb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O } // Generated as internal constructor for term paddw. -pub fn constructor_paddw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_paddw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 886. + // Rule at src/isa/x64/inst.isle line 1478. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Paddw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1265,10 +1693,10 @@ pub fn constructor_paddw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O } // Generated as internal constructor for term paddd. -pub fn constructor_paddd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_paddd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 891. + // Rule at src/isa/x64/inst.isle line 1483. let expr0_0: Type = I32X4; let expr1_0 = SseOpcode::Paddd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1276,10 +1704,10 @@ pub fn constructor_paddd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O } // Generated as internal constructor for term paddq. -pub fn constructor_paddq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_paddq(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 896. + // Rule at src/isa/x64/inst.isle line 1488. let expr0_0: Type = I64X2; let expr1_0 = SseOpcode::Paddq; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1287,10 +1715,10 @@ pub fn constructor_paddq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O } // Generated as internal constructor for term paddsb. -pub fn constructor_paddsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_paddsb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 901. + // Rule at src/isa/x64/inst.isle line 1493. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Paddsb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1298,10 +1726,10 @@ pub fn constructor_paddsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term paddsw. -pub fn constructor_paddsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_paddsw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 906. + // Rule at src/isa/x64/inst.isle line 1498. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Paddsw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1309,10 +1737,10 @@ pub fn constructor_paddsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term paddusb. -pub fn constructor_paddusb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_paddusb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 911. + // Rule at src/isa/x64/inst.isle line 1503. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Paddusb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1320,10 +1748,10 @@ pub fn constructor_paddusb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term paddusw. -pub fn constructor_paddusw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_paddusw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 916. + // Rule at src/isa/x64/inst.isle line 1508. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Paddusw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1331,10 +1759,10 @@ pub fn constructor_paddusw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term psubb. -pub fn constructor_psubb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_psubb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 921. + // Rule at src/isa/x64/inst.isle line 1513. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Psubb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1342,10 +1770,10 @@ pub fn constructor_psubb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O } // Generated as internal constructor for term psubw. -pub fn constructor_psubw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_psubw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 926. + // Rule at src/isa/x64/inst.isle line 1518. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Psubw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1353,10 +1781,10 @@ pub fn constructor_psubw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O } // Generated as internal constructor for term psubd. -pub fn constructor_psubd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_psubd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 931. + // Rule at src/isa/x64/inst.isle line 1523. let expr0_0: Type = I32X4; let expr1_0 = SseOpcode::Psubd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1364,10 +1792,10 @@ pub fn constructor_psubd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O } // Generated as internal constructor for term psubq. -pub fn constructor_psubq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_psubq(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 936. + // Rule at src/isa/x64/inst.isle line 1528. let expr0_0: Type = I64X2; let expr1_0 = SseOpcode::Psubq; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1375,10 +1803,10 @@ pub fn constructor_psubq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O } // Generated as internal constructor for term psubsb. -pub fn constructor_psubsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_psubsb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 941. + // Rule at src/isa/x64/inst.isle line 1533. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Psubsb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1386,10 +1814,10 @@ pub fn constructor_psubsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term psubsw. -pub fn constructor_psubsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_psubsw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 946. + // Rule at src/isa/x64/inst.isle line 1538. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Psubsw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1397,10 +1825,10 @@ pub fn constructor_psubsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term psubusb. -pub fn constructor_psubusb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_psubusb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 951. + // Rule at src/isa/x64/inst.isle line 1543. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Psubusb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1408,10 +1836,10 @@ pub fn constructor_psubusb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term psubusw. -pub fn constructor_psubusw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_psubusw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 956. + // Rule at src/isa/x64/inst.isle line 1548. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Psubusw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1419,10 +1847,10 @@ pub fn constructor_psubusw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term pavgb. -pub fn constructor_pavgb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pavgb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 961. + // Rule at src/isa/x64/inst.isle line 1553. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pavgb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1430,10 +1858,10 @@ pub fn constructor_pavgb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O } // Generated as internal constructor for term pavgw. -pub fn constructor_pavgw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pavgw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 966. + // Rule at src/isa/x64/inst.isle line 1558. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pavgw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1441,10 +1869,10 @@ pub fn constructor_pavgw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O } // Generated as internal constructor for term pand. -pub fn constructor_pand(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pand(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 971. + // Rule at src/isa/x64/inst.isle line 1563. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Pand; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1452,10 +1880,10 @@ pub fn constructor_pand(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Op } // Generated as internal constructor for term andps. -pub fn constructor_andps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_andps(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 976. + // Rule at src/isa/x64/inst.isle line 1568. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Andps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1463,10 +1891,10 @@ pub fn constructor_andps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O } // Generated as internal constructor for term andpd. -pub fn constructor_andpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_andpd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 981. + // Rule at src/isa/x64/inst.isle line 1573. let expr0_0: Type = F64X2; let expr1_0 = SseOpcode::Andpd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1474,10 +1902,10 @@ pub fn constructor_andpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O } // Generated as internal constructor for term por. -pub fn constructor_por(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_por(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 986. + // Rule at src/isa/x64/inst.isle line 1578. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Por; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1485,10 +1913,10 @@ pub fn constructor_por(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Opt } // Generated as internal constructor for term orps. -pub fn constructor_orps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_orps(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 991. + // Rule at src/isa/x64/inst.isle line 1583. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Orps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1496,10 +1924,10 @@ pub fn constructor_orps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Op } // Generated as internal constructor for term orpd. -pub fn constructor_orpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_orpd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 996. + // Rule at src/isa/x64/inst.isle line 1588. let expr0_0: Type = F64X2; let expr1_0 = SseOpcode::Orpd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1507,10 +1935,10 @@ pub fn constructor_orpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Op } // Generated as internal constructor for term pxor. -pub fn constructor_pxor(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pxor(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1001. + // Rule at src/isa/x64/inst.isle line 1593. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pxor; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1518,10 +1946,10 @@ pub fn constructor_pxor(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Op } // Generated as internal constructor for term xorps. -pub fn constructor_xorps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_xorps(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1006. + // Rule at src/isa/x64/inst.isle line 1598. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Xorps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1529,10 +1957,10 @@ pub fn constructor_xorps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O } // Generated as internal constructor for term xorpd. -pub fn constructor_xorpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_xorpd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1011. + // Rule at src/isa/x64/inst.isle line 1603. let expr0_0: Type = F64X2; let expr1_0 = SseOpcode::Xorpd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1540,10 +1968,10 @@ pub fn constructor_xorpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O } // Generated as internal constructor for term pmullw. -pub fn constructor_pmullw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pmullw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1016. + // Rule at src/isa/x64/inst.isle line 1608. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pmullw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1551,10 +1979,10 @@ pub fn constructor_pmullw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term pmulld. -pub fn constructor_pmulld(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pmulld(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1021. + // Rule at src/isa/x64/inst.isle line 1613. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pmulld; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1562,10 +1990,10 @@ pub fn constructor_pmulld(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term pmulhw. -pub fn constructor_pmulhw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pmulhw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1026. + // Rule at src/isa/x64/inst.isle line 1618. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pmulhw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1573,10 +2001,10 @@ pub fn constructor_pmulhw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term pmulhuw. -pub fn constructor_pmulhuw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pmulhuw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1031. + // Rule at src/isa/x64/inst.isle line 1623. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pmulhuw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1584,10 +2012,10 @@ pub fn constructor_pmulhuw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term pmuldq. -pub fn constructor_pmuldq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pmuldq(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1036. + // Rule at src/isa/x64/inst.isle line 1628. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pmuldq; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1595,10 +2023,10 @@ pub fn constructor_pmuldq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term pmuludq. -pub fn constructor_pmuludq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pmuludq(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1041. + // Rule at src/isa/x64/inst.isle line 1633. let expr0_0: Type = I64X2; let expr1_0 = SseOpcode::Pmuludq; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1606,10 +2034,10 @@ pub fn constructor_pmuludq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term punpckhwd. -pub fn constructor_punpckhwd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_punpckhwd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1046. + // Rule at src/isa/x64/inst.isle line 1638. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Punpckhwd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1617,10 +2045,10 @@ pub fn constructor_punpckhwd(ctx: &mut C, arg0: Reg, arg1: &RegMem) } // Generated as internal constructor for term punpcklwd. -pub fn constructor_punpcklwd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_punpcklwd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1051. + // Rule at src/isa/x64/inst.isle line 1643. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Punpcklwd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1628,10 +2056,10 @@ pub fn constructor_punpcklwd(ctx: &mut C, arg0: Reg, arg1: &RegMem) } // Generated as internal constructor for term andnps. -pub fn constructor_andnps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_andnps(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1056. + // Rule at src/isa/x64/inst.isle line 1648. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Andnps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1639,10 +2067,10 @@ pub fn constructor_andnps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term andnpd. -pub fn constructor_andnpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_andnpd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1061. + // Rule at src/isa/x64/inst.isle line 1653. let expr0_0: Type = F64X2; let expr1_0 = SseOpcode::Andnpd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1650,10 +2078,10 @@ pub fn constructor_andnpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term pandn. -pub fn constructor_pandn(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pandn(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1066. + // Rule at src/isa/x64/inst.isle line 1658. let expr0_0: Type = F64X2; let expr1_0 = SseOpcode::Pandn; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1664,17 +2092,17 @@ pub fn constructor_pandn(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_sse_blend_op(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32X4 { - // Rule at src/isa/x64/inst.isle line 1070. + // Rule at src/isa/x64/inst.isle line 1662. let expr0_0 = SseOpcode::Blendvps; return Some(expr0_0); } if pattern0_0 == F64X2 { - // Rule at src/isa/x64/inst.isle line 1071. + // Rule at src/isa/x64/inst.isle line 1663. let expr0_0 = SseOpcode::Blendvpd; return Some(expr0_0); } if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { - // Rule at src/isa/x64/inst.isle line 1072. + // Rule at src/isa/x64/inst.isle line 1664. let expr0_0 = SseOpcode::Pblendvb; return Some(expr0_0); } @@ -1685,17 +2113,17 @@ pub fn constructor_sse_blend_op(ctx: &mut C, arg0: Type) -> Option(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32X4 { - // Rule at src/isa/x64/inst.isle line 1075. + // Rule at src/isa/x64/inst.isle line 1667. let expr0_0 = SseOpcode::Movaps; return Some(expr0_0); } if pattern0_0 == F64X2 { - // Rule at src/isa/x64/inst.isle line 1076. + // Rule at src/isa/x64/inst.isle line 1668. let expr0_0 = SseOpcode::Movapd; return Some(expr0_0); } if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { - // Rule at src/isa/x64/inst.isle line 1077. + // Rule at src/isa/x64/inst.isle line 1669. let expr0_0 = SseOpcode::Movdqa; return Some(expr0_0); } @@ -1706,15 +2134,15 @@ pub fn constructor_sse_mov_op(ctx: &mut C, arg0: Type) -> Option( ctx: &mut C, arg0: Type, - arg1: &RegMem, - arg2: &RegMem, - arg3: Reg, -) -> Option { + arg1: &XmmMem, + arg2: &XmmMem, + arg3: Xmm, +) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1081. + // Rule at src/isa/x64/inst.isle line 1673. let expr0_0 = C::xmm0(ctx); let expr1_0 = constructor_sse_mov_op(ctx, pattern0_0)?; let expr2_0 = MInst::XmmUnaryRmR { @@ -1731,17 +2159,17 @@ pub fn constructor_sse_blend( // Generated as internal constructor for term blendvpd. pub fn constructor_blendvpd( ctx: &mut C, - arg0: Reg, - arg1: &RegMem, - arg2: Reg, -) -> Option { + arg0: Xmm, + arg1: &XmmMem, + arg2: Xmm, +) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1093. + // Rule at src/isa/x64/inst.isle line 1687. let expr0_0 = C::xmm0(ctx); let expr1_0 = SseOpcode::Movapd; - let expr2_0 = RegMem::Reg { reg: pattern2_0 }; + let expr2_0 = C::xmm_to_xmm_mem(ctx, pattern2_0); let expr3_0 = MInst::XmmUnaryRmR { op: expr1_0, src: expr2_0, @@ -1755,10 +2183,10 @@ pub fn constructor_blendvpd( } // Generated as internal constructor for term movsd. -pub fn constructor_movsd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_movsd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1105. + // Rule at src/isa/x64/inst.isle line 1701. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Movsd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1766,10 +2194,10 @@ pub fn constructor_movsd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O } // Generated as internal constructor for term movlhps. -pub fn constructor_movlhps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_movlhps(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1110. + // Rule at src/isa/x64/inst.isle line 1706. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Movlhps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1777,10 +2205,10 @@ pub fn constructor_movlhps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term pmaxsb. -pub fn constructor_pmaxsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pmaxsb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1115. + // Rule at src/isa/x64/inst.isle line 1711. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxsb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1788,10 +2216,10 @@ pub fn constructor_pmaxsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term pmaxsw. -pub fn constructor_pmaxsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pmaxsw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1120. + // Rule at src/isa/x64/inst.isle line 1716. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxsw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1799,10 +2227,10 @@ pub fn constructor_pmaxsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term pmaxsd. -pub fn constructor_pmaxsd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pmaxsd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1125. + // Rule at src/isa/x64/inst.isle line 1721. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxsd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1810,10 +2238,10 @@ pub fn constructor_pmaxsd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term pminsb. -pub fn constructor_pminsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pminsb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1130. + // Rule at src/isa/x64/inst.isle line 1726. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pminsb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1821,10 +2249,10 @@ pub fn constructor_pminsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term pminsw. -pub fn constructor_pminsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pminsw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1135. + // Rule at src/isa/x64/inst.isle line 1731. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pminsw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1832,10 +2260,10 @@ pub fn constructor_pminsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term pminsd. -pub fn constructor_pminsd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pminsd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1140. + // Rule at src/isa/x64/inst.isle line 1736. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pminsd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1843,10 +2271,10 @@ pub fn constructor_pminsd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term pmaxub. -pub fn constructor_pmaxub(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pmaxub(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1145. + // Rule at src/isa/x64/inst.isle line 1741. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxub; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1854,10 +2282,10 @@ pub fn constructor_pmaxub(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term pmaxuw. -pub fn constructor_pmaxuw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pmaxuw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1150. + // Rule at src/isa/x64/inst.isle line 1746. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxuw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1865,10 +2293,10 @@ pub fn constructor_pmaxuw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term pmaxud. -pub fn constructor_pmaxud(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pmaxud(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1155. + // Rule at src/isa/x64/inst.isle line 1751. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxud; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1876,10 +2304,10 @@ pub fn constructor_pmaxud(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term pminub. -pub fn constructor_pminub(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pminub(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1160. + // Rule at src/isa/x64/inst.isle line 1756. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pminub; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1887,10 +2315,10 @@ pub fn constructor_pminub(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term pminuw. -pub fn constructor_pminuw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pminuw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1165. + // Rule at src/isa/x64/inst.isle line 1761. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pminuw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1898,10 +2326,10 @@ pub fn constructor_pminuw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term pminud. -pub fn constructor_pminud(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pminud(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1170. + // Rule at src/isa/x64/inst.isle line 1766. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pminud; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1909,10 +2337,10 @@ pub fn constructor_pminud(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term punpcklbw. -pub fn constructor_punpcklbw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_punpcklbw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1175. + // Rule at src/isa/x64/inst.isle line 1771. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Punpcklbw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1920,10 +2348,10 @@ pub fn constructor_punpcklbw(ctx: &mut C, arg0: Reg, arg1: &RegMem) } // Generated as internal constructor for term punpckhbw. -pub fn constructor_punpckhbw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_punpckhbw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1180. + // Rule at src/isa/x64/inst.isle line 1776. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Punpckhbw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1931,10 +2359,10 @@ pub fn constructor_punpckhbw(ctx: &mut C, arg0: Reg, arg1: &RegMem) } // Generated as internal constructor for term packsswb. -pub fn constructor_packsswb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_packsswb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1185. + // Rule at src/isa/x64/inst.isle line 1781. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Packsswb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1949,15 +2377,15 @@ pub fn constructor_xmm_rm_r_imm( arg2: &RegMem, arg3: u8, arg4: &OperandSize, -) -> Option { +) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; let pattern4_0 = arg4; - // Rule at src/isa/x64/inst.isle line 1190. - let expr0_0: Type = I8X16; - let expr1_0 = C::temp_writable_reg(ctx, expr0_0); + // Rule at src/isa/x64/inst.isle line 1786. + let expr0_0 = C::temp_writable_xmm(ctx); + let expr1_0 = C::writable_xmm_to_reg(ctx, expr0_0); let expr2_0 = MInst::XmmRmRImm { op: pattern0_0.clone(), src1: pattern1_0, @@ -1967,118 +2395,218 @@ pub fn constructor_xmm_rm_r_imm( size: pattern4_0.clone(), }; let expr3_0 = C::emit(ctx, &expr2_0); - let expr4_0 = C::writable_reg_to_reg(ctx, expr1_0); + let expr4_0 = C::writable_xmm_to_xmm(ctx, expr0_0); return Some(expr4_0); } // Generated as internal constructor for term palignr. pub fn constructor_palignr( ctx: &mut C, - arg0: Reg, - arg1: &RegMem, + arg0: Xmm, + arg1: &XmmMem, arg2: u8, arg3: &OperandSize, -) -> Option { +) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1202. + // Rule at src/isa/x64/inst.isle line 1798. let expr0_0 = SseOpcode::Palignr; - let expr1_0 = constructor_xmm_rm_r_imm( - ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0, pattern3_0, - )?; - return Some(expr1_0); + let expr1_0 = C::xmm_to_reg(ctx, pattern0_0); + let expr2_0 = C::xmm_mem_to_reg_mem(ctx, pattern1_0); + let expr3_0 = + constructor_xmm_rm_r_imm(ctx, &expr0_0, expr1_0, &expr2_0, pattern2_0, pattern3_0)?; + return Some(expr3_0); +} + +// Generated as internal constructor for term cmpps. +pub fn constructor_cmpps( + ctx: &mut C, + arg0: Xmm, + arg1: &XmmMem, + arg2: &FcmpImm, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + // Rule at src/isa/x64/inst.isle line 1807. + let expr0_0 = SseOpcode::Cmpps; + let expr1_0 = C::xmm_to_reg(ctx, pattern0_0); + let expr2_0 = C::xmm_mem_to_reg_mem(ctx, pattern1_0); + let expr3_0 = C::encode_fcmp_imm(ctx, pattern2_0); + let expr4_0 = OperandSize::Size32; + let expr5_0 = constructor_xmm_rm_r_imm(ctx, &expr0_0, expr1_0, &expr2_0, expr3_0, &expr4_0)?; + return Some(expr5_0); +} + +// Generated as internal constructor for term pinsrb. +pub fn constructor_pinsrb( + ctx: &mut C, + arg0: Xmm, + arg1: &GprMem, + arg2: u8, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + // Rule at src/isa/x64/inst.isle line 1816. + let expr0_0 = SseOpcode::Pinsrb; + let expr1_0 = C::xmm_to_reg(ctx, pattern0_0); + let expr2_0 = C::gpr_mem_to_reg_mem(ctx, pattern1_0); + let expr3_0 = OperandSize::Size32; + let expr4_0 = constructor_xmm_rm_r_imm(ctx, &expr0_0, expr1_0, &expr2_0, pattern2_0, &expr3_0)?; + return Some(expr4_0); +} + +// Generated as internal constructor for term pinsrw. +pub fn constructor_pinsrw( + ctx: &mut C, + arg0: Xmm, + arg1: &GprMem, + arg2: u8, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + // Rule at src/isa/x64/inst.isle line 1825. + let expr0_0 = SseOpcode::Pinsrw; + let expr1_0 = C::xmm_to_reg(ctx, pattern0_0); + let expr2_0 = C::gpr_mem_to_reg_mem(ctx, pattern1_0); + let expr3_0 = OperandSize::Size32; + let expr4_0 = constructor_xmm_rm_r_imm(ctx, &expr0_0, expr1_0, &expr2_0, pattern2_0, &expr3_0)?; + return Some(expr4_0); +} + +// Generated as internal constructor for term pinsrd. +pub fn constructor_pinsrd( + ctx: &mut C, + arg0: Xmm, + arg1: &GprMem, + arg2: u8, + arg3: &OperandSize, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + let pattern3_0 = arg3; + // Rule at src/isa/x64/inst.isle line 1834. + let expr0_0 = SseOpcode::Pinsrd; + let expr1_0 = C::xmm_to_reg(ctx, pattern0_0); + let expr2_0 = C::gpr_mem_to_reg_mem(ctx, pattern1_0); + let expr3_0 = + constructor_xmm_rm_r_imm(ctx, &expr0_0, expr1_0, &expr2_0, pattern2_0, pattern3_0)?; + return Some(expr3_0); +} + +// Generated as internal constructor for term insertps. +pub fn constructor_insertps( + ctx: &mut C, + arg0: Xmm, + arg1: &XmmMem, + arg2: u8, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + // Rule at src/isa/x64/inst.isle line 1843. + let expr0_0 = SseOpcode::Insertps; + let expr1_0 = C::xmm_to_reg(ctx, pattern0_0); + let expr2_0 = C::xmm_mem_to_reg_mem(ctx, pattern1_0); + let expr3_0 = OperandSize::Size32; + let expr4_0 = constructor_xmm_rm_r_imm(ctx, &expr0_0, expr1_0, &expr2_0, pattern2_0, &expr3_0)?; + return Some(expr4_0); } // Generated as internal constructor for term pshufd. pub fn constructor_pshufd( ctx: &mut C, - arg0: &RegMem, + arg0: &XmmMem, arg1: u8, arg2: &OperandSize, -) -> Option { +) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1211. - let expr0_0: Type = I8X16; - let expr1_0 = C::temp_writable_reg(ctx, expr0_0); - let expr2_0 = C::writable_reg_to_reg(ctx, expr1_0); - let expr3_0 = SseOpcode::Pshufd; - let expr4_0 = MInst::XmmRmRImm { - op: expr3_0, - src1: expr2_0, - src2: pattern0_0.clone(), - dst: expr1_0, + // Rule at src/isa/x64/inst.isle line 1852. + let expr0_0 = C::temp_writable_xmm(ctx); + let expr1_0 = C::writable_xmm_to_xmm(ctx, expr0_0); + let expr2_0 = SseOpcode::Pshufd; + let expr3_0 = C::xmm_to_reg(ctx, expr1_0); + let expr4_0 = C::xmm_mem_to_reg_mem(ctx, pattern0_0); + let expr5_0 = C::writable_xmm_to_reg(ctx, expr0_0); + let expr6_0 = MInst::XmmRmRImm { + op: expr2_0, + src1: expr3_0, + src2: expr4_0, + dst: expr5_0, imm: pattern1_0, size: pattern2_0.clone(), }; - let expr5_0 = C::emit(ctx, &expr4_0); - return Some(expr2_0); + let expr7_0 = C::emit(ctx, &expr6_0); + return Some(expr1_0); } // Generated as internal constructor for term xmm_unary_rm_r. pub fn constructor_xmm_unary_rm_r( ctx: &mut C, arg0: &SseOpcode, - arg1: &RegMem, -) -> Option { + arg1: &XmmMem, +) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1224. - let expr0_0: Type = I8X16; - let expr1_0 = C::temp_writable_reg(ctx, expr0_0); - let expr2_0 = MInst::XmmUnaryRmR { + // Rule at src/isa/x64/inst.isle line 1865. + let expr0_0 = C::temp_writable_xmm(ctx); + let expr1_0 = MInst::XmmUnaryRmR { op: pattern0_0.clone(), src: pattern1_0.clone(), - dst: expr1_0, + dst: expr0_0, }; - let expr3_0 = C::emit(ctx, &expr2_0); - let expr4_0 = C::writable_reg_to_reg(ctx, expr1_0); - return Some(expr4_0); + let expr2_0 = C::emit(ctx, &expr1_0); + let expr3_0 = C::writable_xmm_to_xmm(ctx, expr0_0); + return Some(expr3_0); } // Generated as internal constructor for term pmovsxbw. -pub fn constructor_pmovsxbw(ctx: &mut C, arg0: &RegMem) -> Option { +pub fn constructor_pmovsxbw(ctx: &mut C, arg0: &XmmMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1231. + // Rule at src/isa/x64/inst.isle line 1872. let expr0_0 = SseOpcode::Pmovsxbw; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); } // Generated as internal constructor for term pmovzxbw. -pub fn constructor_pmovzxbw(ctx: &mut C, arg0: &RegMem) -> Option { +pub fn constructor_pmovzxbw(ctx: &mut C, arg0: &XmmMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1236. + // Rule at src/isa/x64/inst.isle line 1877. let expr0_0 = SseOpcode::Pmovzxbw; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); } // Generated as internal constructor for term pabsb. -pub fn constructor_pabsb(ctx: &mut C, arg0: &RegMem) -> Option { +pub fn constructor_pabsb(ctx: &mut C, arg0: &XmmMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1241. + // Rule at src/isa/x64/inst.isle line 1882. let expr0_0 = SseOpcode::Pabsb; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); } // Generated as internal constructor for term pabsw. -pub fn constructor_pabsw(ctx: &mut C, arg0: &RegMem) -> Option { +pub fn constructor_pabsw(ctx: &mut C, arg0: &XmmMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1246. + // Rule at src/isa/x64/inst.isle line 1887. let expr0_0 = SseOpcode::Pabsw; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); } // Generated as internal constructor for term pabsd. -pub fn constructor_pabsd(ctx: &mut C, arg0: &RegMem) -> Option { +pub fn constructor_pabsd(ctx: &mut C, arg0: &XmmMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1251. + // Rule at src/isa/x64/inst.isle line 1892. let expr0_0 = SseOpcode::Pabsd; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -2088,27 +2616,26 @@ pub fn constructor_pabsd(ctx: &mut C, arg0: &RegMem) -> Option pub fn constructor_xmm_unary_rm_r_evex( ctx: &mut C, arg0: &Avx512Opcode, - arg1: &RegMem, -) -> Option { + arg1: &XmmMem, +) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1256. - let expr0_0: Type = I8X16; - let expr1_0 = C::temp_writable_reg(ctx, expr0_0); - let expr2_0 = MInst::XmmUnaryRmREvex { + // Rule at src/isa/x64/inst.isle line 1897. + let expr0_0 = C::temp_writable_xmm(ctx); + let expr1_0 = MInst::XmmUnaryRmREvex { op: pattern0_0.clone(), src: pattern1_0.clone(), - dst: expr1_0, + dst: expr0_0, }; - let expr3_0 = C::emit(ctx, &expr2_0); - let expr4_0 = C::writable_reg_to_reg(ctx, expr1_0); - return Some(expr4_0); + let expr2_0 = C::emit(ctx, &expr1_0); + let expr3_0 = C::writable_xmm_to_xmm(ctx, expr0_0); + return Some(expr3_0); } // Generated as internal constructor for term vpabsq. -pub fn constructor_vpabsq(ctx: &mut C, arg0: &RegMem) -> Option { +pub fn constructor_vpabsq(ctx: &mut C, arg0: &XmmMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1263. + // Rule at src/isa/x64/inst.isle line 1904. let expr0_0 = Avx512Opcode::Vpabsq; let expr1_0 = constructor_xmm_unary_rm_r_evex(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -2118,140 +2645,35 @@ pub fn constructor_vpabsq(ctx: &mut C, arg0: &RegMem) -> Option pub fn constructor_xmm_rm_r_evex( ctx: &mut C, arg0: &Avx512Opcode, - arg1: &RegMem, - arg2: Reg, -) -> Option { + arg1: &XmmMem, + arg2: Xmm, +) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1268. - let expr0_0: Type = I8X16; - let expr1_0 = C::temp_writable_reg(ctx, expr0_0); - let expr2_0 = MInst::XmmRmREvex { + // Rule at src/isa/x64/inst.isle line 1909. + let expr0_0 = C::temp_writable_xmm(ctx); + let expr1_0 = MInst::XmmRmREvex { op: pattern0_0.clone(), src1: pattern1_0.clone(), src2: pattern2_0, - dst: expr1_0, + dst: expr0_0, }; - let expr3_0 = C::emit(ctx, &expr2_0); - let expr4_0 = C::writable_reg_to_reg(ctx, expr1_0); - return Some(expr4_0); + let expr2_0 = C::emit(ctx, &expr1_0); + let expr3_0 = C::writable_xmm_to_xmm(ctx, expr0_0); + return Some(expr3_0); } // Generated as internal constructor for term vpmullq. -pub fn constructor_vpmullq(ctx: &mut C, arg0: &RegMem, arg1: Reg) -> Option { +pub fn constructor_vpmullq(ctx: &mut C, arg0: &XmmMem, arg1: Xmm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1280. + // Rule at src/isa/x64/inst.isle line 1921. let expr0_0 = Avx512Opcode::Vpmullq; let expr1_0 = constructor_xmm_rm_r_evex(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); } -// Generated as internal constructor for term xmm_rmi_reg. -pub fn constructor_xmm_rmi_reg( - ctx: &mut C, - arg0: &SseOpcode, - arg1: Reg, - arg2: &RegMemImm, -) -> Option { - let pattern0_0 = arg0; - let pattern1_0 = arg1; - let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1287. - let expr0_0: Type = I8X16; - let expr1_0 = C::temp_writable_reg(ctx, expr0_0); - let expr2_0 = MInst::XmmRmiReg { - opcode: pattern0_0.clone(), - src1: pattern1_0, - src2: pattern2_0.clone(), - dst: expr1_0, - }; - let expr3_0 = C::emit(ctx, &expr2_0); - let expr4_0 = C::writable_reg_to_reg(ctx, expr1_0); - return Some(expr4_0); -} - -// Generated as internal constructor for term psllw. -pub fn constructor_psllw(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option { - let pattern0_0 = arg0; - let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1297. - let expr0_0 = SseOpcode::Psllw; - let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; - return Some(expr1_0); -} - -// Generated as internal constructor for term pslld. -pub fn constructor_pslld(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option { - let pattern0_0 = arg0; - let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1302. - let expr0_0 = SseOpcode::Pslld; - let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; - return Some(expr1_0); -} - -// Generated as internal constructor for term psllq. -pub fn constructor_psllq(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option { - let pattern0_0 = arg0; - let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1307. - let expr0_0 = SseOpcode::Psllq; - let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; - return Some(expr1_0); -} - -// Generated as internal constructor for term psrlw. -pub fn constructor_psrlw(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option { - let pattern0_0 = arg0; - let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1312. - let expr0_0 = SseOpcode::Psrlw; - let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; - return Some(expr1_0); -} - -// Generated as internal constructor for term psrld. -pub fn constructor_psrld(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option { - let pattern0_0 = arg0; - let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1317. - let expr0_0 = SseOpcode::Psrld; - let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; - return Some(expr1_0); -} - -// Generated as internal constructor for term psrlq. -pub fn constructor_psrlq(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option { - let pattern0_0 = arg0; - let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1322. - let expr0_0 = SseOpcode::Psrlq; - let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; - return Some(expr1_0); -} - -// Generated as internal constructor for term psraw. -pub fn constructor_psraw(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option { - let pattern0_0 = arg0; - let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1327. - let expr0_0 = SseOpcode::Psraw; - let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; - return Some(expr1_0); -} - -// Generated as internal constructor for term psrad. -pub fn constructor_psrad(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option { - let pattern0_0 = arg0; - let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1332. - let expr0_0 = SseOpcode::Psrad; - let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; - return Some(expr1_0); -} - // Generated as internal constructor for term mul_hi. pub fn constructor_mul_hi( ctx: &mut C, @@ -2264,7 +2686,7 @@ pub fn constructor_mul_hi( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1339. + // Rule at src/isa/x64/inst.isle line 1930. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::temp_writable_reg(ctx, pattern0_0); let expr2_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); @@ -2293,179 +2715,192 @@ pub fn constructor_mulhi_u( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1355. + // Rule at src/isa/x64/inst.isle line 1946. let expr0_0: bool = false; let expr1_0 = constructor_mul_hi(ctx, pattern0_0, expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); } -// Generated as internal constructor for term cmpps. -pub fn constructor_cmpps( +// Generated as internal constructor for term xmm_rmi_xmm. +pub fn constructor_xmm_rmi_xmm( ctx: &mut C, - arg0: Reg, - arg1: &RegMem, - arg2: &FcmpImm, -) -> Option { + arg0: &SseOpcode, + arg1: Xmm, + arg2: &XmmMemImm, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + // Rule at src/isa/x64/inst.isle line 1951. + let expr0_0 = C::temp_writable_xmm(ctx); + let expr1_0 = MInst::XmmRmiReg { + opcode: pattern0_0.clone(), + src1: pattern1_0, + src2: pattern2_0.clone(), + dst: expr0_0, + }; + let expr2_0 = C::emit(ctx, &expr1_0); + let expr3_0 = C::writable_xmm_to_xmm(ctx, expr0_0); + return Some(expr3_0); +} + +// Generated as internal constructor for term psllw. +pub fn constructor_psllw(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 1961. + let expr0_0 = SseOpcode::Psllw; + let expr1_0 = constructor_xmm_rmi_xmm(ctx, &expr0_0, pattern0_0, pattern1_0)?; + return Some(expr1_0); +} + +// Generated as internal constructor for term pslld. +pub fn constructor_pslld(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 1966. + let expr0_0 = SseOpcode::Pslld; + let expr1_0 = constructor_xmm_rmi_xmm(ctx, &expr0_0, pattern0_0, pattern1_0)?; + return Some(expr1_0); +} + +// Generated as internal constructor for term psllq. +pub fn constructor_psllq(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 1971. + let expr0_0 = SseOpcode::Psllq; + let expr1_0 = constructor_xmm_rmi_xmm(ctx, &expr0_0, pattern0_0, pattern1_0)?; + return Some(expr1_0); +} + +// Generated as internal constructor for term psrlw. +pub fn constructor_psrlw(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1360. - let expr0_0 = SseOpcode::Cmpps; - let expr1_0 = C::encode_fcmp_imm(ctx, pattern2_0); - let expr2_0 = OperandSize::Size32; - let expr3_0 = - constructor_xmm_rm_r_imm(ctx, &expr0_0, pattern0_0, pattern1_0, expr1_0, &expr2_0)?; - return Some(expr3_0); + // Rule at src/isa/x64/inst.isle line 1976. + let expr0_0 = SseOpcode::Psrlw; + let expr1_0 = constructor_xmm_rmi_xmm(ctx, &expr0_0, pattern0_0, pattern1_0)?; + return Some(expr1_0); } -// Generated as internal constructor for term cmppd. -pub fn constructor_cmppd( - ctx: &mut C, - arg0: Reg, - arg1: &RegMem, - arg2: &FcmpImm, -) -> Option { +// Generated as internal constructor for term psrld. +pub fn constructor_psrld(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1373. - let expr0_0 = SseOpcode::Cmppd; - let expr1_0 = C::encode_fcmp_imm(ctx, pattern2_0); - let expr2_0 = OperandSize::Size32; - let expr3_0 = - constructor_xmm_rm_r_imm(ctx, &expr0_0, pattern0_0, pattern1_0, expr1_0, &expr2_0)?; - return Some(expr3_0); + // Rule at src/isa/x64/inst.isle line 1981. + let expr0_0 = SseOpcode::Psrld; + let expr1_0 = constructor_xmm_rmi_xmm(ctx, &expr0_0, pattern0_0, pattern1_0)?; + return Some(expr1_0); } -// Generated as internal constructor for term gpr_to_xmm. -pub fn constructor_gpr_to_xmm( - ctx: &mut C, - arg0: Type, - arg1: &SseOpcode, - arg2: &RegMem, - arg3: &OperandSize, -) -> Option { +// Generated as internal constructor for term psrlq. +pub fn constructor_psrlq(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - let pattern2_0 = arg2; - let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1382. - let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); - let expr1_0 = MInst::GprToXmm { - op: pattern1_0.clone(), - src: pattern2_0.clone(), - dst: expr0_0, - src_size: pattern3_0.clone(), - }; - let expr2_0 = C::emit(ctx, &expr1_0); - let expr3_0 = C::writable_reg_to_reg(ctx, expr0_0); - return Some(expr3_0); + // Rule at src/isa/x64/inst.isle line 1986. + let expr0_0 = SseOpcode::Psrlq; + let expr1_0 = constructor_xmm_rmi_xmm(ctx, &expr0_0, pattern0_0, pattern1_0)?; + return Some(expr1_0); } -// Generated as internal constructor for term pinsrb. -pub fn constructor_pinsrb( - ctx: &mut C, - arg0: Reg, - arg1: &RegMem, - arg2: u8, -) -> Option { +// Generated as internal constructor for term psraw. +pub fn constructor_psraw(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1389. - let expr0_0 = SseOpcode::Pinsrb; - let expr1_0 = OperandSize::Size32; - let expr2_0 = - constructor_xmm_rm_r_imm(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0, &expr1_0)?; - return Some(expr2_0); + // Rule at src/isa/x64/inst.isle line 1991. + let expr0_0 = SseOpcode::Psraw; + let expr1_0 = constructor_xmm_rmi_xmm(ctx, &expr0_0, pattern0_0, pattern1_0)?; + return Some(expr1_0); } -// Generated as internal constructor for term pinsrw. -pub fn constructor_pinsrw( - ctx: &mut C, - arg0: Reg, - arg1: &RegMem, - arg2: u8, -) -> Option { +// Generated as internal constructor for term psrad. +pub fn constructor_psrad(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1394. - let expr0_0 = SseOpcode::Pinsrw; - let expr1_0 = OperandSize::Size32; - let expr2_0 = - constructor_xmm_rm_r_imm(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0, &expr1_0)?; - return Some(expr2_0); + // Rule at src/isa/x64/inst.isle line 1996. + let expr0_0 = SseOpcode::Psrad; + let expr1_0 = constructor_xmm_rmi_xmm(ctx, &expr0_0, pattern0_0, pattern1_0)?; + return Some(expr1_0); } -// Generated as internal constructor for term pinsrd. -pub fn constructor_pinsrd( - ctx: &mut C, - arg0: Reg, - arg1: &RegMem, - arg2: u8, - arg3: &OperandSize, -) -> Option { +// Generated as internal constructor for term pextrd. +pub fn constructor_pextrd(ctx: &mut C, arg0: Type, arg1: Xmm, arg2: u8) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1399. - let expr0_0 = SseOpcode::Pinsrd; - let expr1_0 = constructor_xmm_rm_r_imm( - ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0, pattern3_0, - )?; + // Rule at src/isa/x64/inst.isle line 2001. + let expr0_0 = C::temp_writable_gpr(ctx); + let expr1_0 = C::writable_gpr_to_gpr(ctx, expr0_0); + let expr2_0 = SseOpcode::Pextrd; + let expr3_0 = C::gpr_to_reg(ctx, expr1_0); + let expr4_0 = C::xmm_to_reg(ctx, pattern1_0); + let expr5_0 = RegMem::Reg { reg: expr4_0 }; + let expr6_0 = C::writable_gpr_to_reg(ctx, expr0_0); + let expr7_0 = C::lane_type(ctx, pattern0_0); + let expr8_0 = C::operand_size_of_type_32_64(ctx, expr7_0); + let expr9_0 = MInst::XmmRmRImm { + op: expr2_0, + src1: expr3_0, + src2: expr5_0, + dst: expr6_0, + imm: pattern2_0, + size: expr8_0, + }; + let expr10_0 = C::emit(ctx, &expr9_0); return Some(expr1_0); } -// Generated as internal constructor for term insertps. -pub fn constructor_insertps( +// Generated as internal constructor for term cmppd. +pub fn constructor_cmppd( ctx: &mut C, - arg0: Reg, - arg1: &RegMem, - arg2: u8, -) -> Option { + arg0: Xmm, + arg1: &XmmMem, + arg2: &FcmpImm, +) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1404. - let expr0_0 = SseOpcode::Insertps; - let expr1_0 = OperandSize::Size32; - let expr2_0 = - constructor_xmm_rm_r_imm(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0, &expr1_0)?; - return Some(expr2_0); + // Rule at src/isa/x64/inst.isle line 2018. + let expr0_0 = SseOpcode::Cmppd; + let expr1_0 = C::xmm_to_reg(ctx, pattern0_0); + let expr2_0 = C::xmm_mem_to_reg_mem(ctx, pattern1_0); + let expr3_0 = C::encode_fcmp_imm(ctx, pattern2_0); + let expr4_0 = OperandSize::Size32; + let expr5_0 = constructor_xmm_rm_r_imm(ctx, &expr0_0, expr1_0, &expr2_0, expr3_0, &expr4_0)?; + return Some(expr5_0); } -// Generated as internal constructor for term pextrd. -pub fn constructor_pextrd(ctx: &mut C, arg0: Type, arg1: Reg, arg2: u8) -> Option { +// Generated as internal constructor for term gpr_to_xmm. +pub fn constructor_gpr_to_xmm( + ctx: &mut C, + arg0: &SseOpcode, + arg1: &GprMem, + arg2: &OperandSize, +) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1409. - let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); - let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); - let expr2_0 = SseOpcode::Pextrd; - let expr3_0 = RegMem::Reg { reg: pattern1_0 }; - let expr4_0 = C::lane_type(ctx, pattern0_0); - let expr5_0 = C::operand_size_of_type_32_64(ctx, expr4_0); - let expr6_0 = MInst::XmmRmRImm { - op: expr2_0, - src1: expr1_0, - src2: expr3_0, + // Rule at src/isa/x64/inst.isle line 2027. + let expr0_0 = C::temp_writable_xmm(ctx); + let expr1_0 = C::gpr_mem_to_reg_mem(ctx, pattern1_0); + let expr2_0 = MInst::GprToXmm { + op: pattern0_0.clone(), + src: expr1_0, dst: expr0_0, - imm: pattern2_0, - size: expr5_0, + src_size: pattern2_0.clone(), }; - let expr7_0 = C::emit(ctx, &expr6_0); - return Some(expr1_0); + let expr3_0 = C::emit(ctx, &expr2_0); + let expr4_0 = C::writable_xmm_to_xmm(ctx, expr0_0); + return Some(expr4_0); } // Generated as internal constructor for term not. -pub fn constructor_not(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { +pub fn constructor_not(ctx: &mut C, arg0: Type, arg1: Gpr) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1422. - let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); + // Rule at src/isa/x64/inst.isle line 2034. + let expr0_0 = C::temp_writable_gpr(ctx); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = MInst::Not { size: expr1_0, @@ -2473,16 +2908,16 @@ pub fn constructor_not(ctx: &mut C, arg0: Type, arg1: Reg) -> Option dst: expr0_0, }; let expr3_0 = C::emit(ctx, &expr2_0); - let expr4_0 = C::writable_reg_to_reg(ctx, expr0_0); + let expr4_0 = C::writable_gpr_to_gpr(ctx, expr0_0); return Some(expr4_0); } // Generated as internal constructor for term neg. -pub fn constructor_neg(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { +pub fn constructor_neg(ctx: &mut C, arg0: Type, arg1: Gpr) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1430. - let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); + // Rule at src/isa/x64/inst.isle line 2042. + let expr0_0 = C::temp_writable_gpr(ctx); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = MInst::Neg { size: expr1_0, @@ -2490,29 +2925,28 @@ pub fn constructor_neg(ctx: &mut C, arg0: Type, arg1: Reg) -> Option dst: expr0_0, }; let expr3_0 = C::emit(ctx, &expr2_0); - let expr4_0 = C::writable_reg_to_reg(ctx, expr0_0); + let expr4_0 = C::writable_gpr_to_gpr(ctx, expr0_0); return Some(expr4_0); } // Generated as internal constructor for term lea. -pub fn constructor_lea(ctx: &mut C, arg0: &SyntheticAmode) -> Option { +pub fn constructor_lea(ctx: &mut C, arg0: &SyntheticAmode) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1437. - let expr0_0: Type = I64; - let expr1_0 = C::temp_writable_reg(ctx, expr0_0); - let expr2_0 = MInst::LoadEffectiveAddress { + // Rule at src/isa/x64/inst.isle line 2049. + let expr0_0 = C::temp_writable_gpr(ctx); + let expr1_0 = MInst::LoadEffectiveAddress { addr: pattern0_0.clone(), - dst: expr1_0, + dst: expr0_0, }; - let expr3_0 = C::emit(ctx, &expr2_0); - let expr4_0 = C::writable_reg_to_reg(ctx, expr1_0); - return Some(expr4_0); + let expr2_0 = C::emit(ctx, &expr1_0); + let expr3_0 = C::writable_gpr_to_gpr(ctx, expr0_0); + return Some(expr3_0); } // Generated as internal constructor for term ud2. pub fn constructor_ud2(ctx: &mut C, arg0: &TrapCode) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1444. + // Rule at src/isa/x64/inst.isle line 2056. let expr0_0 = MInst::Ud2 { trap_code: pattern0_0.clone(), }; @@ -2557,13 +2991,13 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { match &pattern2_0 { &Opcode::Trap => { - // Rule at src/isa/x64/lower.isle line 1438. + // Rule at src/isa/x64/lower.isle line 1463. let expr0_0 = constructor_ud2(ctx, &pattern2_1)?; let expr1_0 = constructor_value_regs_none(ctx, &expr0_0)?; return Some(expr1_0); } &Opcode::ResumableTrap => { - // Rule at src/isa/x64/lower.isle line 1443. + // Rule at src/isa/x64/lower.isle line 1468. let expr0_0 = constructor_ud2(ctx, &pattern2_1)?; let expr1_0 = constructor_value_regs_none(ctx, &expr0_0)?; return Some(expr1_0); @@ -2580,12 +3014,12 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 384. + // Rule at src/isa/x64/lower.isle line 386. let expr0_0 = C::put_in_regs(ctx, pattern7_0); let expr1_0: usize = 0; let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); @@ -2649,7 +3083,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 458. + // Rule at src/isa/x64/lower.isle line 462. let expr0_0 = C::put_in_regs(ctx, pattern7_0); let expr1_0: usize = 0; let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); @@ -2665,7 +3099,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 524. + // Rule at src/isa/x64/lower.isle line 528. let expr0_0 = C::put_in_regs(ctx, pattern7_0); let expr1_0: usize = 0; let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); @@ -2686,7 +3120,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::Bnot = &pattern5_0 { - // Rule at src/isa/x64/lower.isle line 1304. + // Rule at src/isa/x64/lower.isle line 1321. let expr0_0 = constructor_i128_not(ctx, pattern5_1)?; return Some(expr0_0); } @@ -2769,7 +3203,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 992. + // Rule at src/isa/x64/lower.isle line 1007. let expr0_0 = C::put_in_regs(ctx, pattern7_0); let expr1_0: usize = 0; let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); @@ -2805,7 +3239,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 374. + // Rule at src/isa/x64/lower.isle line 376. let expr0_0 = C::put_in_regs(ctx, pattern7_0); let expr1_0: usize = 0; let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); @@ -2828,7 +3262,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 455. + // Rule at src/isa/x64/lower.isle line 459. let expr0_0 = C::put_in_regs(ctx, pattern7_0); let expr1_0 = C::put_in_regs(ctx, pattern7_1); let expr2_0 = constructor_or_i128(ctx, expr0_0, expr1_0)?; @@ -2837,7 +3271,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 514. + // Rule at src/isa/x64/lower.isle line 518. let expr0_0 = C::put_in_regs(ctx, pattern7_0); let expr1_0: usize = 0; let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); @@ -2860,7 +3294,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 860. + // Rule at src/isa/x64/lower.isle line 875. let expr0_0 = C::put_in_regs(ctx, pattern7_0); let expr1_0 = constructor_lo_reg(ctx, pattern7_1)?; let expr2_0 = constructor_shl_i128(ctx, expr0_0, expr1_0)?; @@ -2877,7 +3311,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 900. + // Rule at src/isa/x64/lower.isle line 915. let expr0_0 = C::put_in_regs(ctx, pattern7_0); let expr1_0 = constructor_lo_reg(ctx, pattern7_1)?; let expr2_0 = constructor_shr_i128(ctx, expr0_0, expr1_0)?; @@ -2894,7 +3328,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 568. + // Rule at src/isa/x64/lower.isle line 572. let expr0_0 = constructor_lo_reg(ctx, pattern7_1)?; let expr1_0 = C::put_in_regs(ctx, pattern7_0); let expr2_0 = constructor_shl_i128(ctx, expr1_0, expr0_0)?; @@ -2903,7 +3337,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 664. + // Rule at src/isa/x64/lower.isle line 671. let expr0_0 = constructor_lo_reg(ctx, pattern7_1)?; let expr1_0 = C::put_in_regs(ctx, pattern7_0); let expr2_0 = constructor_shr_i128(ctx, expr1_0, expr0_0)?; @@ -2912,7 +3346,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 761. + // Rule at src/isa/x64/lower.isle line 773. let expr0_0 = constructor_lo_reg(ctx, pattern7_1)?; let expr1_0 = C::put_in_regs(ctx, pattern7_0); let expr2_0 = constructor_sar_i128(ctx, expr1_0, expr0_0)?; @@ -2926,7 +3360,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::Bnot = &pattern5_0 { - // Rule at src/isa/x64/lower.isle line 1301. + // Rule at src/isa/x64/lower.isle line 1318. let expr0_0 = constructor_i128_not(ctx, pattern5_1)?; return Some(expr0_0); } @@ -2971,50 +3405,50 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1405. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); + // Rule at src/isa/x64/lower.isle line 1430. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_pminsb(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::Umin => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1427. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); + // Rule at src/isa/x64/lower.isle line 1452. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_pminub(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::Imax => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1394. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); + // Rule at src/isa/x64/lower.isle line 1419. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_pmaxsb(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::Umax => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1416. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); + // Rule at src/isa/x64/lower.isle line 1441. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_pmaxub(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::Ishl => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 580. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); + // Rule at src/isa/x64/lower.isle line 584. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = C::put_in_reg_mem_imm(ctx, pattern7_1); - let expr2_0 = constructor_reg_mem_imm_to_xmm(ctx, &expr1_0)?; + let expr2_0 = constructor_mov_rmi_to_xmm(ctx, &expr1_0)?; let expr3_0 = constructor_psllw(ctx, expr0_0, &expr2_0)?; let expr4_0 = constructor_ishl_i8x16_mask(ctx, &expr1_0)?; let expr5_0: Type = I8X16; @@ -3022,17 +3456,18 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 674. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); + // Rule at src/isa/x64/lower.isle line 681. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = C::put_in_reg_mem_imm(ctx, pattern7_1); - let expr2_0 = constructor_reg_mem_imm_to_xmm(ctx, &expr1_0)?; + let expr2_0 = constructor_mov_rmi_to_xmm(ctx, &expr1_0)?; let expr3_0 = constructor_psrlw(ctx, expr0_0, &expr2_0)?; let expr4_0 = constructor_ushr_i8x16_mask(ctx, &expr1_0)?; let expr5_0: Type = I8X16; @@ -3040,28 +3475,29 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); let pattern8_0 = C::value_type(ctx, pattern7_1); - // Rule at src/isa/x64/lower.isle line 782. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); - let expr1_0 = RegMem::Reg { reg: expr0_0 }; + // Rule at src/isa/x64/lower.isle line 794. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; + let expr1_0 = C::xmm_to_xmm_mem(ctx, expr0_0); let expr2_0 = constructor_punpcklbw(ctx, expr0_0, &expr1_0)?; - let expr3_0 = RegMem::Reg { reg: expr0_0 }; + let expr3_0 = C::xmm_to_xmm_mem(ctx, expr0_0); let expr4_0 = constructor_punpckhbw(ctx, expr0_0, &expr3_0)?; let expr5_0 = C::put_in_reg_mem_imm(ctx, pattern7_1); let expr6_0 = constructor_sshr_i8x16_bigger_shift(ctx, pattern8_0, &expr5_0)?; let expr7_0 = constructor_psraw(ctx, expr2_0, &expr6_0)?; let expr8_0 = constructor_psraw(ctx, expr4_0, &expr6_0)?; - let expr9_0 = RegMem::Reg { reg: expr8_0 }; + let expr9_0 = C::xmm_to_xmm_mem(ctx, expr8_0); let expr10_0 = constructor_packsswb(ctx, expr7_0, &expr9_0)?; - let expr11_0 = C::value_reg(ctx, expr10_0); + let expr11_0 = constructor_value_xmm(ctx, expr10_0)?; return Some(expr11_0); } _ => {} @@ -3073,20 +3509,21 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { match &pattern5_0 { &Opcode::Ineg => { - // Rule at src/isa/x64/lower.isle line 917. + // Rule at src/isa/x64/lower.isle line 932. let expr0_0: Type = I8X16; let expr1_0: u64 = 0; let expr2_0 = constructor_imm(ctx, expr0_0, expr1_0)?; - let expr3_0 = C::put_in_reg_mem(ctx, pattern5_1); - let expr4_0 = constructor_psubb(ctx, expr2_0, &expr3_0)?; - let expr5_0 = C::value_reg(ctx, expr4_0); - return Some(expr5_0); + let expr3_0 = C::xmm_new(ctx, expr2_0); + let expr4_0 = constructor_put_in_xmm_mem(ctx, pattern5_1)?; + let expr5_0 = constructor_psubb(ctx, expr3_0, &expr4_0)?; + let expr6_0 = constructor_value_xmm(ctx, expr5_0)?; + return Some(expr6_0); } &Opcode::Iabs => { - // Rule at src/isa/x64/lower.isle line 1247. - let expr0_0 = C::put_in_reg_mem(ctx, pattern5_1); + // Rule at src/isa/x64/lower.isle line 1262. + let expr0_0 = constructor_put_in_xmm_mem(ctx, pattern5_1)?; let expr1_0 = constructor_pabsb(ctx, &expr0_0)?; - let expr2_0 = C::value_reg(ctx, expr1_0); + let expr2_0 = constructor_value_xmm(ctx, expr1_0)?; return Some(expr2_0); } _ => {} @@ -3106,74 +3543,74 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1408. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); + // Rule at src/isa/x64/lower.isle line 1433. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_pminsw(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::Umin => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1430. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); + // Rule at src/isa/x64/lower.isle line 1455. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_pminuw(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::Imax => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1397. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); + // Rule at src/isa/x64/lower.isle line 1422. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_pmaxsw(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::Umax => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1419. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); + // Rule at src/isa/x64/lower.isle line 1444. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_pmaxuw(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::Ishl => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 621. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); + // Rule at src/isa/x64/lower.isle line 626. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = C::put_in_reg_mem_imm(ctx, pattern7_1); - let expr2_0 = constructor_reg_mem_imm_to_xmm(ctx, &expr1_0)?; + let expr2_0 = constructor_mov_rmi_to_xmm(ctx, &expr1_0)?; let expr3_0 = constructor_psllw(ctx, expr0_0, &expr2_0)?; - let expr4_0 = C::value_reg(ctx, expr3_0); + let expr4_0 = constructor_value_xmm(ctx, expr3_0)?; return Some(expr4_0); } &Opcode::Ushr => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 715. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); + // Rule at src/isa/x64/lower.isle line 725. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = C::put_in_reg_mem_imm(ctx, pattern7_1); - let expr2_0 = constructor_reg_mem_imm_to_xmm(ctx, &expr1_0)?; + let expr2_0 = constructor_mov_rmi_to_xmm(ctx, &expr1_0)?; let expr3_0 = constructor_psrlw(ctx, expr0_0, &expr2_0)?; - let expr4_0 = C::value_reg(ctx, expr3_0); + let expr4_0 = constructor_value_xmm(ctx, expr3_0)?; return Some(expr4_0); } &Opcode::Sshr => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 804. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); + // Rule at src/isa/x64/lower.isle line 817. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = C::put_in_reg_mem_imm(ctx, pattern7_1); - let expr2_0 = constructor_reg_mem_imm_to_xmm(ctx, &expr1_0)?; + let expr2_0 = constructor_mov_rmi_to_xmm(ctx, &expr1_0)?; let expr3_0 = constructor_psraw(ctx, expr0_0, &expr2_0)?; - let expr4_0 = C::value_reg(ctx, expr3_0); + let expr4_0 = constructor_value_xmm(ctx, expr3_0)?; return Some(expr4_0); } _ => {} @@ -3185,20 +3622,21 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { match &pattern5_0 { &Opcode::Ineg => { - // Rule at src/isa/x64/lower.isle line 921. + // Rule at src/isa/x64/lower.isle line 936. let expr0_0: Type = I16X8; let expr1_0: u64 = 0; let expr2_0 = constructor_imm(ctx, expr0_0, expr1_0)?; - let expr3_0 = C::put_in_reg_mem(ctx, pattern5_1); - let expr4_0 = constructor_psubw(ctx, expr2_0, &expr3_0)?; - let expr5_0 = C::value_reg(ctx, expr4_0); - return Some(expr5_0); + let expr3_0 = C::xmm_new(ctx, expr2_0); + let expr4_0 = constructor_put_in_xmm_mem(ctx, pattern5_1)?; + let expr5_0 = constructor_psubw(ctx, expr3_0, &expr4_0)?; + let expr6_0 = constructor_value_xmm(ctx, expr5_0)?; + return Some(expr6_0); } &Opcode::Iabs => { - // Rule at src/isa/x64/lower.isle line 1250. - let expr0_0 = C::put_in_reg_mem(ctx, pattern5_1); + // Rule at src/isa/x64/lower.isle line 1265. + let expr0_0 = constructor_put_in_xmm_mem(ctx, pattern5_1)?; let expr1_0 = constructor_pabsw(ctx, &expr0_0)?; - let expr2_0 = C::value_reg(ctx, expr1_0); + let expr2_0 = constructor_value_xmm(ctx, expr1_0)?; return Some(expr2_0); } _ => {} @@ -3218,74 +3656,74 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1411. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); + // Rule at src/isa/x64/lower.isle line 1436. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_pminsd(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::Umin => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1433. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); + // Rule at src/isa/x64/lower.isle line 1458. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_pminud(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::Imax => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1400. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); + // Rule at src/isa/x64/lower.isle line 1425. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_pmaxsd(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::Umax => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1422. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); + // Rule at src/isa/x64/lower.isle line 1447. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_pmaxud(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::Ishl => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 624. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); + // Rule at src/isa/x64/lower.isle line 630. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = C::put_in_reg_mem_imm(ctx, pattern7_1); - let expr2_0 = constructor_reg_mem_imm_to_xmm(ctx, &expr1_0)?; + let expr2_0 = constructor_mov_rmi_to_xmm(ctx, &expr1_0)?; let expr3_0 = constructor_pslld(ctx, expr0_0, &expr2_0)?; - let expr4_0 = C::value_reg(ctx, expr3_0); + let expr4_0 = constructor_value_xmm(ctx, expr3_0)?; return Some(expr4_0); } &Opcode::Ushr => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 718. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); + // Rule at src/isa/x64/lower.isle line 729. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = C::put_in_reg_mem_imm(ctx, pattern7_1); - let expr2_0 = constructor_reg_mem_imm_to_xmm(ctx, &expr1_0)?; + let expr2_0 = constructor_mov_rmi_to_xmm(ctx, &expr1_0)?; let expr3_0 = constructor_psrld(ctx, expr0_0, &expr2_0)?; - let expr4_0 = C::value_reg(ctx, expr3_0); + let expr4_0 = constructor_value_xmm(ctx, expr3_0)?; return Some(expr4_0); } &Opcode::Sshr => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 807. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); + // Rule at src/isa/x64/lower.isle line 821. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = C::put_in_reg_mem_imm(ctx, pattern7_1); - let expr2_0 = constructor_reg_mem_imm_to_xmm(ctx, &expr1_0)?; + let expr2_0 = constructor_mov_rmi_to_xmm(ctx, &expr1_0)?; let expr3_0 = constructor_psrad(ctx, expr0_0, &expr2_0)?; - let expr4_0 = C::value_reg(ctx, expr3_0); + let expr4_0 = constructor_value_xmm(ctx, expr3_0)?; return Some(expr4_0); } _ => {} @@ -3297,20 +3735,21 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { match &pattern5_0 { &Opcode::Ineg => { - // Rule at src/isa/x64/lower.isle line 925. + // Rule at src/isa/x64/lower.isle line 940. let expr0_0: Type = I32X4; let expr1_0: u64 = 0; let expr2_0 = constructor_imm(ctx, expr0_0, expr1_0)?; - let expr3_0 = C::put_in_reg_mem(ctx, pattern5_1); - let expr4_0 = constructor_psubd(ctx, expr2_0, &expr3_0)?; - let expr5_0 = C::value_reg(ctx, expr4_0); - return Some(expr5_0); + let expr3_0 = C::xmm_new(ctx, expr2_0); + let expr4_0 = constructor_put_in_xmm_mem(ctx, pattern5_1)?; + let expr5_0 = constructor_psubd(ctx, expr3_0, &expr4_0)?; + let expr6_0 = constructor_value_xmm(ctx, expr5_0)?; + return Some(expr6_0); } &Opcode::Iabs => { - // Rule at src/isa/x64/lower.isle line 1253. - let expr0_0 = C::put_in_reg_mem(ctx, pattern5_1); + // Rule at src/isa/x64/lower.isle line 1268. + let expr0_0 = constructor_put_in_xmm_mem(ctx, pattern5_1)?; let expr1_0 = constructor_pabsd(ctx, &expr0_0)?; - let expr2_0 = C::value_reg(ctx, expr1_0); + let expr2_0 = constructor_value_xmm(ctx, expr1_0)?; return Some(expr2_0); } _ => {} @@ -3330,30 +3769,30 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 627. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); + // Rule at src/isa/x64/lower.isle line 634. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = C::put_in_reg_mem_imm(ctx, pattern7_1); - let expr2_0 = constructor_reg_mem_imm_to_xmm(ctx, &expr1_0)?; + let expr2_0 = constructor_mov_rmi_to_xmm(ctx, &expr1_0)?; let expr3_0 = constructor_psllq(ctx, expr0_0, &expr2_0)?; - let expr4_0 = C::value_reg(ctx, expr3_0); + let expr4_0 = constructor_value_xmm(ctx, expr3_0)?; return Some(expr4_0); } &Opcode::Ushr => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 721. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); + // Rule at src/isa/x64/lower.isle line 733. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = C::put_in_reg_mem_imm(ctx, pattern7_1); - let expr2_0 = constructor_reg_mem_imm_to_xmm(ctx, &expr1_0)?; + let expr2_0 = constructor_mov_rmi_to_xmm(ctx, &expr1_0)?; let expr3_0 = constructor_psrlq(ctx, expr0_0, &expr2_0)?; - let expr4_0 = C::value_reg(ctx, expr3_0); + let expr4_0 = constructor_value_xmm(ctx, expr3_0)?; return Some(expr4_0); } &Opcode::Sshr => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 819. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); + // Rule at src/isa/x64/lower.isle line 833. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0: Type = I64; let expr2_0: u8 = 0; let expr3_0 = constructor_pextrd(ctx, expr1_0, expr0_0, expr2_0)?; @@ -3363,15 +3802,19 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option {} } @@ -3382,27 +3825,29 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { match &pattern5_0 { &Opcode::Ineg => { - // Rule at src/isa/x64/lower.isle line 929. + // Rule at src/isa/x64/lower.isle line 944. let expr0_0: Type = I64X2; let expr1_0: u64 = 0; let expr2_0 = constructor_imm(ctx, expr0_0, expr1_0)?; - let expr3_0 = C::put_in_reg_mem(ctx, pattern5_1); - let expr4_0 = constructor_psubq(ctx, expr2_0, &expr3_0)?; - let expr5_0 = C::value_reg(ctx, expr4_0); - return Some(expr5_0); + let expr3_0 = C::xmm_new(ctx, expr2_0); + let expr4_0 = constructor_put_in_xmm_mem(ctx, pattern5_1)?; + let expr5_0 = constructor_psubq(ctx, expr3_0, &expr4_0)?; + let expr6_0 = constructor_value_xmm(ctx, expr5_0)?; + return Some(expr6_0); } &Opcode::Iabs => { - // Rule at src/isa/x64/lower.isle line 1267. - let expr0_0 = C::put_in_reg(ctx, pattern5_1); + // Rule at src/isa/x64/lower.isle line 1282. + let expr0_0 = constructor_put_in_xmm(ctx, pattern5_1)?; let expr1_0: Type = I64X2; let expr2_0: u64 = 0; let expr3_0 = constructor_imm(ctx, expr1_0, expr2_0)?; - let expr4_0 = RegMem::Reg { reg: expr0_0 }; - let expr5_0 = constructor_psubq(ctx, expr3_0, &expr4_0)?; - let expr6_0 = RegMem::Reg { reg: expr0_0 }; - let expr7_0 = constructor_blendvpd(ctx, expr5_0, &expr6_0, expr5_0)?; - let expr8_0 = C::value_reg(ctx, expr7_0); - return Some(expr8_0); + let expr4_0 = C::xmm_new(ctx, expr3_0); + let expr5_0 = C::xmm_to_xmm_mem(ctx, expr0_0); + let expr6_0 = constructor_psubq(ctx, expr4_0, &expr5_0)?; + let expr7_0 = C::xmm_to_xmm_mem(ctx, expr0_0); + let expr8_0 = constructor_blendvpd(ctx, expr6_0, &expr7_0, expr6_0)?; + let expr9_0 = constructor_value_xmm(ctx, expr8_0)?; + return Some(expr9_0); } _ => {} } @@ -3418,17 +3863,18 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::BandNot = &pattern4_0 { let (pattern6_0, pattern6_1) = C::unpack_value_array_2(ctx, &pattern4_1); - // Rule at src/isa/x64/lower.isle line 1240. - let expr0_0 = C::put_in_reg(ctx, pattern6_1); - let expr1_0 = C::put_in_reg_mem(ctx, pattern6_0); + // Rule at src/isa/x64/lower.isle line 1255. + let expr0_0 = constructor_put_in_xmm(ctx, pattern6_1)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern6_0)?; let expr2_0 = constructor_sse_and_not(ctx, pattern2_0, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } } @@ -3497,11 +3944,11 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { let (pattern9_0, pattern9_1) = C::unpack_value_array_2(ctx, &pattern7_1); - // Rule at src/isa/x64/lower.isle line 935. - let expr0_0 = C::put_in_reg(ctx, pattern9_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); + // Rule at src/isa/x64/lower.isle line 950. + let expr0_0 = constructor_put_in_xmm(ctx, pattern9_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern9_1)?; let expr2_0 = constructor_pavgb(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::UaddSat => { let (pattern9_0, pattern9_1) = C::unpack_value_array_2(ctx, &pattern7_1); // Rule at src/isa/x64/lower.isle line 144. - let expr0_0 = C::put_in_reg(ctx, pattern9_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); + let expr0_0 = constructor_put_in_xmm(ctx, pattern9_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern9_1)?; let expr2_0 = constructor_paddusb(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::SaddSat => { let (pattern9_0, pattern9_1) = C::unpack_value_array_2(ctx, &pattern7_1); // Rule at src/isa/x64/lower.isle line 132. - let expr0_0 = C::put_in_reg(ctx, pattern9_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); + let expr0_0 = constructor_put_in_xmm(ctx, pattern9_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern9_1)?; let expr2_0 = constructor_paddsb(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::UsubSat => { let (pattern9_0, pattern9_1) = C::unpack_value_array_2(ctx, &pattern7_1); // Rule at src/isa/x64/lower.isle line 313. - let expr0_0 = C::put_in_reg(ctx, pattern9_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); + let expr0_0 = constructor_put_in_xmm(ctx, pattern9_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern9_1)?; let expr2_0 = constructor_psubusb(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::SsubSat => { let (pattern9_0, pattern9_1) = C::unpack_value_array_2(ctx, &pattern7_1); // Rule at src/isa/x64/lower.isle line 301. - let expr0_0 = C::put_in_reg(ctx, pattern9_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); + let expr0_0 = constructor_put_in_xmm(ctx, pattern9_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern9_1)?; let expr2_0 = constructor_psubsb(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::Iadd => { let (pattern9_0, pattern9_1) = C::unpack_value_array_2(ctx, &pattern7_1); // Rule at src/isa/x64/lower.isle line 96. - let expr0_0 = C::put_in_reg(ctx, pattern9_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); + let expr0_0 = constructor_put_in_xmm(ctx, pattern9_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern9_1)?; let expr2_0 = constructor_paddb(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::Isub => { let (pattern9_0, pattern9_1) = C::unpack_value_array_2(ctx, &pattern7_1); // Rule at src/isa/x64/lower.isle line 265. - let expr0_0 = C::put_in_reg(ctx, pattern9_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); + let expr0_0 = constructor_put_in_xmm(ctx, pattern9_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern9_1)?; let expr2_0 = constructor_psubb(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } _ => {} @@ -3625,71 +4072,71 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern9_0, pattern9_1) = C::unpack_value_array_2(ctx, &pattern7_1); - // Rule at src/isa/x64/lower.isle line 939. - let expr0_0 = C::put_in_reg(ctx, pattern9_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); + // Rule at src/isa/x64/lower.isle line 954. + let expr0_0 = constructor_put_in_xmm(ctx, pattern9_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern9_1)?; let expr2_0 = constructor_pavgw(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::UaddSat => { let (pattern9_0, pattern9_1) = C::unpack_value_array_2(ctx, &pattern7_1); // Rule at src/isa/x64/lower.isle line 149. - let expr0_0 = C::put_in_reg(ctx, pattern9_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); + let expr0_0 = constructor_put_in_xmm(ctx, pattern9_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern9_1)?; let expr2_0 = constructor_paddusw(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::SaddSat => { let (pattern9_0, pattern9_1) = C::unpack_value_array_2(ctx, &pattern7_1); // Rule at src/isa/x64/lower.isle line 137. - let expr0_0 = C::put_in_reg(ctx, pattern9_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); + let expr0_0 = constructor_put_in_xmm(ctx, pattern9_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern9_1)?; let expr2_0 = constructor_paddsw(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::UsubSat => { let (pattern9_0, pattern9_1) = C::unpack_value_array_2(ctx, &pattern7_1); // Rule at src/isa/x64/lower.isle line 318. - let expr0_0 = C::put_in_reg(ctx, pattern9_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); + let expr0_0 = constructor_put_in_xmm(ctx, pattern9_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern9_1)?; let expr2_0 = constructor_psubusw(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::SsubSat => { let (pattern9_0, pattern9_1) = C::unpack_value_array_2(ctx, &pattern7_1); // Rule at src/isa/x64/lower.isle line 306. - let expr0_0 = C::put_in_reg(ctx, pattern9_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); + let expr0_0 = constructor_put_in_xmm(ctx, pattern9_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern9_1)?; let expr2_0 = constructor_psubsw(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::Iadd => { let (pattern9_0, pattern9_1) = C::unpack_value_array_2(ctx, &pattern7_1); // Rule at src/isa/x64/lower.isle line 101. - let expr0_0 = C::put_in_reg(ctx, pattern9_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); + let expr0_0 = constructor_put_in_xmm(ctx, pattern9_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern9_1)?; let expr2_0 = constructor_paddw(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::Isub => { let (pattern9_0, pattern9_1) = C::unpack_value_array_2(ctx, &pattern7_1); // Rule at src/isa/x64/lower.isle line 270. - let expr0_0 = C::put_in_reg(ctx, pattern9_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); + let expr0_0 = constructor_put_in_xmm(ctx, pattern9_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern9_1)?; let expr2_0 = constructor_psubw(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::Imul => { @@ -3738,16 +4185,14 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option {} @@ -4000,20 +4423,20 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern9_0, pattern9_1) = C::unpack_value_array_2(ctx, &pattern7_1); // Rule at src/isa/x64/lower.isle line 275. - let expr0_0 = C::put_in_reg(ctx, pattern9_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); + let expr0_0 = constructor_put_in_xmm(ctx, pattern9_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern9_1)?; let expr2_0 = constructor_psubd(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::Imul => { @@ -4062,22 +4485,16 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option {} @@ -4312,20 +4711,20 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern9_0, pattern9_1) = C::unpack_value_array_2(ctx, &pattern7_1); // Rule at src/isa/x64/lower.isle line 280. - let expr0_0 = C::put_in_reg(ctx, pattern9_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); + let expr0_0 = constructor_put_in_xmm(ctx, pattern9_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern9_1)?; let expr2_0 = constructor_psubq(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::Imul => { @@ -4374,20 +4773,18 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option {} } @@ -4632,30 +5026,30 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 440. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); + // Rule at src/isa/x64/lower.isle line 442. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_sse_or(ctx, pattern2_0, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::Bxor => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 509. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); + // Rule at src/isa/x64/lower.isle line 513. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_sse_xor(ctx, pattern2_0, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } _ => {} @@ -4669,30 +5063,30 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1, pattern7_2) = C::unpack_value_array_3(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1314. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); - let expr1_0 = C::put_in_reg(ctx, pattern7_1); - let expr2_0 = RegMem::Reg { reg: expr0_0 }; + // Rule at src/isa/x64/lower.isle line 1331. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; + let expr1_0 = constructor_put_in_xmm(ctx, pattern7_1)?; + let expr2_0 = C::xmm_to_xmm_mem(ctx, expr0_0); let expr3_0 = constructor_sse_and(ctx, pattern2_0, expr1_0, &expr2_0)?; - let expr4_0 = C::put_in_reg_mem(ctx, pattern7_2); + let expr4_0 = constructor_put_in_xmm_mem(ctx, pattern7_2)?; let expr5_0 = constructor_sse_and_not(ctx, pattern2_0, expr0_0, &expr4_0)?; - let expr6_0 = RegMem::Reg { reg: expr3_0 }; + let expr6_0 = C::xmm_to_xmm_mem(ctx, expr3_0); let expr7_0 = constructor_sse_or(ctx, pattern2_0, expr5_0, &expr6_0)?; - let expr8_0 = C::value_reg(ctx, expr7_0); + let expr8_0 = constructor_value_xmm(ctx, expr7_0)?; return Some(expr8_0); } &Opcode::Vselect => { let (pattern7_0, pattern7_1, pattern7_2) = C::unpack_value_array_3(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1328. - let expr0_0 = C::put_in_reg_mem(ctx, pattern7_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); - let expr2_0 = C::put_in_reg(ctx, pattern7_2); + // Rule at src/isa/x64/lower.isle line 1345. + let expr0_0 = constructor_put_in_xmm_mem(ctx, pattern7_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; + let expr2_0 = constructor_put_in_xmm(ctx, pattern7_2)?; let expr3_0 = constructor_sse_blend( ctx, pattern2_0, &expr0_0, &expr1_0, expr2_0, )?; - let expr4_0 = C::value_reg(ctx, expr3_0); + let expr4_0 = constructor_value_xmm(ctx, expr3_0)?; return Some(expr4_0); } _ => {} @@ -4703,12 +5097,12 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::Bnot = &pattern5_0 { - // Rule at src/isa/x64/lower.isle line 1309. - let expr0_0 = C::put_in_reg(ctx, pattern5_1); + // Rule at src/isa/x64/lower.isle line 1326. + let expr0_0 = constructor_put_in_xmm(ctx, pattern5_1)?; let expr1_0 = constructor_vector_all_ones(ctx, pattern2_0)?; - let expr2_0 = RegMem::Reg { reg: expr1_0 }; + let expr2_0 = C::xmm_to_xmm_mem(ctx, expr1_0); let expr3_0 = constructor_sse_xor(ctx, pattern2_0, expr0_0, &expr2_0)?; - let expr4_0 = C::value_reg(ctx, expr3_0); + let expr4_0 = constructor_value_xmm(ctx, expr3_0)?; return Some(expr4_0); } } @@ -4830,7 +5224,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 539. + // Rule at src/isa/x64/lower.isle line 543. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_masked_in_imm8_reg(ctx, pattern7_1, pattern3_0); let expr2_0 = constructor_shl(ctx, pattern3_0, expr0_0, &expr1_0)?; @@ -5071,7 +5465,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 635. + // Rule at src/isa/x64/lower.isle line 642. let expr0_0 = ExtendKind::Zero; let expr1_0 = constructor_extend_to_reg(ctx, pattern7_0, pattern3_0, &expr0_0)?; @@ -5083,7 +5477,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 729. + // Rule at src/isa/x64/lower.isle line 741. let expr0_0 = ExtendKind::Sign; let expr1_0 = constructor_extend_to_reg(ctx, pattern7_0, pattern3_0, &expr0_0)?; @@ -5101,17 +5495,17 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { match &pattern5_0 { &Opcode::Ineg => { - // Rule at src/isa/x64/lower.isle line 912. - let expr0_0 = C::put_in_reg(ctx, pattern5_1); + // Rule at src/isa/x64/lower.isle line 927. + let expr0_0 = constructor_put_in_gpr(ctx, pattern5_1)?; let expr1_0 = constructor_neg(ctx, pattern3_0, expr0_0)?; - let expr2_0 = C::value_reg(ctx, expr1_0); + let expr2_0 = constructor_value_gpr(ctx, expr1_0)?; return Some(expr2_0); } &Opcode::Bnot => { - // Rule at src/isa/x64/lower.isle line 1288. - let expr0_0 = C::put_in_reg(ctx, pattern5_1); + // Rule at src/isa/x64/lower.isle line 1305. + let expr0_0 = constructor_put_in_gpr(ctx, pattern5_1)?; let expr1_0 = constructor_not(ctx, pattern3_0, expr0_0)?; - let expr2_0 = C::value_reg(ctx, expr1_0); + let expr2_0 = constructor_value_gpr(ctx, expr1_0)?; return Some(expr2_0); } _ => {} @@ -5155,7 +5549,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option( ctx: &mut C, arg0: Type, - arg1: Reg, - arg2: &RegMem, -) -> Option { + arg1: Xmm, + arg2: &XmmMem, +) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32X4 { let pattern2_0 = arg1; @@ -5326,28 +5720,28 @@ pub fn constructor_sse_and( pub fn constructor_sse_or( ctx: &mut C, arg0: Type, - arg1: Reg, - arg2: &RegMem, -) -> Option { + arg1: Xmm, + arg2: &XmmMem, +) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32X4 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/lower.isle line 436. + // Rule at src/isa/x64/lower.isle line 438. let expr0_0 = constructor_orps(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if pattern0_0 == F64X2 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/lower.isle line 437. + // Rule at src/isa/x64/lower.isle line 439. let expr0_0 = constructor_orpd(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/lower.isle line 438. + // Rule at src/isa/x64/lower.isle line 440. let expr0_0 = constructor_por(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } @@ -5362,7 +5756,7 @@ pub fn constructor_or_i128( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/lower.isle line 447. + // Rule at src/isa/x64/lower.isle line 451. let expr0_0: usize = 0; let expr1_0 = C::value_regs_get(ctx, pattern0_0, expr0_0); let expr2_0: usize = 1; @@ -5389,7 +5783,7 @@ pub fn constructor_shl_i128( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/lower.isle line 545. + // Rule at src/isa/x64/lower.isle line 549. let expr0_0: usize = 0; let expr1_0 = C::value_regs_get(ctx, pattern0_0, expr0_0); let expr2_0: usize = 1; @@ -5448,12 +5842,12 @@ pub fn constructor_ishl_i8x16_mask( let pattern0_0 = arg0; match pattern0_0 { &RegMemImm::Imm { simm32: pattern1_0 } => { - // Rule at src/isa/x64/lower.isle line 600. + // Rule at src/isa/x64/lower.isle line 604. let expr0_0 = C::ishl_i8x16_mask_for_const(ctx, pattern1_0); return Some(expr0_0); } &RegMemImm::Reg { reg: pattern1_0 } => { - // Rule at src/isa/x64/lower.isle line 609. + // Rule at src/isa/x64/lower.isle line 613. let expr0_0 = C::ishl_i8x16_mask_table(ctx); let expr1_0 = constructor_lea(ctx, &expr0_0)?; let expr2_0: Type = I64; @@ -5461,15 +5855,16 @@ pub fn constructor_ishl_i8x16_mask( let expr4_0 = Imm8Reg::Imm8 { imm: expr3_0 }; let expr5_0 = constructor_shl(ctx, expr2_0, pattern1_0, &expr4_0)?; let expr6_0: u32 = 0; - let expr7_0: u8 = 0; - let expr8_0 = C::amode_imm_reg_reg_shift(ctx, expr6_0, expr1_0, expr5_0, expr7_0); - let expr9_0 = C::amode_to_synthetic_amode(ctx, &expr8_0); - return Some(expr9_0); + let expr7_0 = C::gpr_new(ctx, expr5_0); + let expr8_0: u8 = 0; + let expr9_0 = C::amode_imm_reg_reg_shift(ctx, expr6_0, expr1_0, expr7_0, expr8_0); + let expr10_0 = C::amode_to_synthetic_amode(ctx, &expr9_0); + return Some(expr10_0); } &RegMemImm::Mem { addr: ref pattern1_0, } => { - // Rule at src/isa/x64/lower.isle line 617. + // Rule at src/isa/x64/lower.isle line 621. let expr0_0: Type = I64; let expr1_0 = ExtKind::None; let expr2_0 = constructor_x64_load(ctx, expr0_0, &pattern1_0, &expr1_0)?; @@ -5490,7 +5885,7 @@ pub fn constructor_shr_i128( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/lower.isle line 642. + // Rule at src/isa/x64/lower.isle line 649. let expr0_0: usize = 0; let expr1_0 = C::value_regs_get(ctx, pattern0_0, expr0_0); let expr2_0: usize = 1; @@ -5552,12 +5947,12 @@ pub fn constructor_ushr_i8x16_mask( let pattern0_0 = arg0; match pattern0_0 { &RegMemImm::Imm { simm32: pattern1_0 } => { - // Rule at src/isa/x64/lower.isle line 694. + // Rule at src/isa/x64/lower.isle line 703. let expr0_0 = C::ushr_i8x16_mask_for_const(ctx, pattern1_0); return Some(expr0_0); } &RegMemImm::Reg { reg: pattern1_0 } => { - // Rule at src/isa/x64/lower.isle line 703. + // Rule at src/isa/x64/lower.isle line 712. let expr0_0 = C::ushr_i8x16_mask_table(ctx); let expr1_0 = constructor_lea(ctx, &expr0_0)?; let expr2_0: Type = I64; @@ -5565,15 +5960,16 @@ pub fn constructor_ushr_i8x16_mask( let expr4_0 = Imm8Reg::Imm8 { imm: expr3_0 }; let expr5_0 = constructor_shl(ctx, expr2_0, pattern1_0, &expr4_0)?; let expr6_0: u32 = 0; - let expr7_0: u8 = 0; - let expr8_0 = C::amode_imm_reg_reg_shift(ctx, expr6_0, expr1_0, expr5_0, expr7_0); - let expr9_0 = C::amode_to_synthetic_amode(ctx, &expr8_0); - return Some(expr9_0); + let expr7_0 = C::gpr_new(ctx, expr5_0); + let expr8_0: u8 = 0; + let expr9_0 = C::amode_imm_reg_reg_shift(ctx, expr6_0, expr1_0, expr7_0, expr8_0); + let expr10_0 = C::amode_to_synthetic_amode(ctx, &expr9_0); + return Some(expr10_0); } &RegMemImm::Mem { addr: ref pattern1_0, } => { - // Rule at src/isa/x64/lower.isle line 711. + // Rule at src/isa/x64/lower.isle line 720. let expr0_0: Type = I64; let expr1_0 = ExtKind::None; let expr2_0 = constructor_x64_load(ctx, expr0_0, &pattern1_0, &expr1_0)?; @@ -5594,7 +5990,7 @@ pub fn constructor_sar_i128( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/lower.isle line 736. + // Rule at src/isa/x64/lower.isle line 748. let expr0_0: usize = 0; let expr1_0 = C::value_regs_get(ctx, pattern0_0, expr0_0); let expr2_0: usize = 1; @@ -5654,35 +6050,36 @@ pub fn constructor_sshr_i8x16_bigger_shift( ctx: &mut C, arg0: Type, arg1: &RegMemImm, -) -> Option { +) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; match pattern1_0 { &RegMemImm::Imm { simm32: pattern2_0 } => { - // Rule at src/isa/x64/lower.isle line 795. + // Rule at src/isa/x64/lower.isle line 807. let expr0_0: u32 = 8; let expr1_0 = C::u32_add(ctx, pattern2_0, expr0_0); let expr2_0 = RegMemImm::Imm { simm32: expr1_0 }; - return Some(expr2_0); + let expr3_0 = C::xmm_mem_imm_new(ctx, &expr2_0); + return Some(expr3_0); } &RegMemImm::Reg { reg: pattern2_0 } => { - // Rule at src/isa/x64/lower.isle line 797. + // Rule at src/isa/x64/lower.isle line 809. let expr0_0: u32 = 8; let expr1_0 = RegMemImm::Imm { simm32: expr0_0 }; let expr2_0 = constructor_add(ctx, pattern0_0, pattern2_0, &expr1_0)?; let expr3_0 = RegMemImm::Reg { reg: expr2_0 }; - let expr4_0 = constructor_reg_mem_imm_to_xmm(ctx, &expr3_0)?; + let expr4_0 = constructor_mov_rmi_to_xmm(ctx, &expr3_0)?; return Some(expr4_0); } &RegMemImm::Mem { addr: ref pattern2_0, } => { - // Rule at src/isa/x64/lower.isle line 799. + // Rule at src/isa/x64/lower.isle line 811. let expr0_0: u64 = 8; let expr1_0 = constructor_imm(ctx, pattern0_0, expr0_0)?; let expr2_0 = constructor_add(ctx, pattern0_0, expr1_0, pattern1_0)?; let expr3_0 = RegMemImm::Reg { reg: expr2_0 }; - let expr4_0 = constructor_reg_mem_imm_to_xmm(ctx, &expr3_0)?; + let expr4_0 = constructor_mov_rmi_to_xmm(ctx, &expr3_0)?; return Some(expr4_0); } _ => {} @@ -5694,28 +6091,28 @@ pub fn constructor_sshr_i8x16_bigger_shift( pub fn constructor_sse_and_not( ctx: &mut C, arg0: Type, - arg1: Reg, - arg2: &RegMem, -) -> Option { + arg1: Xmm, + arg2: &XmmMem, +) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32X4 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/lower.isle line 1229. + // Rule at src/isa/x64/lower.isle line 1244. let expr0_0 = constructor_andnps(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if pattern0_0 == F64X2 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/lower.isle line 1230. + // Rule at src/isa/x64/lower.isle line 1245. let expr0_0 = constructor_andnpd(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/lower.isle line 1231. + // Rule at src/isa/x64/lower.isle line 1246. let expr0_0 = constructor_pandn(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } @@ -5725,71 +6122,78 @@ pub fn constructor_sse_and_not( // Generated as internal constructor for term i128_not. pub fn constructor_i128_not(ctx: &mut C, arg0: Value) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/lower.isle line 1294. + // Rule at src/isa/x64/lower.isle line 1311. let expr0_0 = C::put_in_regs(ctx, pattern0_0); let expr1_0: usize = 0; let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); - let expr3_0: usize = 1; - let expr4_0 = C::value_regs_get(ctx, expr0_0, expr3_0); - let expr5_0: Type = I64; - let expr6_0 = constructor_not(ctx, expr5_0, expr2_0)?; + let expr3_0 = C::gpr_new(ctx, expr2_0); + let expr4_0: usize = 1; + let expr5_0 = C::value_regs_get(ctx, expr0_0, expr4_0); + let expr6_0 = C::gpr_new(ctx, expr5_0); let expr7_0: Type = I64; - let expr8_0 = constructor_not(ctx, expr7_0, expr4_0)?; - let expr9_0 = C::value_regs(ctx, expr6_0, expr8_0); - return Some(expr9_0); + let expr8_0 = constructor_not(ctx, expr7_0, expr3_0)?; + let expr9_0: Type = I64; + let expr10_0 = constructor_not(ctx, expr9_0, expr6_0)?; + let expr11_0 = constructor_value_gprs(ctx, expr8_0, expr10_0)?; + return Some(expr11_0); } // Generated as internal constructor for term vec_insert_lane. pub fn constructor_vec_insert_lane( ctx: &mut C, arg0: Type, - arg1: Reg, + arg1: Xmm, arg2: &RegMem, arg3: u8, -) -> Option { +) -> Option { let pattern0_0 = arg0; if pattern0_0 == I8X16 { let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/lower.isle line 1348. - let expr0_0 = constructor_pinsrb(ctx, pattern2_0, pattern3_0, pattern4_0)?; - return Some(expr0_0); + // Rule at src/isa/x64/lower.isle line 1365. + let expr0_0 = C::gpr_mem_new(ctx, pattern3_0); + let expr1_0 = constructor_pinsrb(ctx, pattern2_0, &expr0_0, pattern4_0)?; + return Some(expr1_0); } if pattern0_0 == I16X8 { let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/lower.isle line 1351. - let expr0_0 = constructor_pinsrw(ctx, pattern2_0, pattern3_0, pattern4_0)?; - return Some(expr0_0); + // Rule at src/isa/x64/lower.isle line 1369. + let expr0_0 = C::gpr_mem_new(ctx, pattern3_0); + let expr1_0 = constructor_pinsrw(ctx, pattern2_0, &expr0_0, pattern4_0)?; + return Some(expr1_0); } if pattern0_0 == I32X4 { let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/lower.isle line 1354. - let expr0_0 = OperandSize::Size32; - let expr1_0 = constructor_pinsrd(ctx, pattern2_0, pattern3_0, pattern4_0, &expr0_0)?; - return Some(expr1_0); + // Rule at src/isa/x64/lower.isle line 1373. + let expr0_0 = C::gpr_mem_new(ctx, pattern3_0); + let expr1_0 = OperandSize::Size32; + let expr2_0 = constructor_pinsrd(ctx, pattern2_0, &expr0_0, pattern4_0, &expr1_0)?; + return Some(expr2_0); } if pattern0_0 == I64X2 { let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/lower.isle line 1357. - let expr0_0 = OperandSize::Size64; - let expr1_0 = constructor_pinsrd(ctx, pattern2_0, pattern3_0, pattern4_0, &expr0_0)?; - return Some(expr1_0); + // Rule at src/isa/x64/lower.isle line 1377. + let expr0_0 = C::gpr_mem_new(ctx, pattern3_0); + let expr1_0 = OperandSize::Size64; + let expr2_0 = constructor_pinsrd(ctx, pattern2_0, &expr0_0, pattern4_0, &expr1_0)?; + return Some(expr2_0); } if pattern0_0 == F32X4 { let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/lower.isle line 1360. - let expr0_0 = C::sse_insertps_lane_imm(ctx, pattern4_0); - let expr1_0 = constructor_insertps(ctx, pattern2_0, pattern3_0, expr0_0)?; - return Some(expr1_0); + // Rule at src/isa/x64/lower.isle line 1381. + let expr0_0 = C::xmm_mem_new(ctx, pattern3_0); + let expr1_0 = C::sse_insertps_lane_imm(ctx, pattern4_0); + let expr2_0 = constructor_insertps(ctx, pattern2_0, &expr0_0, expr1_0)?; + return Some(expr2_0); } if pattern0_0 == F64X2 { let pattern2_0 = arg1; @@ -5797,25 +6201,28 @@ pub fn constructor_vec_insert_lane( if let &RegMem::Reg { reg: pattern4_0 } = pattern3_0 { let pattern5_0 = arg3; if pattern5_0 == 0 { - // Rule at src/isa/x64/lower.isle line 1381. + // Rule at src/isa/x64/lower.isle line 1403. let expr0_0 = RegMem::Reg { reg: pattern4_0 }; - let expr1_0 = constructor_movsd(ctx, pattern2_0, &expr0_0)?; - return Some(expr1_0); + let expr1_0 = C::xmm_mem_new(ctx, &expr0_0); + let expr2_0 = constructor_movsd(ctx, pattern2_0, &expr1_0)?; + return Some(expr2_0); } } let pattern4_0 = arg3; if pattern4_0 == 0 { - // Rule at src/isa/x64/lower.isle line 1382. + // Rule at src/isa/x64/lower.isle line 1405. let expr0_0 = SseOpcode::Movsd; - let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern3_0)?; - let expr2_0 = RegMem::Reg { reg: expr1_0 }; - let expr3_0 = constructor_movsd(ctx, pattern2_0, &expr2_0)?; - return Some(expr3_0); + let expr1_0 = C::xmm_mem_new(ctx, pattern3_0); + let expr2_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, &expr1_0)?; + let expr3_0 = C::xmm_to_xmm_mem(ctx, expr2_0); + let expr4_0 = constructor_movsd(ctx, pattern2_0, &expr3_0)?; + return Some(expr4_0); } if pattern4_0 == 1 { - // Rule at src/isa/x64/lower.isle line 1390. - let expr0_0 = constructor_movlhps(ctx, pattern2_0, pattern3_0)?; - return Some(expr0_0); + // Rule at src/isa/x64/lower.isle line 1414. + let expr0_0 = C::xmm_mem_new(ctx, pattern3_0); + let expr1_0 = constructor_movlhps(ctx, pattern2_0, &expr0_0)?; + return Some(expr1_0); } } return None; diff --git a/cranelift/codegen/src/machinst/isle.rs b/cranelift/codegen/src/machinst/isle.rs index 1869ba6762a8..b54ba51a5366 100644 --- a/cranelift/codegen/src/machinst/isle.rs +++ b/cranelift/codegen/src/machinst/isle.rs @@ -1,14 +1,25 @@ use crate::ir::{Inst, Value}; use crate::machinst::{get_output_reg, InsnOutput, LowerCtx, MachInst, RegRenamer}; +use alloc::boxed::Box; +use alloc::vec::Vec; use regalloc::{Reg, Writable}; use smallvec::SmallVec; +pub use super::MachLabel; +pub use crate::ir::ExternalName; +pub use crate::isa::unwind::UnwindInst; + pub type Unit = (); pub type ValueSlice<'a> = &'a [Value]; pub type ValueArray2 = [Value; 2]; pub type ValueArray3 = [Value; 3]; pub type WritableReg = Writable; +pub type OptionWritableReg = Option; +pub type VecReg = Vec; +pub type VecWritableReg = Vec; pub type ValueRegs = crate::machinst::ValueRegs; +pub type VecMachLabel = Vec; +pub type BoxExternalName = Box; /// Helper macro to define methods in `prelude.isle` within `impl Context for /// ...` for each backend. These methods are shared amongst all backends. diff --git a/cranelift/codegen/src/prelude.isle b/cranelift/codegen/src/prelude.isle index 5b7d161d7ea9..78f0395f42aa 100644 --- a/cranelift/codegen/src/prelude.isle +++ b/cranelift/codegen/src/prelude.isle @@ -48,6 +48,9 @@ (type Reg (primitive Reg)) (type WritableReg (primitive WritableReg)) +(type OptionWritableReg (primitive OptionWritableReg)) +(type VecReg extern (enum)) +(type VecWritableReg extern (enum)) ;; Construct a `ValueRegs` of one register. (decl value_reg (Reg) ValueRegs) @@ -106,6 +109,15 @@ (let ((regs ValueRegs (put_in_regs val))) (value_regs_get regs 0))) +;;;; Common Mach Types ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(type MachLabel (primitive MachLabel)) +(type VecMachLabel extern (enum)) +(type ValueLabel (primitive ValueLabel)) +(type UnwindInst (primitive UnwindInst)) +(type ExternalName (primitive ExternalName)) +(type BoxExternalName (primitive BoxExternalName)) + ;;;; Primitive Type Conversions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (decl u8_as_u64 (u8) u64) @@ -368,4 +380,3 @@ (decl avoid_div_traps () Type) (extern extractor avoid_div_traps avoid_div_traps) -