diff --git a/Cargo.lock b/Cargo.lock index 488e4f7b3aa7..a2eda03a5480 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2350,9 +2350,9 @@ dependencies = [ [[package]] name = "regalloc" -version = "0.0.33" +version = "0.0.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d808cff91dfca7b239d40b972ba628add94892b1d9e19a842aedc5cfae8ab1a" +checksum = "62446b1d3ebf980bdc68837700af1d77b37bc430e524bf95319c6eada2a4cc02" dependencies = [ "log", "rustc-hash", diff --git a/cranelift/codegen/Cargo.toml b/cranelift/codegen/Cargo.toml index 3e20edcc8d56..27979797bf9d 100644 --- a/cranelift/codegen/Cargo.toml +++ b/cranelift/codegen/Cargo.toml @@ -23,7 +23,7 @@ serde = { version = "1.0.94", features = ["derive"], optional = true } bincode = { version = "1.2.1", optional = true } gimli = { version = "0.26.0", default-features = false, features = ["write"], optional = true } smallvec = { version = "1.6.1" } -regalloc = { version = "0.0.33" } +regalloc = "0.0.34" souper-ir = { version = "2.1.0", optional = true } # It is a goal of the cranelift-codegen crate to have minimal external dependencies. # Please don't add any unless they are essential to the task of creating binary diff --git a/cranelift/codegen/build.rs b/cranelift/codegen/build.rs index cb704cb0abbb..1b95bf02a834 100644 --- a/cranelift/codegen/build.rs +++ b/cranelift/codegen/build.rs @@ -406,6 +406,8 @@ fn rebuild_isle( ) -> Result<(), Box> { use cranelift_isle as isle; + println!("Rebuilding {}", compilation.output.display()); + // First, remove the manifest, if any; we will recreate it // below if the compilation is successful. Ignore error if no // manifest was present. diff --git a/cranelift/codegen/src/isa/aarch64/inst.isle b/cranelift/codegen/src/isa/aarch64/inst.isle index 839b16f7bfa7..c64922858736 100644 --- a/cranelift/codegen/src/isa/aarch64/inst.isle +++ b/cranelift/codegen/src/isa/aarch64/inst.isle @@ -871,15 +871,10 @@ (type BoxCallInfo (primitive BoxCallInfo)) (type BoxCallIndInfo (primitive BoxCallIndInfo)) -(type VecMachLabel (primitive VecMachLabel)) (type CondBrKind (primitive CondBrKind)) (type BranchTarget (primitive BranchTarget)) (type BoxJTSequenceInfo (primitive BoxJTSequenceInfo)) -(type BoxExternalName (primitive BoxExternalName)) (type CodeOffset (primitive CodeOffset)) -(type ExternalName (primitive ExternalName)) -(type ValueLabel (primitive ValueLabel)) -(type UnwindInst (primitive UnwindInst)) (type ExtendOp extern (enum diff --git a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest index 2b6effae74bd..831df6d3cc36 100644 --- a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest +++ b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest @@ -1,4 +1,4 @@ src/clif.isle 9ea75a6f790b5c03 -src/prelude.isle 51d2aef2566c1c96 -src/isa/aarch64/inst.isle f946561093de4ff5 +src/prelude.isle 3d00cfa50c4d52d9 +src/isa/aarch64/inst.isle 13b123492c3c859d src/isa/aarch64/lower.isle 2d2e1e076a0c8a23 diff --git a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs index 231c90bbc184..0f13e498acb0 100644 --- a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs +++ b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs @@ -95,19 +95,19 @@ pub trait Context { fn rotr_opposite_amount(&mut self, arg0: Type, arg1: ImmShift) -> ImmShift; } -/// Internal type SideEffectNoResult: defined at src/prelude.isle line 282. +/// Internal type SideEffectNoResult: defined at src/prelude.isle line 294. #[derive(Clone, Debug)] pub enum SideEffectNoResult { Inst { inst: MInst }, } -/// Internal type ProducesFlags: defined at src/prelude.isle line 295. +/// Internal type ProducesFlags: defined at src/prelude.isle line 307. #[derive(Clone, Debug)] pub enum ProducesFlags { ProducesFlags { inst: MInst, result: Reg }, } -/// Internal type ConsumesFlags: defined at src/prelude.isle line 298. +/// Internal type ConsumesFlags: defined at src/prelude.isle line 310. #[derive(Clone, Debug)] pub enum ConsumesFlags { ConsumesFlags { inst: MInst, result: Reg }, @@ -682,7 +682,7 @@ pub enum MInst { } /// Internal type ALUOp: defined at src/isa/aarch64/inst.isle line 789. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum ALUOp { Add32, Add64, @@ -729,7 +729,7 @@ pub enum ALUOp { } /// Internal type ALUOp3: defined at src/isa/aarch64/inst.isle line 850. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum ALUOp3 { MAdd32, MAdd64, @@ -737,8 +737,8 @@ pub enum ALUOp3 { MSub64, } -/// Internal type BitOp: defined at src/isa/aarch64/inst.isle line 898. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type BitOp: defined at src/isa/aarch64/inst.isle line 893. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum BitOp { RBit32, RBit64, @@ -748,8 +748,8 @@ pub enum BitOp { Cls64, } -/// Internal type FPUOp1: defined at src/isa/aarch64/inst.isle line 964. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type FPUOp1: defined at src/isa/aarch64/inst.isle line 959. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum FPUOp1 { Abs32, Abs64, @@ -761,8 +761,8 @@ pub enum FPUOp1 { Cvt64To32, } -/// Internal type FPUOp2: defined at src/isa/aarch64/inst.isle line 977. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type FPUOp2: defined at src/isa/aarch64/inst.isle line 972. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum FPUOp2 { Add32, Add64, @@ -782,15 +782,15 @@ pub enum FPUOp2 { Uqsub64, } -/// Internal type FPUOp3: defined at src/isa/aarch64/inst.isle line 1002. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type FPUOp3: defined at src/isa/aarch64/inst.isle line 997. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum FPUOp3 { MAdd32, MAdd64, } -/// Internal type FpuToIntOp: defined at src/isa/aarch64/inst.isle line 1009. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type FpuToIntOp: defined at src/isa/aarch64/inst.isle line 1004. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum FpuToIntOp { F32ToU32, F32ToI32, @@ -802,8 +802,8 @@ pub enum FpuToIntOp { F64ToI64, } -/// Internal type IntToFpuOp: defined at src/isa/aarch64/inst.isle line 1022. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type IntToFpuOp: defined at src/isa/aarch64/inst.isle line 1017. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum IntToFpuOp { U32ToF32, I32ToF32, @@ -815,8 +815,8 @@ pub enum IntToFpuOp { I64ToF64, } -/// Internal type FpuRoundMode: defined at src/isa/aarch64/inst.isle line 1036. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type FpuRoundMode: defined at src/isa/aarch64/inst.isle line 1031. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum FpuRoundMode { Minus32, Minus64, @@ -828,8 +828,8 @@ pub enum FpuRoundMode { Nearest64, } -/// Internal type VecExtendOp: defined at src/isa/aarch64/inst.isle line 1049. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type VecExtendOp: defined at src/isa/aarch64/inst.isle line 1044. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum VecExtendOp { Sxtl8, Sxtl16, @@ -839,8 +839,8 @@ pub enum VecExtendOp { Uxtl32, } -/// Internal type VecALUOp: defined at src/isa/aarch64/inst.isle line 1066. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type VecALUOp: defined at src/isa/aarch64/inst.isle line 1061. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum VecALUOp { Sqadd, Uqadd, @@ -881,8 +881,8 @@ pub enum VecALUOp { Sqrdmulh, } -/// Internal type VecMisc2: defined at src/isa/aarch64/inst.isle line 1145. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type VecMisc2: defined at src/isa/aarch64/inst.isle line 1140. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum VecMisc2 { Not, Neg, @@ -903,8 +903,8 @@ pub enum VecMisc2 { Cmeq0, } -/// Internal type VecRRLongOp: defined at src/isa/aarch64/inst.isle line 1184. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type VecRRLongOp: defined at src/isa/aarch64/inst.isle line 1179. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum VecRRLongOp { Fcvtl16, Fcvtl32, @@ -913,8 +913,8 @@ pub enum VecRRLongOp { Shll32, } -/// Internal type VecRRNarrowOp: defined at src/isa/aarch64/inst.isle line 1199. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type VecRRNarrowOp: defined at src/isa/aarch64/inst.isle line 1194. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum VecRRNarrowOp { Xtn16, Xtn32, @@ -932,8 +932,8 @@ pub enum VecRRNarrowOp { Fcvtn64, } -/// Internal type VecRRRLongOp: defined at src/isa/aarch64/inst.isle line 1231. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type VecRRRLongOp: defined at src/isa/aarch64/inst.isle line 1226. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum VecRRRLongOp { Smull8, Smull16, @@ -946,14 +946,14 @@ pub enum VecRRRLongOp { Umlal32, } -/// Internal type VecPairOp: defined at src/isa/aarch64/inst.isle line 1248. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type VecPairOp: defined at src/isa/aarch64/inst.isle line 1243. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum VecPairOp { Addp, } -/// Internal type VecRRPairLongOp: defined at src/isa/aarch64/inst.isle line 1256. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type VecRRPairLongOp: defined at src/isa/aarch64/inst.isle line 1251. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum VecRRPairLongOp { Saddlp8, Saddlp16, @@ -961,23 +961,23 @@ pub enum VecRRPairLongOp { Uaddlp16, } -/// Internal type VecLanesOp: defined at src/isa/aarch64/inst.isle line 1267. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type VecLanesOp: defined at src/isa/aarch64/inst.isle line 1262. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum VecLanesOp { Addv, Uminv, } -/// Internal type VecShiftImmOp: defined at src/isa/aarch64/inst.isle line 1276. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type VecShiftImmOp: defined at src/isa/aarch64/inst.isle line 1271. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum VecShiftImmOp { Shl, Ushr, Sshr, } -/// Internal type AtomicRMWOp: defined at src/isa/aarch64/inst.isle line 1287. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type AtomicRMWOp: defined at src/isa/aarch64/inst.isle line 1282. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum AtomicRMWOp { Add, Clr, @@ -992,7 +992,7 @@ pub enum AtomicRMWOp { // Generated as internal constructor for term temp_reg. pub fn constructor_temp_reg(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; - // Rule at src/prelude.isle line 70. + // Rule at src/prelude.isle line 73. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); return Some(expr1_0); @@ -1001,7 +1001,7 @@ pub fn constructor_temp_reg(ctx: &mut C, arg0: Type) -> Option // Generated as internal constructor for term lo_reg. pub fn constructor_lo_reg(ctx: &mut C, arg0: Value) -> Option { let pattern0_0 = arg0; - // Rule at src/prelude.isle line 105. + // Rule at src/prelude.isle line 108. let expr0_0 = C::put_in_regs(ctx, pattern0_0); let expr1_0: usize = 0; let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); @@ -1018,7 +1018,7 @@ pub fn constructor_value_regs_none( inst: ref pattern1_0, } = pattern0_0 { - // Rule at src/prelude.isle line 287. + // Rule at src/prelude.isle line 299. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::value_regs_invalid(ctx); return Some(expr1_0); @@ -1044,7 +1044,7 @@ pub fn constructor_with_flags( result: pattern3_1, } = pattern2_0 { - // Rule at src/prelude.isle line 308. + // Rule at src/prelude.isle line 320. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern3_0); let expr2_0 = C::value_regs(ctx, pattern1_1, pattern3_1); @@ -1072,7 +1072,7 @@ pub fn constructor_with_flags_1( result: pattern3_1, } = pattern2_0 { - // Rule at src/prelude.isle line 316. + // Rule at src/prelude.isle line 328. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern3_0); return Some(pattern3_1); @@ -1106,7 +1106,7 @@ pub fn constructor_with_flags_2( result: pattern5_1, } = pattern4_0 { - // Rule at src/prelude.isle line 326. + // Rule at src/prelude.isle line 338. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern5_0); let expr2_0 = C::emit(ctx, &pattern3_0); @@ -1124,28 +1124,28 @@ pub fn constructor_vector_size(ctx: &mut C, arg0: Type) -> Option( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1390. + // Rule at src/isa/aarch64/inst.isle line 1385. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::MovZ { @@ -1183,7 +1183,7 @@ pub fn constructor_movn( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1397. + // Rule at src/isa/aarch64/inst.isle line 1392. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::MovN { @@ -1206,7 +1206,7 @@ pub fn constructor_alu_rr_imm_logic( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1404. + // Rule at src/isa/aarch64/inst.isle line 1399. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::AluRRImmLogic { @@ -1230,7 +1230,7 @@ pub fn constructor_alu_rr_imm_shift( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1411. + // Rule at src/isa/aarch64/inst.isle line 1406. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::AluRRImmShift { @@ -1254,7 +1254,7 @@ pub fn constructor_alu_rrr( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1418. + // Rule at src/isa/aarch64/inst.isle line 1413. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::AluRRR { @@ -1280,7 +1280,7 @@ pub fn constructor_vec_rrr( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1425. + // Rule at src/isa/aarch64/inst.isle line 1420. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::VecRRR { @@ -1305,7 +1305,7 @@ pub fn constructor_vec_lanes( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1432. + // Rule at src/isa/aarch64/inst.isle line 1427. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::VecLanes { @@ -1323,7 +1323,7 @@ pub fn constructor_vec_lanes( pub fn constructor_vec_dup(ctx: &mut C, arg0: Reg, arg1: &VectorSize) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1439. + // Rule at src/isa/aarch64/inst.isle line 1434. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::VecDup { @@ -1346,7 +1346,7 @@ pub fn constructor_alu_rr_imm12( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1446. + // Rule at src/isa/aarch64/inst.isle line 1441. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::AluRRImm12 { @@ -1372,7 +1372,7 @@ pub fn constructor_alu_rrr_shift( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1453. + // Rule at src/isa/aarch64/inst.isle line 1448. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::AluRRRShift { @@ -1399,7 +1399,7 @@ pub fn constructor_alu_rrr_extend( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1460. + // Rule at src/isa/aarch64/inst.isle line 1455. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::AluRRRExtend { @@ -1424,7 +1424,7 @@ pub fn constructor_alu_rr_extend_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1468. + // Rule at src/isa/aarch64/inst.isle line 1463. let expr0_0 = C::put_extended_in_reg(ctx, pattern2_0); let expr1_0 = C::get_extended_op(ctx, pattern2_0); let expr2_0 = constructor_alu_rrr_extend(ctx, pattern0_0, pattern1_0, expr0_0, &expr1_0)?; @@ -1443,7 +1443,7 @@ pub fn constructor_alu_rrrr( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1475. + // Rule at src/isa/aarch64/inst.isle line 1470. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::AluRRRR { @@ -1462,7 +1462,7 @@ pub fn constructor_alu_rrrr( pub fn constructor_bit_rr(ctx: &mut C, arg0: &BitOp, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1482. + // Rule at src/isa/aarch64/inst.isle line 1477. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::BitRR { @@ -1483,7 +1483,7 @@ pub fn constructor_add64_with_flags( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1489. + // Rule at src/isa/aarch64/inst.isle line 1484. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = ALUOp::AddS64; @@ -1505,7 +1505,7 @@ pub fn constructor_add64_with_flags( pub fn constructor_adc64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1496. + // Rule at src/isa/aarch64/inst.isle line 1491. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = ALUOp::Adc64; @@ -1531,7 +1531,7 @@ pub fn constructor_sub64_with_flags( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1503. + // Rule at src/isa/aarch64/inst.isle line 1498. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = ALUOp::SubS64; @@ -1557,7 +1557,7 @@ pub fn constructor_cmp64_imm( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1509. + // Rule at src/isa/aarch64/inst.isle line 1504. let expr0_0 = ALUOp::SubS64; let expr1_0 = C::writable_zero_reg(ctx); let expr2_0 = MInst::AluRRImm12 { @@ -1578,7 +1578,7 @@ pub fn constructor_cmp64_imm( pub fn constructor_sbc64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1515. + // Rule at src/isa/aarch64/inst.isle line 1510. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = ALUOp::Sbc64; @@ -1606,7 +1606,7 @@ pub fn constructor_vec_misc( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1522. + // Rule at src/isa/aarch64/inst.isle line 1517. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::VecMisc { @@ -1632,7 +1632,7 @@ pub fn constructor_vec_rrr_long( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1529. + // Rule at src/isa/aarch64/inst.isle line 1524. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::VecRRRLong { @@ -1661,7 +1661,7 @@ pub fn constructor_vec_rrrr_long( let pattern2_0 = arg2; let pattern3_0 = arg3; let pattern4_0 = arg4; - // Rule at src/isa/aarch64/inst.isle line 1539. + // Rule at src/isa/aarch64/inst.isle line 1534. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::FpuMove128 { @@ -1691,7 +1691,7 @@ pub fn constructor_vec_rr_narrow( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1547. + // Rule at src/isa/aarch64/inst.isle line 1542. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::VecRRNarrow { @@ -1715,7 +1715,7 @@ pub fn constructor_vec_rr_long( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1554. + // Rule at src/isa/aarch64/inst.isle line 1549. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::VecRRLong { @@ -1737,7 +1737,7 @@ pub fn constructor_mov_to_fpu( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1561. + // Rule at src/isa/aarch64/inst.isle line 1556. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::MovToFpu { @@ -1762,7 +1762,7 @@ pub fn constructor_mov_to_vec( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1568. + // Rule at src/isa/aarch64/inst.isle line 1563. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::FpuMove128 { @@ -1791,7 +1791,7 @@ pub fn constructor_mov_from_vec( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1576. + // Rule at src/isa/aarch64/inst.isle line 1571. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::MovFromVec { @@ -1817,7 +1817,7 @@ pub fn constructor_mov_from_vec_signed( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1583. + // Rule at src/isa/aarch64/inst.isle line 1578. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::MovFromVecSigned { @@ -1844,7 +1844,7 @@ pub fn constructor_extend( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1590. + // Rule at src/isa/aarch64/inst.isle line 1585. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::Extend { @@ -1863,7 +1863,7 @@ pub fn constructor_extend( pub fn constructor_load_acquire(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1597. + // Rule at src/isa/aarch64/inst.isle line 1592. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::LoadAcquire { @@ -1884,7 +1884,7 @@ pub fn constructor_tst64_imm( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1607. + // Rule at src/isa/aarch64/inst.isle line 1602. let expr0_0 = ALUOp::AndS64; let expr1_0 = C::writable_zero_reg(ctx); let expr2_0 = MInst::AluRRImmLogic { @@ -1911,7 +1911,7 @@ pub fn constructor_csel( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1620. + // Rule at src/isa/aarch64/inst.isle line 1615. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::CSel { @@ -1934,14 +1934,14 @@ pub fn constructor_add(ctx: &mut C, arg0: Type, arg1: Reg, arg2: Reg if pattern0_0 == I64 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1629. + // Rule at src/isa/aarch64/inst.isle line 1624. let expr0_0 = constructor_add64(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if let Some(pattern1_0) = C::fits_in_32(ctx, pattern0_0) { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1628. + // Rule at src/isa/aarch64/inst.isle line 1623. let expr0_0 = constructor_add32(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } @@ -1952,7 +1952,7 @@ pub fn constructor_add(ctx: &mut C, arg0: Type, arg1: Reg, arg2: Reg pub fn constructor_add32(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1632. + // Rule at src/isa/aarch64/inst.isle line 1627. let expr0_0 = ALUOp::Add32; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -1962,7 +1962,7 @@ pub fn constructor_add32(ctx: &mut C, arg0: Reg, arg1: Reg) -> Optio pub fn constructor_add64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1635. + // Rule at src/isa/aarch64/inst.isle line 1630. let expr0_0 = ALUOp::Add64; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -1979,14 +1979,14 @@ pub fn constructor_add_imm( if pattern0_0 == I64 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1639. + // Rule at src/isa/aarch64/inst.isle line 1634. let expr0_0 = constructor_add64_imm(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if let Some(pattern1_0) = C::fits_in_32(ctx, pattern0_0) { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1638. + // Rule at src/isa/aarch64/inst.isle line 1633. let expr0_0 = constructor_add32_imm(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } @@ -1997,7 +1997,7 @@ pub fn constructor_add_imm( pub fn constructor_add32_imm(ctx: &mut C, arg0: Reg, arg1: Imm12) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1642. + // Rule at src/isa/aarch64/inst.isle line 1637. let expr0_0 = ALUOp::Add32; let expr1_0 = constructor_alu_rr_imm12(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2007,7 +2007,7 @@ pub fn constructor_add32_imm(ctx: &mut C, arg0: Reg, arg1: Imm12) -> pub fn constructor_add64_imm(ctx: &mut C, arg0: Reg, arg1: Imm12) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1645. + // Rule at src/isa/aarch64/inst.isle line 1640. let expr0_0 = ALUOp::Add64; let expr1_0 = constructor_alu_rr_imm12(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2024,14 +2024,14 @@ pub fn constructor_add_extend( if pattern0_0 == I64 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1649. + // Rule at src/isa/aarch64/inst.isle line 1644. let expr0_0 = constructor_add64_extend(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if let Some(pattern1_0) = C::fits_in_32(ctx, pattern0_0) { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1648. + // Rule at src/isa/aarch64/inst.isle line 1643. let expr0_0 = constructor_add32_extend(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } @@ -2046,7 +2046,7 @@ pub fn constructor_add32_extend( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1652. + // Rule at src/isa/aarch64/inst.isle line 1647. let expr0_0 = ALUOp::Add32; let expr1_0 = constructor_alu_rr_extend_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2060,7 +2060,7 @@ pub fn constructor_add64_extend( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1655. + // Rule at src/isa/aarch64/inst.isle line 1650. let expr0_0 = ALUOp::Add64; let expr1_0 = constructor_alu_rr_extend_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2079,7 +2079,7 @@ pub fn constructor_add_shift( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1659. + // Rule at src/isa/aarch64/inst.isle line 1654. let expr0_0 = constructor_add64_shift(ctx, pattern2_0, pattern3_0, pattern4_0)?; return Some(expr0_0); } @@ -2087,7 +2087,7 @@ pub fn constructor_add_shift( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1658. + // Rule at src/isa/aarch64/inst.isle line 1653. let expr0_0 = constructor_add32_shift(ctx, pattern2_0, pattern3_0, pattern4_0)?; return Some(expr0_0); } @@ -2104,7 +2104,7 @@ pub fn constructor_add32_shift( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1662. + // Rule at src/isa/aarch64/inst.isle line 1657. let expr0_0 = ALUOp::Add32; let expr1_0 = constructor_alu_rrr_shift(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2120,7 +2120,7 @@ pub fn constructor_add64_shift( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1665. + // Rule at src/isa/aarch64/inst.isle line 1660. let expr0_0 = ALUOp::Add64; let expr1_0 = constructor_alu_rrr_shift(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2136,7 +2136,7 @@ pub fn constructor_add_vec( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1668. + // Rule at src/isa/aarch64/inst.isle line 1663. let expr0_0 = VecALUOp::Add; let expr1_0 = constructor_vec_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2148,14 +2148,14 @@ pub fn constructor_sub(ctx: &mut C, arg0: Type, arg1: Reg, arg2: Reg if pattern0_0 == I64 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1674. + // Rule at src/isa/aarch64/inst.isle line 1669. let expr0_0 = constructor_sub64(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if let Some(pattern1_0) = C::fits_in_32(ctx, pattern0_0) { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1673. + // Rule at src/isa/aarch64/inst.isle line 1668. let expr0_0 = constructor_sub32(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } @@ -2166,7 +2166,7 @@ pub fn constructor_sub(ctx: &mut C, arg0: Type, arg1: Reg, arg2: Reg pub fn constructor_sub32(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1677. + // Rule at src/isa/aarch64/inst.isle line 1672. let expr0_0 = ALUOp::Sub32; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2176,7 +2176,7 @@ pub fn constructor_sub32(ctx: &mut C, arg0: Reg, arg1: Reg) -> Optio pub fn constructor_sub64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1680. + // Rule at src/isa/aarch64/inst.isle line 1675. let expr0_0 = ALUOp::Sub64; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2193,14 +2193,14 @@ pub fn constructor_sub_imm( if pattern0_0 == I64 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1684. + // Rule at src/isa/aarch64/inst.isle line 1679. let expr0_0 = constructor_sub64_imm(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if let Some(pattern1_0) = C::fits_in_32(ctx, pattern0_0) { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1683. + // Rule at src/isa/aarch64/inst.isle line 1678. let expr0_0 = constructor_sub32_imm(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } @@ -2211,7 +2211,7 @@ pub fn constructor_sub_imm( pub fn constructor_sub32_imm(ctx: &mut C, arg0: Reg, arg1: Imm12) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1687. + // Rule at src/isa/aarch64/inst.isle line 1682. let expr0_0 = ALUOp::Sub32; let expr1_0 = constructor_alu_rr_imm12(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2221,7 +2221,7 @@ pub fn constructor_sub32_imm(ctx: &mut C, arg0: Reg, arg1: Imm12) -> pub fn constructor_sub64_imm(ctx: &mut C, arg0: Reg, arg1: Imm12) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1690. + // Rule at src/isa/aarch64/inst.isle line 1685. let expr0_0 = ALUOp::Sub64; let expr1_0 = constructor_alu_rr_imm12(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2238,14 +2238,14 @@ pub fn constructor_sub_extend( if pattern0_0 == I64 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1694. + // Rule at src/isa/aarch64/inst.isle line 1689. let expr0_0 = constructor_sub64_extend(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if let Some(pattern1_0) = C::fits_in_32(ctx, pattern0_0) { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1693. + // Rule at src/isa/aarch64/inst.isle line 1688. let expr0_0 = constructor_sub32_extend(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } @@ -2260,7 +2260,7 @@ pub fn constructor_sub32_extend( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1697. + // Rule at src/isa/aarch64/inst.isle line 1692. let expr0_0 = ALUOp::Sub32; let expr1_0 = constructor_alu_rr_extend_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2274,7 +2274,7 @@ pub fn constructor_sub64_extend( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1700. + // Rule at src/isa/aarch64/inst.isle line 1695. let expr0_0 = ALUOp::Sub64; let expr1_0 = constructor_alu_rr_extend_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2293,7 +2293,7 @@ pub fn constructor_sub_shift( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1704. + // Rule at src/isa/aarch64/inst.isle line 1699. let expr0_0 = constructor_sub64_shift(ctx, pattern2_0, pattern3_0, pattern4_0)?; return Some(expr0_0); } @@ -2301,7 +2301,7 @@ pub fn constructor_sub_shift( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1703. + // Rule at src/isa/aarch64/inst.isle line 1698. let expr0_0 = constructor_sub32_shift(ctx, pattern2_0, pattern3_0, pattern4_0)?; return Some(expr0_0); } @@ -2318,7 +2318,7 @@ pub fn constructor_sub32_shift( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1707. + // Rule at src/isa/aarch64/inst.isle line 1702. let expr0_0 = ALUOp::Sub32; let expr1_0 = constructor_alu_rrr_shift(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2334,7 +2334,7 @@ pub fn constructor_sub64_shift( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1710. + // Rule at src/isa/aarch64/inst.isle line 1705. let expr0_0 = ALUOp::Sub64; let expr1_0 = constructor_alu_rrr_shift(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2350,7 +2350,7 @@ pub fn constructor_sub_vec( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1713. + // Rule at src/isa/aarch64/inst.isle line 1708. let expr0_0 = VecALUOp::Sub; let expr1_0 = constructor_vec_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2369,7 +2369,7 @@ pub fn constructor_madd( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1719. + // Rule at src/isa/aarch64/inst.isle line 1714. let expr0_0 = constructor_madd64(ctx, pattern2_0, pattern3_0, pattern4_0)?; return Some(expr0_0); } @@ -2377,7 +2377,7 @@ pub fn constructor_madd( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1718. + // Rule at src/isa/aarch64/inst.isle line 1713. let expr0_0 = constructor_madd32(ctx, pattern2_0, pattern3_0, pattern4_0)?; return Some(expr0_0); } @@ -2389,7 +2389,7 @@ pub fn constructor_madd32(ctx: &mut C, arg0: Reg, arg1: Reg, arg2: R let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1722. + // Rule at src/isa/aarch64/inst.isle line 1717. let expr0_0 = ALUOp3::MAdd32; let expr1_0 = constructor_alu_rrrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2400,7 +2400,7 @@ pub fn constructor_madd64(ctx: &mut C, arg0: Reg, arg1: Reg, arg2: R let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1725. + // Rule at src/isa/aarch64/inst.isle line 1720. let expr0_0 = ALUOp3::MAdd64; let expr1_0 = constructor_alu_rrrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2411,7 +2411,7 @@ pub fn constructor_msub64(ctx: &mut C, arg0: Reg, arg1: Reg, arg2: R let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1730. + // Rule at src/isa/aarch64/inst.isle line 1725. let expr0_0 = ALUOp3::MSub64; let expr1_0 = constructor_alu_rrrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2427,7 +2427,7 @@ pub fn constructor_uqadd( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1734. + // Rule at src/isa/aarch64/inst.isle line 1729. let expr0_0 = VecALUOp::Uqadd; let expr1_0 = constructor_vec_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2443,7 +2443,7 @@ pub fn constructor_sqadd( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1738. + // Rule at src/isa/aarch64/inst.isle line 1733. let expr0_0 = VecALUOp::Sqadd; let expr1_0 = constructor_vec_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2459,7 +2459,7 @@ pub fn constructor_uqsub( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1742. + // Rule at src/isa/aarch64/inst.isle line 1737. let expr0_0 = VecALUOp::Uqsub; let expr1_0 = constructor_vec_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2475,7 +2475,7 @@ pub fn constructor_sqsub( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1746. + // Rule at src/isa/aarch64/inst.isle line 1741. let expr0_0 = VecALUOp::Sqsub; let expr1_0 = constructor_vec_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2485,7 +2485,7 @@ pub fn constructor_sqsub( pub fn constructor_umulh(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1750. + // Rule at src/isa/aarch64/inst.isle line 1745. let expr0_0 = ALUOp::UMulH; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2495,7 +2495,7 @@ pub fn constructor_umulh(ctx: &mut C, arg0: Reg, arg1: Reg) -> Optio pub fn constructor_smulh(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1754. + // Rule at src/isa/aarch64/inst.isle line 1749. let expr0_0 = ALUOp::SMulH; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2511,7 +2511,7 @@ pub fn constructor_mul( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1758. + // Rule at src/isa/aarch64/inst.isle line 1753. let expr0_0 = VecALUOp::Mul; let expr1_0 = constructor_vec_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2521,7 +2521,7 @@ pub fn constructor_mul( pub fn constructor_neg(ctx: &mut C, arg0: Reg, arg1: &VectorSize) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1762. + // Rule at src/isa/aarch64/inst.isle line 1757. let expr0_0 = VecMisc2::Neg; let expr1_0 = constructor_vec_misc(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2531,7 +2531,7 @@ pub fn constructor_neg(ctx: &mut C, arg0: Reg, arg1: &VectorSize) -> pub fn constructor_rev64(ctx: &mut C, arg0: Reg, arg1: &VectorSize) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1766. + // Rule at src/isa/aarch64/inst.isle line 1761. let expr0_0 = VecMisc2::Rev64; let expr1_0 = constructor_vec_misc(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2541,7 +2541,7 @@ pub fn constructor_rev64(ctx: &mut C, arg0: Reg, arg1: &VectorSize) pub fn constructor_xtn64(ctx: &mut C, arg0: Reg, arg1: bool) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1770. + // Rule at src/isa/aarch64/inst.isle line 1765. let expr0_0 = VecRRNarrowOp::Xtn64; let expr1_0 = constructor_vec_rr_narrow(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2557,7 +2557,7 @@ pub fn constructor_addp( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1774. + // Rule at src/isa/aarch64/inst.isle line 1769. let expr0_0 = VecALUOp::Addp; let expr1_0 = constructor_vec_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2567,7 +2567,7 @@ pub fn constructor_addp( pub fn constructor_addv(ctx: &mut C, arg0: Reg, arg1: &VectorSize) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1778. + // Rule at src/isa/aarch64/inst.isle line 1773. let expr0_0 = VecLanesOp::Addv; let expr1_0 = constructor_vec_lanes(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2577,7 +2577,7 @@ pub fn constructor_addv(ctx: &mut C, arg0: Reg, arg1: &VectorSize) - pub fn constructor_shll32(ctx: &mut C, arg0: Reg, arg1: bool) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1782. + // Rule at src/isa/aarch64/inst.isle line 1777. let expr0_0 = VecRRLongOp::Shll32; let expr1_0 = constructor_vec_rr_long(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2595,7 +2595,7 @@ pub fn constructor_umlal32( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1786. + // Rule at src/isa/aarch64/inst.isle line 1781. let expr0_0 = VecRRRLongOp::Umlal32; let expr1_0 = constructor_vec_rrrr_long( ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0, pattern3_0, @@ -2613,7 +2613,7 @@ pub fn constructor_smull8( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1790. + // Rule at src/isa/aarch64/inst.isle line 1785. let expr0_0 = VecRRRLongOp::Smull8; let expr1_0 = constructor_vec_rrr_long(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2629,7 +2629,7 @@ pub fn constructor_umull8( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1794. + // Rule at src/isa/aarch64/inst.isle line 1789. let expr0_0 = VecRRRLongOp::Umull8; let expr1_0 = constructor_vec_rrr_long(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2645,7 +2645,7 @@ pub fn constructor_smull16( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1798. + // Rule at src/isa/aarch64/inst.isle line 1793. let expr0_0 = VecRRRLongOp::Smull16; let expr1_0 = constructor_vec_rrr_long(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2661,7 +2661,7 @@ pub fn constructor_umull16( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1802. + // Rule at src/isa/aarch64/inst.isle line 1797. let expr0_0 = VecRRRLongOp::Umull16; let expr1_0 = constructor_vec_rrr_long(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2677,7 +2677,7 @@ pub fn constructor_smull32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1806. + // Rule at src/isa/aarch64/inst.isle line 1801. let expr0_0 = VecRRRLongOp::Smull32; let expr1_0 = constructor_vec_rrr_long(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2693,7 +2693,7 @@ pub fn constructor_umull32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1810. + // Rule at src/isa/aarch64/inst.isle line 1805. let expr0_0 = VecRRRLongOp::Umull32; let expr1_0 = constructor_vec_rrr_long(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2703,7 +2703,7 @@ pub fn constructor_umull32( pub fn constructor_asr64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1814. + // Rule at src/isa/aarch64/inst.isle line 1809. let expr0_0 = ALUOp::Asr64; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2713,7 +2713,7 @@ pub fn constructor_asr64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Optio pub fn constructor_asr64_imm(ctx: &mut C, arg0: Reg, arg1: ImmShift) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1817. + // Rule at src/isa/aarch64/inst.isle line 1812. let expr0_0 = ALUOp::Asr64; let expr1_0 = constructor_alu_rr_imm_shift(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2723,7 +2723,7 @@ pub fn constructor_asr64_imm(ctx: &mut C, arg0: Reg, arg1: ImmShift) pub fn constructor_lsr32(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1821. + // Rule at src/isa/aarch64/inst.isle line 1816. let expr0_0 = ALUOp::Lsr32; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2733,7 +2733,7 @@ pub fn constructor_lsr32(ctx: &mut C, arg0: Reg, arg1: Reg) -> Optio pub fn constructor_lsr32_imm(ctx: &mut C, arg0: Reg, arg1: ImmShift) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1824. + // Rule at src/isa/aarch64/inst.isle line 1819. let expr0_0 = ALUOp::Lsr32; let expr1_0 = constructor_alu_rr_imm_shift(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2743,7 +2743,7 @@ pub fn constructor_lsr32_imm(ctx: &mut C, arg0: Reg, arg1: ImmShift) pub fn constructor_lsr64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1827. + // Rule at src/isa/aarch64/inst.isle line 1822. let expr0_0 = ALUOp::Lsr64; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2753,7 +2753,7 @@ pub fn constructor_lsr64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Optio pub fn constructor_lsr64_imm(ctx: &mut C, arg0: Reg, arg1: ImmShift) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1830. + // Rule at src/isa/aarch64/inst.isle line 1825. let expr0_0 = ALUOp::Lsr64; let expr1_0 = constructor_alu_rr_imm_shift(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2763,7 +2763,7 @@ pub fn constructor_lsr64_imm(ctx: &mut C, arg0: Reg, arg1: ImmShift) pub fn constructor_lsl32(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1834. + // Rule at src/isa/aarch64/inst.isle line 1829. let expr0_0 = ALUOp::Lsl32; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2773,7 +2773,7 @@ pub fn constructor_lsl32(ctx: &mut C, arg0: Reg, arg1: Reg) -> Optio pub fn constructor_lsl32_imm(ctx: &mut C, arg0: Reg, arg1: ImmShift) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1837. + // Rule at src/isa/aarch64/inst.isle line 1832. let expr0_0 = ALUOp::Lsl32; let expr1_0 = constructor_alu_rr_imm_shift(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2783,7 +2783,7 @@ pub fn constructor_lsl32_imm(ctx: &mut C, arg0: Reg, arg1: ImmShift) pub fn constructor_lsl64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1840. + // Rule at src/isa/aarch64/inst.isle line 1835. let expr0_0 = ALUOp::Lsl64; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2793,7 +2793,7 @@ pub fn constructor_lsl64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Optio pub fn constructor_lsl64_imm(ctx: &mut C, arg0: Reg, arg1: ImmShift) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1843. + // Rule at src/isa/aarch64/inst.isle line 1838. let expr0_0 = ALUOp::Lsl64; let expr1_0 = constructor_alu_rr_imm_shift(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2803,7 +2803,7 @@ pub fn constructor_lsl64_imm(ctx: &mut C, arg0: Reg, arg1: ImmShift) pub fn constructor_udiv64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1847. + // Rule at src/isa/aarch64/inst.isle line 1842. let expr0_0 = ALUOp::UDiv64; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2813,7 +2813,7 @@ pub fn constructor_udiv64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Opti pub fn constructor_sdiv64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1851. + // Rule at src/isa/aarch64/inst.isle line 1846. let expr0_0 = ALUOp::SDiv64; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2823,7 +2823,7 @@ pub fn constructor_sdiv64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Opti pub fn constructor_not(ctx: &mut C, arg0: Reg, arg1: &VectorSize) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1855. + // Rule at src/isa/aarch64/inst.isle line 1850. let expr0_0 = VecMisc2::Not; let expr1_0 = constructor_vec_misc(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2840,14 +2840,14 @@ pub fn constructor_orr_not( if pattern0_0 == I64 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1861. + // Rule at src/isa/aarch64/inst.isle line 1856. let expr0_0 = constructor_orr_not64(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if let Some(pattern1_0) = C::fits_in_32(ctx, pattern0_0) { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1860. + // Rule at src/isa/aarch64/inst.isle line 1855. let expr0_0 = constructor_orr_not32(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } @@ -2858,7 +2858,7 @@ pub fn constructor_orr_not( pub fn constructor_orr_not32(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1864. + // Rule at src/isa/aarch64/inst.isle line 1859. let expr0_0 = ALUOp::OrrNot32; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2868,7 +2868,7 @@ pub fn constructor_orr_not32(ctx: &mut C, arg0: Reg, arg1: Reg) -> O pub fn constructor_orr_not64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1867. + // Rule at src/isa/aarch64/inst.isle line 1862. let expr0_0 = ALUOp::OrrNot64; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2887,7 +2887,7 @@ pub fn constructor_orr_not_shift( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1871. + // Rule at src/isa/aarch64/inst.isle line 1866. let expr0_0 = constructor_orr_not_shift64(ctx, pattern2_0, pattern3_0, pattern4_0)?; return Some(expr0_0); } @@ -2895,7 +2895,7 @@ pub fn constructor_orr_not_shift( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1870. + // Rule at src/isa/aarch64/inst.isle line 1865. let expr0_0 = constructor_orr_not_shift32(ctx, pattern2_0, pattern3_0, pattern4_0)?; return Some(expr0_0); } @@ -2912,7 +2912,7 @@ pub fn constructor_orr_not_shift32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1874. + // Rule at src/isa/aarch64/inst.isle line 1869. let expr0_0 = ALUOp::OrrNot32; let expr1_0 = constructor_alu_rrr_shift(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2928,7 +2928,7 @@ pub fn constructor_orr_not_shift64( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1877. + // Rule at src/isa/aarch64/inst.isle line 1872. let expr0_0 = ALUOp::OrrNot64; let expr1_0 = constructor_alu_rrr_shift(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2938,7 +2938,7 @@ pub fn constructor_orr_not_shift64( pub fn constructor_orr32(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1882. + // Rule at src/isa/aarch64/inst.isle line 1877. let expr0_0 = ALUOp::Orr32; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2948,7 +2948,7 @@ pub fn constructor_orr32(ctx: &mut C, arg0: Reg, arg1: Reg) -> Optio pub fn constructor_orr32_imm(ctx: &mut C, arg0: Reg, arg1: ImmLogic) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1885. + // Rule at src/isa/aarch64/inst.isle line 1880. let expr0_0 = ALUOp::Orr32; let expr1_0 = constructor_alu_rr_imm_logic(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2958,7 +2958,7 @@ pub fn constructor_orr32_imm(ctx: &mut C, arg0: Reg, arg1: ImmLogic) pub fn constructor_orr64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1888. + // Rule at src/isa/aarch64/inst.isle line 1883. let expr0_0 = ALUOp::Orr64; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2968,7 +2968,7 @@ pub fn constructor_orr64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Optio pub fn constructor_orr64_imm(ctx: &mut C, arg0: Reg, arg1: ImmLogic) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1891. + // Rule at src/isa/aarch64/inst.isle line 1886. let expr0_0 = ALUOp::Orr64; let expr1_0 = constructor_alu_rr_imm_logic(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2984,7 +2984,7 @@ pub fn constructor_orr_vec( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1894. + // Rule at src/isa/aarch64/inst.isle line 1889. let expr0_0 = VecALUOp::Orr; let expr1_0 = constructor_vec_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2994,7 +2994,7 @@ pub fn constructor_orr_vec( pub fn constructor_and32_imm(ctx: &mut C, arg0: Reg, arg1: ImmLogic) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1899. + // Rule at src/isa/aarch64/inst.isle line 1894. let expr0_0 = ALUOp::And32; let expr1_0 = constructor_alu_rr_imm_logic(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -3010,7 +3010,7 @@ pub fn constructor_and_vec( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1902. + // Rule at src/isa/aarch64/inst.isle line 1897. let expr0_0 = VecALUOp::And; let expr1_0 = constructor_vec_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -3026,7 +3026,7 @@ pub fn constructor_eor_vec( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1906. + // Rule at src/isa/aarch64/inst.isle line 1901. let expr0_0 = VecALUOp::Eor; let expr1_0 = constructor_vec_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -3042,7 +3042,7 @@ pub fn constructor_bic_vec( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1910. + // Rule at src/isa/aarch64/inst.isle line 1905. let expr0_0 = VecALUOp::Bic; let expr1_0 = constructor_vec_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -3058,7 +3058,7 @@ pub fn constructor_sshl( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1914. + // Rule at src/isa/aarch64/inst.isle line 1909. let expr0_0 = VecALUOp::Sshl; let expr1_0 = constructor_vec_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -3074,7 +3074,7 @@ pub fn constructor_ushl( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1918. + // Rule at src/isa/aarch64/inst.isle line 1913. let expr0_0 = VecALUOp::Ushl; let expr1_0 = constructor_vec_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -3084,7 +3084,7 @@ pub fn constructor_ushl( pub fn constructor_rotr32(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1923. + // Rule at src/isa/aarch64/inst.isle line 1918. let expr0_0 = ALUOp::RotR32; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -3094,7 +3094,7 @@ pub fn constructor_rotr32(ctx: &mut C, arg0: Reg, arg1: Reg) -> Opti pub fn constructor_rotr32_imm(ctx: &mut C, arg0: Reg, arg1: ImmShift) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1926. + // Rule at src/isa/aarch64/inst.isle line 1921. let expr0_0 = ALUOp::RotR32; let expr1_0 = constructor_alu_rr_imm_shift(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -3104,7 +3104,7 @@ pub fn constructor_rotr32_imm(ctx: &mut C, arg0: Reg, arg1: ImmShift pub fn constructor_rotr64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1929. + // Rule at src/isa/aarch64/inst.isle line 1924. let expr0_0 = ALUOp::RotR64; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -3114,7 +3114,7 @@ pub fn constructor_rotr64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Opti pub fn constructor_rotr64_imm(ctx: &mut C, arg0: Reg, arg1: ImmShift) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1932. + // Rule at src/isa/aarch64/inst.isle line 1927. let expr0_0 = ALUOp::RotR64; let expr1_0 = constructor_alu_rr_imm_shift(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -3123,7 +3123,7 @@ pub fn constructor_rotr64_imm(ctx: &mut C, arg0: Reg, arg1: ImmShift // Generated as internal constructor for term rbit32. pub fn constructor_rbit32(ctx: &mut C, arg0: Reg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/aarch64/inst.isle line 1937. + // Rule at src/isa/aarch64/inst.isle line 1932. let expr0_0 = BitOp::RBit32; let expr1_0 = constructor_bit_rr(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -3132,7 +3132,7 @@ pub fn constructor_rbit32(ctx: &mut C, arg0: Reg) -> Option { // Generated as internal constructor for term rbit64. pub fn constructor_rbit64(ctx: &mut C, arg0: Reg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/aarch64/inst.isle line 1940. + // Rule at src/isa/aarch64/inst.isle line 1935. let expr0_0 = BitOp::RBit64; let expr1_0 = constructor_bit_rr(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -3141,7 +3141,7 @@ pub fn constructor_rbit64(ctx: &mut C, arg0: Reg) -> Option { // Generated as internal constructor for term clz32. pub fn constructor_clz32(ctx: &mut C, arg0: Reg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/aarch64/inst.isle line 1945. + // Rule at src/isa/aarch64/inst.isle line 1940. let expr0_0 = BitOp::Clz32; let expr1_0 = constructor_bit_rr(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -3150,7 +3150,7 @@ pub fn constructor_clz32(ctx: &mut C, arg0: Reg) -> Option { // Generated as internal constructor for term clz64. pub fn constructor_clz64(ctx: &mut C, arg0: Reg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/aarch64/inst.isle line 1948. + // Rule at src/isa/aarch64/inst.isle line 1943. let expr0_0 = BitOp::Clz64; let expr1_0 = constructor_bit_rr(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -3159,7 +3159,7 @@ pub fn constructor_clz64(ctx: &mut C, arg0: Reg) -> Option { // Generated as internal constructor for term cls32. pub fn constructor_cls32(ctx: &mut C, arg0: Reg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/aarch64/inst.isle line 1953. + // Rule at src/isa/aarch64/inst.isle line 1948. let expr0_0 = BitOp::Cls32; let expr1_0 = constructor_bit_rr(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -3168,7 +3168,7 @@ pub fn constructor_cls32(ctx: &mut C, arg0: Reg) -> Option { // Generated as internal constructor for term cls64. pub fn constructor_cls64(ctx: &mut C, arg0: Reg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/aarch64/inst.isle line 1956. + // Rule at src/isa/aarch64/inst.isle line 1951. let expr0_0 = BitOp::Cls64; let expr1_0 = constructor_bit_rr(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -3178,7 +3178,7 @@ pub fn constructor_cls64(ctx: &mut C, arg0: Reg) -> Option { pub fn constructor_eon32(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1961. + // Rule at src/isa/aarch64/inst.isle line 1956. let expr0_0 = ALUOp::EorNot32; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -3188,7 +3188,7 @@ pub fn constructor_eon32(ctx: &mut C, arg0: Reg, arg1: Reg) -> Optio pub fn constructor_eon64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1964. + // Rule at src/isa/aarch64/inst.isle line 1959. let expr0_0 = ALUOp::EorNot64; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -3198,7 +3198,7 @@ pub fn constructor_eon64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Optio pub fn constructor_vec_cnt(ctx: &mut C, arg0: Reg, arg1: &VectorSize) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1969. + // Rule at src/isa/aarch64/inst.isle line 1964. let expr0_0 = VecMisc2::Cnt; let expr1_0 = constructor_vec_misc(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -3215,25 +3215,25 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option }; if let Some(pattern3_0) = closure3() { if let Some(pattern4_0) = C::imm_logic_from_u64(ctx, pattern2_0, pattern3_0) { - // Rule at src/isa/aarch64/inst.isle line 1984. + // Rule at src/isa/aarch64/inst.isle line 1979. let expr0_0 = C::zero_reg(ctx); let expr1_0 = constructor_orr64_imm(ctx, expr0_0, pattern4_0)?; return Some(expr1_0); } } if let Some(pattern3_0) = C::move_wide_const_from_u64(ctx, pattern2_0) { - // Rule at src/isa/aarch64/inst.isle line 1976. + // Rule at src/isa/aarch64/inst.isle line 1971. let expr0_0 = OperandSize::Size64; let expr1_0 = constructor_movz(ctx, pattern3_0, &expr0_0)?; return Some(expr1_0); } if let Some(pattern3_0) = C::move_wide_const_from_negated_u64(ctx, pattern2_0) { - // Rule at src/isa/aarch64/inst.isle line 1980. + // Rule at src/isa/aarch64/inst.isle line 1975. let expr0_0 = OperandSize::Size64; let expr1_0 = constructor_movn(ctx, pattern3_0, &expr0_0)?; return Some(expr1_0); } - // Rule at src/isa/aarch64/inst.isle line 1991. + // Rule at src/isa/aarch64/inst.isle line 1986. let expr0_0 = C::load_constant64_full(ctx, pattern2_0); return Some(expr0_0); } @@ -3245,17 +3245,17 @@ pub fn constructor_put_in_reg_sext32(ctx: &mut C, arg0: Value) -> Op let pattern0_0 = arg0; let pattern1_0 = C::value_type(ctx, pattern0_0); if pattern1_0 == I32 { - // Rule at src/isa/aarch64/inst.isle line 2002. + // Rule at src/isa/aarch64/inst.isle line 1997. let expr0_0 = C::put_in_reg(ctx, pattern0_0); return Some(expr0_0); } if pattern1_0 == I64 { - // Rule at src/isa/aarch64/inst.isle line 2003. + // Rule at src/isa/aarch64/inst.isle line 1998. let expr0_0 = C::put_in_reg(ctx, pattern0_0); return Some(expr0_0); } if let Some(pattern2_0) = C::fits_in_32(ctx, pattern1_0) { - // Rule at src/isa/aarch64/inst.isle line 1998. + // Rule at src/isa/aarch64/inst.isle line 1993. let expr0_0 = C::put_in_reg(ctx, pattern0_0); let expr1_0: bool = true; let expr2_0 = C::ty_bits(ctx, pattern2_0); @@ -3271,17 +3271,17 @@ pub fn constructor_put_in_reg_zext32(ctx: &mut C, arg0: Value) -> Op let pattern0_0 = arg0; let pattern1_0 = C::value_type(ctx, pattern0_0); if pattern1_0 == I32 { - // Rule at src/isa/aarch64/inst.isle line 2011. + // Rule at src/isa/aarch64/inst.isle line 2006. let expr0_0 = C::put_in_reg(ctx, pattern0_0); return Some(expr0_0); } if pattern1_0 == I64 { - // Rule at src/isa/aarch64/inst.isle line 2012. + // Rule at src/isa/aarch64/inst.isle line 2007. let expr0_0 = C::put_in_reg(ctx, pattern0_0); return Some(expr0_0); } if let Some(pattern2_0) = C::fits_in_32(ctx, pattern1_0) { - // Rule at src/isa/aarch64/inst.isle line 2007. + // Rule at src/isa/aarch64/inst.isle line 2002. let expr0_0 = C::put_in_reg(ctx, pattern0_0); let expr1_0: bool = false; let expr2_0 = C::ty_bits(ctx, pattern2_0); @@ -3297,12 +3297,12 @@ pub fn constructor_put_in_reg_sext64(ctx: &mut C, arg0: Value) -> Op let pattern0_0 = arg0; let pattern1_0 = C::value_type(ctx, pattern0_0); if pattern1_0 == I64 { - // Rule at src/isa/aarch64/inst.isle line 2020. + // Rule at src/isa/aarch64/inst.isle line 2015. let expr0_0 = C::put_in_reg(ctx, pattern0_0); return Some(expr0_0); } if let Some(pattern2_0) = C::fits_in_32(ctx, pattern1_0) { - // Rule at src/isa/aarch64/inst.isle line 2016. + // Rule at src/isa/aarch64/inst.isle line 2011. let expr0_0 = C::put_in_reg(ctx, pattern0_0); let expr1_0: bool = true; let expr2_0 = C::ty_bits(ctx, pattern2_0); @@ -3318,12 +3318,12 @@ pub fn constructor_put_in_reg_zext64(ctx: &mut C, arg0: Value) -> Op let pattern0_0 = arg0; let pattern1_0 = C::value_type(ctx, pattern0_0); if pattern1_0 == I64 { - // Rule at src/isa/aarch64/inst.isle line 2028. + // Rule at src/isa/aarch64/inst.isle line 2023. let expr0_0 = C::put_in_reg(ctx, pattern0_0); return Some(expr0_0); } if let Some(pattern2_0) = C::fits_in_32(ctx, pattern1_0) { - // Rule at src/isa/aarch64/inst.isle line 2024. + // Rule at src/isa/aarch64/inst.isle line 2019. let expr0_0 = C::put_in_reg(ctx, pattern0_0); let expr1_0: bool = false; let expr2_0 = C::ty_bits(ctx, pattern2_0); @@ -3337,7 +3337,7 @@ pub fn constructor_put_in_reg_zext64(ctx: &mut C, arg0: Value) -> Op // Generated as internal constructor for term trap_if_zero_divisor. pub fn constructor_trap_if_zero_divisor(ctx: &mut C, arg0: Reg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/aarch64/inst.isle line 2033. + // Rule at src/isa/aarch64/inst.isle line 2028. let expr0_0 = C::cond_br_zero(ctx, pattern0_0); let expr1_0 = C::trap_code_division_by_zero(ctx); let expr2_0 = MInst::TrapIf { @@ -3352,12 +3352,12 @@ pub fn constructor_trap_if_zero_divisor(ctx: &mut C, arg0: Reg) -> O pub fn constructor_size_from_ty(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I64 { - // Rule at src/isa/aarch64/inst.isle line 2039. + // Rule at src/isa/aarch64/inst.isle line 2034. let expr0_0 = OperandSize::Size64; return Some(expr0_0); } if let Some(pattern1_0) = C::fits_in_32(ctx, pattern0_0) { - // Rule at src/isa/aarch64/inst.isle line 2038. + // Rule at src/isa/aarch64/inst.isle line 2033. let expr0_0 = OperandSize::Size32; return Some(expr0_0); } @@ -3374,7 +3374,7 @@ pub fn constructor_trap_if_div_overflow( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 2045. + // Rule at src/isa/aarch64/inst.isle line 2040. let expr0_0 = constructor_adds_op(ctx, pattern0_0)?; let expr1_0 = C::writable_zero_reg(ctx); let expr2_0: u8 = 1; @@ -3418,12 +3418,12 @@ pub fn constructor_trap_if_div_overflow( pub fn constructor_adds_op(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I64 { - // Rule at src/isa/aarch64/inst.isle line 2065. + // Rule at src/isa/aarch64/inst.isle line 2060. let expr0_0 = ALUOp::AddS64; return Some(expr0_0); } if let Some(pattern1_0) = C::fits_in_32(ctx, pattern0_0) { - // Rule at src/isa/aarch64/inst.isle line 2064. + // Rule at src/isa/aarch64/inst.isle line 2059. let expr0_0 = ALUOp::AddS32; return Some(expr0_0); } @@ -3457,7 +3457,7 @@ pub fn constructor_alu_rs_imm_logic_commutative( C::imm_logic_from_imm64(ctx, pattern5_1, pattern7_0) { let pattern9_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 2095. + // Rule at src/isa/aarch64/inst.isle line 2090. let expr0_0 = C::put_in_reg(ctx, pattern9_0); let expr1_0 = constructor_alu_rr_imm_logic(ctx, pattern0_0, expr0_0, pattern8_0)?; @@ -3488,7 +3488,7 @@ pub fn constructor_alu_rs_imm_logic_commutative( C::lshl_from_imm64(ctx, pattern10_1, pattern12_0) { let pattern14_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 2101. + // Rule at src/isa/aarch64/inst.isle line 2096. let expr0_0 = C::put_in_reg(ctx, pattern14_0); let expr1_0 = C::put_in_reg(ctx, pattern7_0); let expr2_0 = constructor_alu_rrr_shift( @@ -3525,7 +3525,7 @@ pub fn constructor_alu_rs_imm_logic_commutative( if let Some(pattern9_0) = C::imm_logic_from_imm64(ctx, pattern6_1, pattern8_0) { - // Rule at src/isa/aarch64/inst.isle line 2093. + // Rule at src/isa/aarch64/inst.isle line 2088. let expr0_0 = C::put_in_reg(ctx, pattern2_0); let expr1_0 = constructor_alu_rr_imm_logic(ctx, pattern0_0, expr0_0, pattern9_0)?; @@ -3555,7 +3555,7 @@ pub fn constructor_alu_rs_imm_logic_commutative( if let Some(pattern14_0) = C::lshl_from_imm64(ctx, pattern11_1, pattern13_0) { - // Rule at src/isa/aarch64/inst.isle line 2099. + // Rule at src/isa/aarch64/inst.isle line 2094. let expr0_0 = C::put_in_reg(ctx, pattern2_0); let expr1_0 = C::put_in_reg(ctx, pattern8_0); let expr2_0 = constructor_alu_rrr_shift( @@ -3576,7 +3576,7 @@ pub fn constructor_alu_rs_imm_logic_commutative( _ => {} } } - // Rule at src/isa/aarch64/inst.isle line 2089. + // Rule at src/isa/aarch64/inst.isle line 2084. let expr0_0 = C::put_in_reg(ctx, pattern2_0); let expr1_0 = C::put_in_reg(ctx, pattern3_0); let expr2_0 = constructor_alu_rrr(ctx, pattern0_0, expr0_0, expr1_0)?; @@ -3610,7 +3610,7 @@ pub fn constructor_alu_rs_imm_logic( if let Some(pattern9_0) = C::imm_logic_from_imm64(ctx, pattern6_1, pattern8_0) { - // Rule at src/isa/aarch64/inst.isle line 2109. + // Rule at src/isa/aarch64/inst.isle line 2104. let expr0_0 = C::put_in_reg(ctx, pattern2_0); let expr1_0 = constructor_alu_rr_imm_logic(ctx, pattern0_0, expr0_0, pattern9_0)?; @@ -3640,7 +3640,7 @@ pub fn constructor_alu_rs_imm_logic( if let Some(pattern14_0) = C::lshl_from_imm64(ctx, pattern11_1, pattern13_0) { - // Rule at src/isa/aarch64/inst.isle line 2111. + // Rule at src/isa/aarch64/inst.isle line 2106. let expr0_0 = C::put_in_reg(ctx, pattern2_0); let expr1_0 = C::put_in_reg(ctx, pattern8_0); let expr2_0 = constructor_alu_rrr_shift( @@ -3661,7 +3661,7 @@ pub fn constructor_alu_rs_imm_logic( _ => {} } } - // Rule at src/isa/aarch64/inst.isle line 2107. + // Rule at src/isa/aarch64/inst.isle line 2102. let expr0_0 = C::put_in_reg(ctx, pattern2_0); let expr1_0 = C::put_in_reg(ctx, pattern3_0); let expr2_0 = constructor_alu_rrr(ctx, pattern0_0, expr0_0, expr1_0)?; @@ -3678,7 +3678,7 @@ pub fn constructor_i128_alu_bitop( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 2119. + // Rule at src/isa/aarch64/inst.isle line 2114. let expr0_0 = C::put_in_regs(ctx, pattern1_0); let expr1_0: usize = 0; let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); diff --git a/cranelift/codegen/src/isa/s390x/inst.isle b/cranelift/codegen/src/isa/s390x/inst.isle index 377a68e7dd74..ae2e0303eb24 100644 --- a/cranelift/codegen/src/isa/s390x/inst.isle +++ b/cranelift/codegen/src/isa/s390x/inst.isle @@ -678,12 +678,7 @@ (type BoxCallInfo (primitive BoxCallInfo)) (type BoxCallIndInfo (primitive BoxCallIndInfo)) -(type MachLabel (primitive MachLabel)) -(type VecMachLabel (primitive VecMachLabel)) (type BoxJTSequenceInfo (primitive BoxJTSequenceInfo)) -(type BoxExternalName (primitive BoxExternalName)) -(type ValueLabel (primitive ValueLabel)) -(type UnwindInst (primitive UnwindInst)) ;; An ALU operation. (type ALUOp @@ -2908,5 +2903,3 @@ (decl fcmp_reg (Type Reg Reg) ProducesFlags) (rule (fcmp_reg $F32 src1 src2) (fpu_cmp32 src1 src2)) (rule (fcmp_reg $F64 src1 src2) (fpu_cmp64 src1 src2)) - - diff --git a/cranelift/codegen/src/isa/s390x/lower/isle/generated_code.manifest b/cranelift/codegen/src/isa/s390x/lower/isle/generated_code.manifest index f829980d23c7..092b167d3a6a 100644 --- a/cranelift/codegen/src/isa/s390x/lower/isle/generated_code.manifest +++ b/cranelift/codegen/src/isa/s390x/lower/isle/generated_code.manifest @@ -1,4 +1,4 @@ src/clif.isle 9ea75a6f790b5c03 -src/prelude.isle 51d2aef2566c1c96 -src/isa/s390x/inst.isle 17b77476355c4509 +src/prelude.isle 3d00cfa50c4d52d9 +src/isa/s390x/inst.isle ded9e2516999aa15 src/isa/s390x/lower.isle a0e21a567040bc33 diff --git a/cranelift/codegen/src/isa/s390x/lower/isle/generated_code.rs b/cranelift/codegen/src/isa/s390x/lower/isle/generated_code.rs index ea1edb2ebfb3..cf0ee2b052c7 100644 --- a/cranelift/codegen/src/isa/s390x/lower/isle/generated_code.rs +++ b/cranelift/codegen/src/isa/s390x/lower/isle/generated_code.rs @@ -124,19 +124,19 @@ pub trait Context { fn emit(&mut self, arg0: &MInst) -> Unit; } -/// Internal type SideEffectNoResult: defined at src/prelude.isle line 282. +/// Internal type SideEffectNoResult: defined at src/prelude.isle line 294. #[derive(Clone, Debug)] pub enum SideEffectNoResult { Inst { inst: MInst }, } -/// Internal type ProducesFlags: defined at src/prelude.isle line 295. +/// Internal type ProducesFlags: defined at src/prelude.isle line 307. #[derive(Clone, Debug)] pub enum ProducesFlags { ProducesFlags { inst: MInst, result: Reg }, } -/// Internal type ConsumesFlags: defined at src/prelude.isle line 298. +/// Internal type ConsumesFlags: defined at src/prelude.isle line 310. #[derive(Clone, Debug)] pub enum ConsumesFlags { ConsumesFlags { inst: MInst, result: Reg }, @@ -649,8 +649,8 @@ pub enum MInst { }, } -/// Internal type ALUOp: defined at src/isa/s390x/inst.isle line 689. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type ALUOp: defined at src/isa/s390x/inst.isle line 684. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum ALUOp { Add32, Add32Ext16, @@ -687,8 +687,8 @@ pub enum ALUOp { XorNot64, } -/// Internal type UnaryOp: defined at src/isa/s390x/inst.isle line 730. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type UnaryOp: defined at src/isa/s390x/inst.isle line 725. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum UnaryOp { Abs32, Abs64, @@ -700,8 +700,8 @@ pub enum UnaryOp { PopcntReg, } -/// Internal type ShiftOp: defined at src/isa/s390x/inst.isle line 743. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type ShiftOp: defined at src/isa/s390x/inst.isle line 738. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum ShiftOp { RotL32, RotL64, @@ -713,8 +713,8 @@ pub enum ShiftOp { AShR64, } -/// Internal type CmpOp: defined at src/isa/s390x/inst.isle line 756. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type CmpOp: defined at src/isa/s390x/inst.isle line 751. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum CmpOp { CmpS32, CmpS32Ext16, @@ -728,8 +728,8 @@ pub enum CmpOp { CmpL64Ext32, } -/// Internal type FPUOp1: defined at src/isa/s390x/inst.isle line 771. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type FPUOp1: defined at src/isa/s390x/inst.isle line 766. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum FPUOp1 { Abs32, Abs64, @@ -743,8 +743,8 @@ pub enum FPUOp1 { Cvt64To32, } -/// Internal type FPUOp2: defined at src/isa/s390x/inst.isle line 786. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type FPUOp2: defined at src/isa/s390x/inst.isle line 781. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum FPUOp2 { Add32, Add64, @@ -760,8 +760,8 @@ pub enum FPUOp2 { Min64, } -/// Internal type FPUOp3: defined at src/isa/s390x/inst.isle line 803. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type FPUOp3: defined at src/isa/s390x/inst.isle line 798. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum FPUOp3 { MAdd32, MAdd64, @@ -769,8 +769,8 @@ pub enum FPUOp3 { MSub64, } -/// Internal type FpuToIntOp: defined at src/isa/s390x/inst.isle line 812. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type FpuToIntOp: defined at src/isa/s390x/inst.isle line 807. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum FpuToIntOp { F32ToU32, F32ToI32, @@ -782,8 +782,8 @@ pub enum FpuToIntOp { F64ToI64, } -/// Internal type IntToFpuOp: defined at src/isa/s390x/inst.isle line 825. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type IntToFpuOp: defined at src/isa/s390x/inst.isle line 820. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum IntToFpuOp { U32ToF32, I32ToF32, @@ -795,8 +795,8 @@ pub enum IntToFpuOp { I64ToF64, } -/// Internal type FpuRoundMode: defined at src/isa/s390x/inst.isle line 839. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type FpuRoundMode: defined at src/isa/s390x/inst.isle line 834. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum FpuRoundMode { Minus32, Minus64, @@ -808,19 +808,19 @@ pub enum FpuRoundMode { Nearest64, } -/// Internal type WritableRegPair: defined at src/isa/s390x/inst.isle line 1236. +/// Internal type WritableRegPair: defined at src/isa/s390x/inst.isle line 1231. #[derive(Clone, Debug)] pub enum WritableRegPair { WritableRegPair { hi: WritableReg, lo: WritableReg }, } -/// Internal type RegPair: defined at src/isa/s390x/inst.isle line 1258. +/// Internal type RegPair: defined at src/isa/s390x/inst.isle line 1253. #[derive(Clone, Debug)] pub enum RegPair { RegPair { hi: Reg, lo: Reg }, } -/// Internal type ProducesBool: defined at src/isa/s390x/inst.isle line 2158. +/// Internal type ProducesBool: defined at src/isa/s390x/inst.isle line 2153. #[derive(Clone, Debug)] pub enum ProducesBool { ProducesBool { producer: ProducesFlags, cond: Cond }, @@ -829,7 +829,7 @@ pub enum ProducesBool { // Generated as internal constructor for term temp_reg. pub fn constructor_temp_reg(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; - // Rule at src/prelude.isle line 70. + // Rule at src/prelude.isle line 73. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); return Some(expr1_0); @@ -838,7 +838,7 @@ pub fn constructor_temp_reg(ctx: &mut C, arg0: Type) -> Option // Generated as internal constructor for term lo_reg. pub fn constructor_lo_reg(ctx: &mut C, arg0: Value) -> Option { let pattern0_0 = arg0; - // Rule at src/prelude.isle line 105. + // Rule at src/prelude.isle line 108. let expr0_0 = C::put_in_regs(ctx, pattern0_0); let expr1_0: usize = 0; let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); @@ -855,7 +855,7 @@ pub fn constructor_value_regs_none( inst: ref pattern1_0, } = pattern0_0 { - // Rule at src/prelude.isle line 287. + // Rule at src/prelude.isle line 299. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::value_regs_invalid(ctx); return Some(expr1_0); @@ -881,7 +881,7 @@ pub fn constructor_with_flags( result: pattern3_1, } = pattern2_0 { - // Rule at src/prelude.isle line 308. + // Rule at src/prelude.isle line 320. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern3_0); let expr2_0 = C::value_regs(ctx, pattern1_1, pattern3_1); @@ -909,7 +909,7 @@ pub fn constructor_with_flags_1( result: pattern3_1, } = pattern2_0 { - // Rule at src/prelude.isle line 316. + // Rule at src/prelude.isle line 328. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern3_0); return Some(pattern3_1); @@ -943,7 +943,7 @@ pub fn constructor_with_flags_2( result: pattern5_1, } = pattern4_0 { - // Rule at src/prelude.isle line 326. + // Rule at src/prelude.isle line 338. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern5_0); let expr2_0 = C::emit(ctx, &pattern3_0); @@ -960,7 +960,7 @@ pub fn constructor_mask_amt_reg(ctx: &mut C, arg0: Type, arg1: Reg) let pattern0_0 = arg0; if let Some(pattern1_0) = C::gpr32_ty(ctx, pattern0_0) { let pattern2_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1014. + // Rule at src/isa/s390x/inst.isle line 1009. let expr0_0: i64 = -1; let expr1_0 = C::mask_amt_imm(ctx, pattern1_0, expr0_0); let expr2_0 = C::u8_as_u16(ctx, expr1_0); @@ -971,7 +971,7 @@ pub fn constructor_mask_amt_reg(ctx: &mut C, arg0: Type, arg1: Reg) } if let Some(pattern1_0) = C::gpr64_ty(ctx, pattern0_0) { let pattern2_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1017. + // Rule at src/isa/s390x/inst.isle line 1012. return Some(pattern2_0); } return None; @@ -998,7 +998,7 @@ pub fn constructor_lower_address( let pattern7_0 = arg2; let pattern8_0 = C::i64_from_offset(ctx, pattern7_0); if pattern8_0 == 0 { - // Rule at src/isa/s390x/inst.isle line 1100. + // Rule at src/isa/s390x/inst.isle line 1095. let expr0_0 = C::put_in_reg(ctx, pattern6_0); let expr1_0 = C::put_in_reg(ctx, pattern6_1); let expr2_0 = C::memarg_reg_plus_reg(ctx, expr0_0, expr1_0, pattern0_0); @@ -1017,7 +1017,7 @@ pub fn constructor_lower_address( if let Some(pattern8_0) = C::memarg_symbol_offset_sum(ctx, pattern6_0, pattern7_0) { - // Rule at src/isa/s390x/inst.isle line 1103. + // Rule at src/isa/s390x/inst.isle line 1098. let expr0_0 = C::memarg_symbol(ctx, pattern3_0, pattern8_0, pattern0_0); return Some(expr0_0); } @@ -1027,7 +1027,7 @@ pub fn constructor_lower_address( } let pattern2_0 = arg2; let pattern3_0 = C::i64_from_offset(ctx, pattern2_0); - // Rule at src/isa/s390x/inst.isle line 1097. + // Rule at src/isa/s390x/inst.isle line 1092. let expr0_0 = C::put_in_reg(ctx, pattern1_0); let expr1_0 = C::memarg_reg_plus_off(ctx, expr0_0, pattern3_0, pattern0_0); return Some(expr1_0); @@ -1043,7 +1043,7 @@ pub fn constructor_stack_addr_impl( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1130. + // Rule at src/isa/s390x/inst.isle line 1125. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::abi_stackslot_addr(ctx, expr0_0, pattern1_0, pattern2_0); let expr2_0 = C::emit(ctx, &expr1_0); @@ -1063,7 +1063,7 @@ pub fn constructor_sink_load(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C) -> Option { - // Rule at src/isa/s390x/inst.isle line 1241. + // Rule at src/isa/s390x/inst.isle line 1236. let expr0_0: u8 = 0; let expr1_0 = C::writable_gpr(ctx, expr0_0); let expr2_0: u8 = 1; @@ -1176,7 +1176,7 @@ pub fn constructor_copy_writable_regpair( arg0: &RegPair, ) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/s390x/inst.isle line 1247. + // Rule at src/isa/s390x/inst.isle line 1242. let expr0_0 = constructor_temp_writable_regpair(ctx)?; return Some(expr0_0); } @@ -1192,7 +1192,7 @@ pub fn constructor_writable_regpair_hi( lo: pattern1_1, } = pattern0_0 { - // Rule at src/isa/s390x/inst.isle line 1251. + // Rule at src/isa/s390x/inst.isle line 1246. return Some(pattern1_0); } return None; @@ -1209,7 +1209,7 @@ pub fn constructor_writable_regpair_lo( lo: pattern1_1, } = pattern0_0 { - // Rule at src/isa/s390x/inst.isle line 1255. + // Rule at src/isa/s390x/inst.isle line 1250. return Some(pattern1_1); } return None; @@ -1226,7 +1226,7 @@ pub fn constructor_writable_regpair_to_regpair( lo: pattern1_1, } = pattern0_0 { - // Rule at src/isa/s390x/inst.isle line 1262. + // Rule at src/isa/s390x/inst.isle line 1257. let expr0_0 = C::writable_reg_to_reg(ctx, pattern1_0); let expr1_0 = C::writable_reg_to_reg(ctx, pattern1_1); let expr2_0 = RegPair::RegPair { @@ -1240,7 +1240,7 @@ pub fn constructor_writable_regpair_to_regpair( // Generated as internal constructor for term uninitialized_regpair. pub fn constructor_uninitialized_regpair(ctx: &mut C) -> Option { - // Rule at src/isa/s390x/inst.isle line 1267. + // Rule at src/isa/s390x/inst.isle line 1262. let expr0_0 = constructor_temp_writable_regpair(ctx)?; let expr1_0 = constructor_writable_regpair_to_regpair(ctx, &expr0_0)?; return Some(expr1_0); @@ -1254,7 +1254,7 @@ pub fn constructor_regpair_hi(ctx: &mut C, arg0: &RegPair) -> Option lo: pattern1_1, } = pattern0_0 { - // Rule at src/isa/s390x/inst.isle line 1272. + // Rule at src/isa/s390x/inst.isle line 1267. return Some(pattern1_0); } return None; @@ -1268,7 +1268,7 @@ pub fn constructor_regpair_lo(ctx: &mut C, arg0: &RegPair) -> Option lo: pattern1_1, } = pattern0_0 { - // Rule at src/isa/s390x/inst.isle line 1276. + // Rule at src/isa/s390x/inst.isle line 1271. return Some(pattern1_1); } return None; @@ -1286,7 +1286,7 @@ pub fn constructor_alu_rrr( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 1290. + // Rule at src/isa/s390x/inst.isle line 1285. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::AluRRR { alu_op: pattern1_0.clone(), @@ -1311,7 +1311,7 @@ pub fn constructor_alu_rrsimm16( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 1297. + // Rule at src/isa/s390x/inst.isle line 1292. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::AluRRSImm16 { alu_op: pattern1_0.clone(), @@ -1336,7 +1336,7 @@ pub fn constructor_alu_rr( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 1304. + // Rule at src/isa/s390x/inst.isle line 1299. let expr0_0 = constructor_copy_writable_reg(ctx, pattern0_0, pattern2_0)?; let expr1_0 = MInst::AluRR { alu_op: pattern1_0.clone(), @@ -1360,7 +1360,7 @@ pub fn constructor_alu_rx( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 1311. + // Rule at src/isa/s390x/inst.isle line 1306. let expr0_0 = constructor_copy_writable_reg(ctx, pattern0_0, pattern2_0)?; let expr1_0 = MInst::AluRX { alu_op: pattern1_0.clone(), @@ -1384,7 +1384,7 @@ pub fn constructor_alu_rsimm16( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 1318. + // Rule at src/isa/s390x/inst.isle line 1313. let expr0_0 = constructor_copy_writable_reg(ctx, pattern0_0, pattern2_0)?; let expr1_0 = MInst::AluRSImm16 { alu_op: pattern1_0.clone(), @@ -1408,7 +1408,7 @@ pub fn constructor_alu_rsimm32( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 1325. + // Rule at src/isa/s390x/inst.isle line 1320. let expr0_0 = constructor_copy_writable_reg(ctx, pattern0_0, pattern2_0)?; let expr1_0 = MInst::AluRSImm32 { alu_op: pattern1_0.clone(), @@ -1432,7 +1432,7 @@ pub fn constructor_alu_ruimm32( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 1332. + // Rule at src/isa/s390x/inst.isle line 1327. let expr0_0 = constructor_copy_writable_reg(ctx, pattern0_0, pattern2_0)?; let expr1_0 = MInst::AluRUImm32 { alu_op: pattern1_0.clone(), @@ -1456,7 +1456,7 @@ pub fn constructor_alu_ruimm16shifted( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 1339. + // Rule at src/isa/s390x/inst.isle line 1334. let expr0_0 = constructor_copy_writable_reg(ctx, pattern0_0, pattern2_0)?; let expr1_0 = MInst::AluRUImm16Shifted { alu_op: pattern1_0.clone(), @@ -1480,7 +1480,7 @@ pub fn constructor_alu_ruimm32shifted( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 1346. + // Rule at src/isa/s390x/inst.isle line 1341. let expr0_0 = constructor_copy_writable_reg(ctx, pattern0_0, pattern2_0)?; let expr1_0 = MInst::AluRUImm32Shifted { alu_op: pattern1_0.clone(), @@ -1496,7 +1496,7 @@ pub fn constructor_alu_ruimm32shifted( pub fn constructor_smul_wide(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1353. + // Rule at src/isa/s390x/inst.isle line 1348. let expr0_0 = constructor_temp_writable_regpair(ctx)?; let expr1_0 = MInst::SMulWide { rn: pattern0_0, @@ -1511,7 +1511,7 @@ pub fn constructor_smul_wide(ctx: &mut C, arg0: Reg, arg1: Reg) -> O pub fn constructor_umul_wide(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1360. + // Rule at src/isa/s390x/inst.isle line 1355. let expr0_0 = constructor_temp_writable_regpair(ctx)?; let expr1_0 = constructor_writable_regpair_lo(ctx, &expr0_0)?; let expr2_0 = MInst::Mov64 { @@ -1533,7 +1533,7 @@ pub fn constructor_sdivmod32( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1368. + // Rule at src/isa/s390x/inst.isle line 1363. let expr0_0 = constructor_copy_writable_regpair(ctx, pattern0_0)?; let expr1_0 = MInst::SDivMod32 { rn: pattern1_0 }; let expr2_0 = C::emit(ctx, &expr1_0); @@ -1549,7 +1549,7 @@ pub fn constructor_sdivmod64( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1375. + // Rule at src/isa/s390x/inst.isle line 1370. let expr0_0 = constructor_copy_writable_regpair(ctx, pattern0_0)?; let expr1_0 = MInst::SDivMod64 { rn: pattern1_0 }; let expr2_0 = C::emit(ctx, &expr1_0); @@ -1565,7 +1565,7 @@ pub fn constructor_udivmod32( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1382. + // Rule at src/isa/s390x/inst.isle line 1377. let expr0_0 = constructor_copy_writable_regpair(ctx, pattern0_0)?; let expr1_0 = MInst::UDivMod32 { rn: pattern1_0 }; let expr2_0 = C::emit(ctx, &expr1_0); @@ -1581,7 +1581,7 @@ pub fn constructor_udivmod64( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1389. + // Rule at src/isa/s390x/inst.isle line 1384. let expr0_0 = constructor_copy_writable_regpair(ctx, pattern0_0)?; let expr1_0 = MInst::UDivMod64 { rn: pattern1_0 }; let expr2_0 = C::emit(ctx, &expr1_0); @@ -1603,7 +1603,7 @@ pub fn constructor_shift_rr( let pattern2_0 = arg2; let pattern3_0 = arg3; let pattern4_0 = arg4; - // Rule at src/isa/s390x/inst.isle line 1396. + // Rule at src/isa/s390x/inst.isle line 1391. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::ShiftRR { shift_op: pattern1_0.clone(), @@ -1627,7 +1627,7 @@ pub fn constructor_unary_rr( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1403. + // Rule at src/isa/s390x/inst.isle line 1398. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::UnaryRR { op: pattern1_0.clone(), @@ -1649,7 +1649,7 @@ pub fn constructor_cmp_rr( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1410. + // Rule at src/isa/s390x/inst.isle line 1405. let expr0_0 = MInst::CmpRR { op: pattern0_0.clone(), rn: pattern1_0, @@ -1673,7 +1673,7 @@ pub fn constructor_cmp_rx( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1416. + // Rule at src/isa/s390x/inst.isle line 1411. let expr0_0 = MInst::CmpRX { op: pattern0_0.clone(), rn: pattern1_0, @@ -1697,7 +1697,7 @@ pub fn constructor_cmp_rsimm16( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1422. + // Rule at src/isa/s390x/inst.isle line 1417. let expr0_0 = MInst::CmpRSImm16 { op: pattern0_0.clone(), rn: pattern1_0, @@ -1721,7 +1721,7 @@ pub fn constructor_cmp_rsimm32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1428. + // Rule at src/isa/s390x/inst.isle line 1423. let expr0_0 = MInst::CmpRSImm32 { op: pattern0_0.clone(), rn: pattern1_0, @@ -1745,7 +1745,7 @@ pub fn constructor_cmp_ruimm32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1434. + // Rule at src/isa/s390x/inst.isle line 1429. let expr0_0 = MInst::CmpRUImm32 { op: pattern0_0.clone(), rn: pattern1_0, @@ -1771,7 +1771,7 @@ pub fn constructor_atomic_rmw_impl( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 1440. + // Rule at src/isa/s390x/inst.isle line 1435. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::AtomicRmw { alu_op: pattern1_0.clone(), @@ -1794,7 +1794,7 @@ pub fn constructor_atomic_cas32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1447. + // Rule at src/isa/s390x/inst.isle line 1442. let expr0_0: Type = I32; let expr1_0 = constructor_copy_writable_reg(ctx, expr0_0, pattern0_0)?; let expr2_0 = MInst::AtomicCas32 { @@ -1817,7 +1817,7 @@ pub fn constructor_atomic_cas64( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1454. + // Rule at src/isa/s390x/inst.isle line 1449. let expr0_0: Type = I64; let expr1_0 = constructor_copy_writable_reg(ctx, expr0_0, pattern0_0)?; let expr2_0 = MInst::AtomicCas64 { @@ -1832,7 +1832,7 @@ pub fn constructor_atomic_cas64( // Generated as internal constructor for term fence_impl. pub fn constructor_fence_impl(ctx: &mut C) -> Option { - // Rule at src/isa/s390x/inst.isle line 1461. + // Rule at src/isa/s390x/inst.isle line 1456. let expr0_0 = MInst::Fence; let expr1_0 = SideEffectNoResult::Inst { inst: expr0_0 }; return Some(expr1_0); @@ -1841,7 +1841,7 @@ pub fn constructor_fence_impl(ctx: &mut C) -> Option(ctx: &mut C, arg0: &MemArg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/s390x/inst.isle line 1466. + // Rule at src/isa/s390x/inst.isle line 1461. let expr0_0: Type = I32; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::Load32 { @@ -1856,7 +1856,7 @@ pub fn constructor_load32(ctx: &mut C, arg0: &MemArg) -> Option // Generated as internal constructor for term load64. pub fn constructor_load64(ctx: &mut C, arg0: &MemArg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/s390x/inst.isle line 1473. + // Rule at src/isa/s390x/inst.isle line 1468. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::Load64 { @@ -1871,7 +1871,7 @@ pub fn constructor_load64(ctx: &mut C, arg0: &MemArg) -> Option // Generated as internal constructor for term loadrev16. pub fn constructor_loadrev16(ctx: &mut C, arg0: &MemArg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/s390x/inst.isle line 1480. + // Rule at src/isa/s390x/inst.isle line 1475. let expr0_0: Type = I32; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::LoadRev16 { @@ -1886,7 +1886,7 @@ pub fn constructor_loadrev16(ctx: &mut C, arg0: &MemArg) -> Option(ctx: &mut C, arg0: &MemArg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/s390x/inst.isle line 1487. + // Rule at src/isa/s390x/inst.isle line 1482. let expr0_0: Type = I32; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::LoadRev32 { @@ -1901,7 +1901,7 @@ pub fn constructor_loadrev32(ctx: &mut C, arg0: &MemArg) -> Option(ctx: &mut C, arg0: &MemArg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/s390x/inst.isle line 1494. + // Rule at src/isa/s390x/inst.isle line 1489. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::LoadRev64 { @@ -1921,7 +1921,7 @@ pub fn constructor_store8( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1501. + // Rule at src/isa/s390x/inst.isle line 1496. let expr0_0 = MInst::Store8 { rd: pattern0_0, mem: pattern1_0.clone(), @@ -1938,7 +1938,7 @@ pub fn constructor_store16( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1506. + // Rule at src/isa/s390x/inst.isle line 1501. let expr0_0 = MInst::Store16 { rd: pattern0_0, mem: pattern1_0.clone(), @@ -1955,7 +1955,7 @@ pub fn constructor_store32( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1511. + // Rule at src/isa/s390x/inst.isle line 1506. let expr0_0 = MInst::Store32 { rd: pattern0_0, mem: pattern1_0.clone(), @@ -1972,7 +1972,7 @@ pub fn constructor_store64( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1516. + // Rule at src/isa/s390x/inst.isle line 1511. let expr0_0 = MInst::Store64 { rd: pattern0_0, mem: pattern1_0.clone(), @@ -1989,7 +1989,7 @@ pub fn constructor_store8_imm( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1521. + // Rule at src/isa/s390x/inst.isle line 1516. let expr0_0 = MInst::StoreImm8 { imm: pattern0_0, mem: pattern1_0.clone(), @@ -2006,7 +2006,7 @@ pub fn constructor_store16_imm( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1526. + // Rule at src/isa/s390x/inst.isle line 1521. let expr0_0 = MInst::StoreImm16 { imm: pattern0_0, mem: pattern1_0.clone(), @@ -2023,7 +2023,7 @@ pub fn constructor_store32_simm16( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1531. + // Rule at src/isa/s390x/inst.isle line 1526. let expr0_0 = MInst::StoreImm32SExt16 { imm: pattern0_0, mem: pattern1_0.clone(), @@ -2040,7 +2040,7 @@ pub fn constructor_store64_simm16( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1536. + // Rule at src/isa/s390x/inst.isle line 1531. let expr0_0 = MInst::StoreImm64SExt16 { imm: pattern0_0, mem: pattern1_0.clone(), @@ -2057,7 +2057,7 @@ pub fn constructor_storerev16( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1541. + // Rule at src/isa/s390x/inst.isle line 1536. let expr0_0 = MInst::StoreRev16 { rd: pattern0_0, mem: pattern1_0.clone(), @@ -2074,7 +2074,7 @@ pub fn constructor_storerev32( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1546. + // Rule at src/isa/s390x/inst.isle line 1541. let expr0_0 = MInst::StoreRev32 { rd: pattern0_0, mem: pattern1_0.clone(), @@ -2091,7 +2091,7 @@ pub fn constructor_storerev64( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1551. + // Rule at src/isa/s390x/inst.isle line 1546. let expr0_0 = MInst::StoreRev64 { rd: pattern0_0, mem: pattern1_0.clone(), @@ -2110,7 +2110,7 @@ pub fn constructor_fpu_rr( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1556. + // Rule at src/isa/s390x/inst.isle line 1551. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::FpuRR { fpu_op: pattern1_0.clone(), @@ -2134,7 +2134,7 @@ pub fn constructor_fpu_rrr( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 1563. + // Rule at src/isa/s390x/inst.isle line 1558. let expr0_0 = constructor_copy_writable_reg(ctx, pattern0_0, pattern2_0)?; let expr1_0 = MInst::FpuRRR { fpu_op: pattern1_0.clone(), @@ -2160,7 +2160,7 @@ pub fn constructor_fpu_rrrr( let pattern2_0 = arg2; let pattern3_0 = arg3; let pattern4_0 = arg4; - // Rule at src/isa/s390x/inst.isle line 1570. + // Rule at src/isa/s390x/inst.isle line 1565. let expr0_0 = constructor_copy_writable_reg(ctx, pattern0_0, pattern2_0)?; let expr1_0 = MInst::FpuRRRR { fpu_op: pattern1_0.clone(), @@ -2183,7 +2183,7 @@ pub fn constructor_fpu_copysign( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1577. + // Rule at src/isa/s390x/inst.isle line 1572. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::FpuCopysign { rd: expr0_0, @@ -2203,7 +2203,7 @@ pub fn constructor_fpu_cmp32( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1584. + // Rule at src/isa/s390x/inst.isle line 1579. let expr0_0 = MInst::FpuCmp32 { rn: pattern0_0, rm: pattern1_0, @@ -2224,7 +2224,7 @@ pub fn constructor_fpu_cmp64( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1590. + // Rule at src/isa/s390x/inst.isle line 1585. let expr0_0 = MInst::FpuCmp64 { rn: pattern0_0, rm: pattern1_0, @@ -2247,7 +2247,7 @@ pub fn constructor_fpu_to_int( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1596. + // Rule at src/isa/s390x/inst.isle line 1591. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::FpuToInt { op: pattern1_0.clone(), @@ -2272,7 +2272,7 @@ pub fn constructor_int_to_fpu( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1603. + // Rule at src/isa/s390x/inst.isle line 1598. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::IntToFpu { op: pattern1_0.clone(), @@ -2294,7 +2294,7 @@ pub fn constructor_fpu_round( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1610. + // Rule at src/isa/s390x/inst.isle line 1605. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::FpuRound { op: pattern1_0.clone(), @@ -2318,7 +2318,7 @@ pub fn constructor_fpuvec_rrr( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 1617. + // Rule at src/isa/s390x/inst.isle line 1612. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::FpuVecRRR { fpu_op: pattern1_0.clone(), @@ -2334,7 +2334,7 @@ pub fn constructor_fpuvec_rrr( // Generated as internal constructor for term mov_to_fpr. pub fn constructor_mov_to_fpr(ctx: &mut C, arg0: Reg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/s390x/inst.isle line 1624. + // Rule at src/isa/s390x/inst.isle line 1619. let expr0_0: Type = F64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::MovToFpr { @@ -2349,7 +2349,7 @@ pub fn constructor_mov_to_fpr(ctx: &mut C, arg0: Reg) -> Option // Generated as internal constructor for term mov_from_fpr. pub fn constructor_mov_from_fpr(ctx: &mut C, arg0: Reg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/s390x/inst.isle line 1631. + // Rule at src/isa/s390x/inst.isle line 1626. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::MovFromFpr { @@ -2364,7 +2364,7 @@ pub fn constructor_mov_from_fpr(ctx: &mut C, arg0: Reg) -> Option(ctx: &mut C, arg0: &MemArg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/s390x/inst.isle line 1638. + // Rule at src/isa/s390x/inst.isle line 1633. let expr0_0: Type = F32; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::FpuLoad32 { @@ -2379,7 +2379,7 @@ pub fn constructor_fpu_load32(ctx: &mut C, arg0: &MemArg) -> Option< // Generated as internal constructor for term fpu_load64. pub fn constructor_fpu_load64(ctx: &mut C, arg0: &MemArg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/s390x/inst.isle line 1645. + // Rule at src/isa/s390x/inst.isle line 1640. let expr0_0: Type = F64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::FpuLoad64 { @@ -2394,7 +2394,7 @@ pub fn constructor_fpu_load64(ctx: &mut C, arg0: &MemArg) -> Option< // Generated as internal constructor for term fpu_loadrev32. pub fn constructor_fpu_loadrev32(ctx: &mut C, arg0: &MemArg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/s390x/inst.isle line 1652. + // Rule at src/isa/s390x/inst.isle line 1647. let expr0_0: Type = F32; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::FpuLoadRev32 { @@ -2409,7 +2409,7 @@ pub fn constructor_fpu_loadrev32(ctx: &mut C, arg0: &MemArg) -> Opti // Generated as internal constructor for term fpu_loadrev64. pub fn constructor_fpu_loadrev64(ctx: &mut C, arg0: &MemArg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/s390x/inst.isle line 1659. + // Rule at src/isa/s390x/inst.isle line 1654. let expr0_0: Type = F64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::FpuLoadRev64 { @@ -2429,7 +2429,7 @@ pub fn constructor_fpu_store32( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1666. + // Rule at src/isa/s390x/inst.isle line 1661. let expr0_0 = MInst::FpuStore32 { rd: pattern0_0, mem: pattern1_0.clone(), @@ -2446,7 +2446,7 @@ pub fn constructor_fpu_store64( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1671. + // Rule at src/isa/s390x/inst.isle line 1666. let expr0_0 = MInst::FpuStore64 { rd: pattern0_0, mem: pattern1_0.clone(), @@ -2463,7 +2463,7 @@ pub fn constructor_fpu_storerev32( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1676. + // Rule at src/isa/s390x/inst.isle line 1671. let expr0_0 = MInst::FpuStoreRev32 { rd: pattern0_0, mem: pattern1_0.clone(), @@ -2480,7 +2480,7 @@ pub fn constructor_fpu_storerev64( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1681. + // Rule at src/isa/s390x/inst.isle line 1676. let expr0_0 = MInst::FpuStoreRev64 { rd: pattern0_0, mem: pattern1_0.clone(), @@ -2497,7 +2497,7 @@ pub fn constructor_load_ext_name_far( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1686. + // Rule at src/isa/s390x/inst.isle line 1681. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::LoadExtNameFar { @@ -2513,7 +2513,7 @@ pub fn constructor_load_ext_name_far( // Generated as internal constructor for term load_addr. pub fn constructor_load_addr(ctx: &mut C, arg0: &MemArg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/s390x/inst.isle line 1693. + // Rule at src/isa/s390x/inst.isle line 1688. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::LoadAddr { @@ -2533,7 +2533,7 @@ pub fn constructor_drop_flags(ctx: &mut C, arg0: &ProducesFlags) -> result: pattern1_1, } = pattern0_0 { - // Rule at src/isa/s390x/inst.isle line 1700. + // Rule at src/isa/s390x/inst.isle line 1695. let expr0_0 = C::emit(ctx, &pattern1_0); return Some(pattern1_1); } @@ -2551,7 +2551,7 @@ pub fn constructor_emit_mov( if pattern0_0 == F32 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1716. + // Rule at src/isa/s390x/inst.isle line 1711. let expr0_0 = MInst::FpuMove32 { rd: pattern2_0, rn: pattern3_0, @@ -2562,7 +2562,7 @@ pub fn constructor_emit_mov( if pattern0_0 == F64 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1719. + // Rule at src/isa/s390x/inst.isle line 1714. let expr0_0 = MInst::FpuMove64 { rd: pattern2_0, rn: pattern3_0, @@ -2573,7 +2573,7 @@ pub fn constructor_emit_mov( if let Some(pattern1_0) = C::gpr32_ty(ctx, pattern0_0) { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1710. + // Rule at src/isa/s390x/inst.isle line 1705. let expr0_0 = MInst::Mov32 { rd: pattern2_0, rm: pattern3_0, @@ -2584,7 +2584,7 @@ pub fn constructor_emit_mov( if let Some(pattern1_0) = C::gpr64_ty(ctx, pattern0_0) { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1713. + // Rule at src/isa/s390x/inst.isle line 1708. let expr0_0 = MInst::Mov64 { rd: pattern2_0, rm: pattern3_0, @@ -2603,7 +2603,7 @@ pub fn constructor_copy_writable_reg( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1724. + // Rule at src/isa/s390x/inst.isle line 1719. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = constructor_emit_mov(ctx, pattern0_0, expr0_0, pattern1_0)?; return Some(expr0_0); @@ -2613,7 +2613,7 @@ pub fn constructor_copy_writable_reg( pub fn constructor_copy_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1731. + // Rule at src/isa/s390x/inst.isle line 1726. let expr0_0 = constructor_copy_writable_reg(ctx, pattern0_0, pattern1_0)?; let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); return Some(expr1_0); @@ -2630,7 +2630,7 @@ pub fn constructor_emit_imm( if pattern0_0 == F32 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1786. + // Rule at src/isa/s390x/inst.isle line 1781. let expr0_0 = C::u64_as_u32(ctx, pattern3_0); let expr1_0 = MInst::LoadFpuConst32 { rd: pattern2_0, @@ -2642,7 +2642,7 @@ pub fn constructor_emit_imm( if pattern0_0 == F64 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1791. + // Rule at src/isa/s390x/inst.isle line 1786. let expr0_0 = MInst::LoadFpuConst64 { rd: pattern2_0, const_data: pattern3_0, @@ -2653,7 +2653,7 @@ pub fn constructor_emit_imm( if let Some(pattern1_0) = C::fits_in_16(ctx, pattern0_0) { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1740. + // Rule at src/isa/s390x/inst.isle line 1735. let expr0_0 = C::u64_as_i16(ctx, pattern3_0); let expr1_0 = MInst::Mov32SImm16 { rd: pattern2_0, @@ -2666,7 +2666,7 @@ pub fn constructor_emit_imm( let pattern2_0 = arg1; let pattern3_0 = arg2; if let Some(pattern4_0) = C::i16_from_u64(ctx, pattern3_0) { - // Rule at src/isa/s390x/inst.isle line 1744. + // Rule at src/isa/s390x/inst.isle line 1739. let expr0_0 = MInst::Mov32SImm16 { rd: pattern2_0, imm: pattern4_0, @@ -2674,7 +2674,7 @@ pub fn constructor_emit_imm( let expr1_0 = C::emit(ctx, &expr0_0); return Some(expr1_0); } - // Rule at src/isa/s390x/inst.isle line 1748. + // Rule at src/isa/s390x/inst.isle line 1743. let expr0_0 = C::u64_as_u32(ctx, pattern3_0); let expr1_0 = MInst::Mov32Imm { rd: pattern2_0, @@ -2688,14 +2688,14 @@ pub fn constructor_emit_imm( let pattern3_0 = arg2; if let Some(pattern4_0) = C::u64_nonzero_hipart(ctx, pattern3_0) { if let Some(pattern5_0) = C::u64_nonzero_lopart(ctx, pattern3_0) { - // Rule at src/isa/s390x/inst.isle line 1768. + // Rule at src/isa/s390x/inst.isle line 1763. let expr0_0 = constructor_emit_imm(ctx, pattern1_0, pattern2_0, pattern4_0)?; let expr1_0 = constructor_emit_insert_imm(ctx, pattern2_0, pattern5_0)?; return Some(expr1_0); } } if let Some(pattern4_0) = C::i16_from_u64(ctx, pattern3_0) { - // Rule at src/isa/s390x/inst.isle line 1752. + // Rule at src/isa/s390x/inst.isle line 1747. let expr0_0 = MInst::Mov64SImm16 { rd: pattern2_0, imm: pattern4_0, @@ -2704,7 +2704,7 @@ pub fn constructor_emit_imm( return Some(expr1_0); } if let Some(pattern4_0) = C::i32_from_u64(ctx, pattern3_0) { - // Rule at src/isa/s390x/inst.isle line 1756. + // Rule at src/isa/s390x/inst.isle line 1751. let expr0_0 = MInst::Mov64SImm32 { rd: pattern2_0, imm: pattern4_0, @@ -2713,7 +2713,7 @@ pub fn constructor_emit_imm( return Some(expr1_0); } if let Some(pattern4_0) = C::uimm32shifted_from_u64(ctx, pattern3_0) { - // Rule at src/isa/s390x/inst.isle line 1764. + // Rule at src/isa/s390x/inst.isle line 1759. let expr0_0 = MInst::Mov64UImm32Shifted { rd: pattern2_0, imm: pattern4_0, @@ -2722,7 +2722,7 @@ pub fn constructor_emit_imm( return Some(expr1_0); } if let Some(pattern4_0) = C::uimm16shifted_from_u64(ctx, pattern3_0) { - // Rule at src/isa/s390x/inst.isle line 1760. + // Rule at src/isa/s390x/inst.isle line 1755. let expr0_0 = MInst::Mov64UImm16Shifted { rd: pattern2_0, imm: pattern4_0, @@ -2743,7 +2743,7 @@ pub fn constructor_emit_insert_imm( let pattern0_0 = arg0; let pattern1_0 = arg1; if let Some(pattern2_0) = C::uimm32shifted_from_u64(ctx, pattern1_0) { - // Rule at src/isa/s390x/inst.isle line 1781. + // Rule at src/isa/s390x/inst.isle line 1776. let expr0_0 = MInst::Insert64UImm32Shifted { rd: pattern0_0, imm: pattern2_0, @@ -2752,7 +2752,7 @@ pub fn constructor_emit_insert_imm( return Some(expr1_0); } if let Some(pattern2_0) = C::uimm16shifted_from_u64(ctx, pattern1_0) { - // Rule at src/isa/s390x/inst.isle line 1777. + // Rule at src/isa/s390x/inst.isle line 1772. let expr0_0 = MInst::Insert64UImm16Shifted { rd: pattern0_0, imm: pattern2_0, @@ -2767,7 +2767,7 @@ pub fn constructor_emit_insert_imm( pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1796. + // Rule at src/isa/s390x/inst.isle line 1791. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = constructor_emit_imm(ctx, pattern0_0, expr0_0, pattern1_0)?; let expr2_0 = C::writable_reg_to_reg(ctx, expr0_0); @@ -2784,7 +2784,7 @@ pub fn constructor_imm_regpair_lo( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1804. + // Rule at src/isa/s390x/inst.isle line 1799. let expr0_0 = constructor_copy_writable_regpair(ctx, pattern2_0)?; let expr1_0 = constructor_writable_regpair_lo(ctx, &expr0_0)?; let expr2_0 = constructor_emit_imm(ctx, pattern0_0, expr1_0, pattern1_0)?; @@ -2802,7 +2802,7 @@ pub fn constructor_imm_regpair_hi( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1812. + // Rule at src/isa/s390x/inst.isle line 1807. let expr0_0 = constructor_copy_writable_regpair(ctx, pattern2_0)?; let expr1_0 = constructor_writable_regpair_hi(ctx, &expr0_0)?; let expr2_0 = constructor_emit_imm(ctx, pattern0_0, expr1_0, pattern1_0)?; @@ -2814,22 +2814,22 @@ pub fn constructor_imm_regpair_hi( pub fn constructor_ty_ext32(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I8 { - // Rule at src/isa/s390x/inst.isle line 1822. + // Rule at src/isa/s390x/inst.isle line 1817. let expr0_0: Type = I32; return Some(expr0_0); } if pattern0_0 == I16 { - // Rule at src/isa/s390x/inst.isle line 1823. + // Rule at src/isa/s390x/inst.isle line 1818. let expr0_0: Type = I32; return Some(expr0_0); } if pattern0_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 1824. + // Rule at src/isa/s390x/inst.isle line 1819. let expr0_0: Type = I32; return Some(expr0_0); } if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 1825. + // Rule at src/isa/s390x/inst.isle line 1820. let expr0_0: Type = I64; return Some(expr0_0); } @@ -2840,22 +2840,22 @@ pub fn constructor_ty_ext32(ctx: &mut C, arg0: Type) -> Option pub fn constructor_ty_ext64(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I8 { - // Rule at src/isa/s390x/inst.isle line 1829. + // Rule at src/isa/s390x/inst.isle line 1824. let expr0_0: Type = I64; return Some(expr0_0); } if pattern0_0 == I16 { - // Rule at src/isa/s390x/inst.isle line 1830. + // Rule at src/isa/s390x/inst.isle line 1825. let expr0_0: Type = I64; return Some(expr0_0); } if pattern0_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 1831. + // Rule at src/isa/s390x/inst.isle line 1826. let expr0_0: Type = I64; return Some(expr0_0); } if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 1832. + // Rule at src/isa/s390x/inst.isle line 1827. let expr0_0: Type = I64; return Some(expr0_0); } @@ -2872,7 +2872,7 @@ pub fn constructor_emit_zext32_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1837. + // Rule at src/isa/s390x/inst.isle line 1832. let expr0_0: bool = false; let expr1_0 = C::ty_bits(ctx, pattern1_0); let expr2_0: u8 = 32; @@ -2897,7 +2897,7 @@ pub fn constructor_emit_sext32_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1843. + // Rule at src/isa/s390x/inst.isle line 1838. let expr0_0: bool = true; let expr1_0 = C::ty_bits(ctx, pattern1_0); let expr2_0: u8 = 32; @@ -2922,7 +2922,7 @@ pub fn constructor_emit_zext64_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1849. + // Rule at src/isa/s390x/inst.isle line 1844. let expr0_0: bool = false; let expr1_0 = C::ty_bits(ctx, pattern1_0); let expr2_0: u8 = 64; @@ -2947,7 +2947,7 @@ pub fn constructor_emit_sext64_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1855. + // Rule at src/isa/s390x/inst.isle line 1850. let expr0_0: bool = true; let expr1_0 = C::ty_bits(ctx, pattern1_0); let expr2_0: u8 = 64; @@ -2966,7 +2966,7 @@ pub fn constructor_emit_sext64_reg( pub fn constructor_zext32_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1861. + // Rule at src/isa/s390x/inst.isle line 1856. let expr0_0: Type = I32; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = constructor_emit_zext32_reg(ctx, expr1_0, pattern0_0, pattern1_0)?; @@ -2978,7 +2978,7 @@ pub fn constructor_zext32_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> pub fn constructor_sext32_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1869. + // Rule at src/isa/s390x/inst.isle line 1864. let expr0_0: Type = I32; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = constructor_emit_sext32_reg(ctx, expr1_0, pattern0_0, pattern1_0)?; @@ -2990,7 +2990,7 @@ pub fn constructor_sext32_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> pub fn constructor_zext64_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1877. + // Rule at src/isa/s390x/inst.isle line 1872. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = constructor_emit_zext64_reg(ctx, expr1_0, pattern0_0, pattern1_0)?; @@ -3002,7 +3002,7 @@ pub fn constructor_zext64_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> pub fn constructor_sext64_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1885. + // Rule at src/isa/s390x/inst.isle line 1880. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = constructor_emit_sext64_reg(ctx, expr1_0, pattern0_0, pattern1_0)?; @@ -3021,7 +3021,7 @@ pub fn constructor_emit_zext32_mem( let pattern1_0 = arg1; if pattern1_0 == I8 { let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1893. + // Rule at src/isa/s390x/inst.isle line 1888. let expr0_0 = MInst::Load32ZExt8 { rd: pattern0_0, mem: pattern3_0.clone(), @@ -3031,7 +3031,7 @@ pub fn constructor_emit_zext32_mem( } if pattern1_0 == I16 { let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1894. + // Rule at src/isa/s390x/inst.isle line 1889. let expr0_0 = MInst::Load32ZExt16 { rd: pattern0_0, mem: pattern3_0.clone(), @@ -3053,7 +3053,7 @@ pub fn constructor_emit_sext32_mem( let pattern1_0 = arg1; if pattern1_0 == I8 { let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1898. + // Rule at src/isa/s390x/inst.isle line 1893. let expr0_0 = MInst::Load32SExt8 { rd: pattern0_0, mem: pattern3_0.clone(), @@ -3063,7 +3063,7 @@ pub fn constructor_emit_sext32_mem( } if pattern1_0 == I16 { let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1899. + // Rule at src/isa/s390x/inst.isle line 1894. let expr0_0 = MInst::Load32SExt16 { rd: pattern0_0, mem: pattern3_0.clone(), @@ -3085,7 +3085,7 @@ pub fn constructor_emit_zext64_mem( let pattern1_0 = arg1; if pattern1_0 == I8 { let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1903. + // Rule at src/isa/s390x/inst.isle line 1898. let expr0_0 = MInst::Load64ZExt8 { rd: pattern0_0, mem: pattern3_0.clone(), @@ -3095,7 +3095,7 @@ pub fn constructor_emit_zext64_mem( } if pattern1_0 == I16 { let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1904. + // Rule at src/isa/s390x/inst.isle line 1899. let expr0_0 = MInst::Load64ZExt16 { rd: pattern0_0, mem: pattern3_0.clone(), @@ -3105,7 +3105,7 @@ pub fn constructor_emit_zext64_mem( } if pattern1_0 == I32 { let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1905. + // Rule at src/isa/s390x/inst.isle line 1900. let expr0_0 = MInst::Load64ZExt32 { rd: pattern0_0, mem: pattern3_0.clone(), @@ -3127,7 +3127,7 @@ pub fn constructor_emit_sext64_mem( let pattern1_0 = arg1; if pattern1_0 == I8 { let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1909. + // Rule at src/isa/s390x/inst.isle line 1904. let expr0_0 = MInst::Load64SExt8 { rd: pattern0_0, mem: pattern3_0.clone(), @@ -3137,7 +3137,7 @@ pub fn constructor_emit_sext64_mem( } if pattern1_0 == I16 { let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1910. + // Rule at src/isa/s390x/inst.isle line 1905. let expr0_0 = MInst::Load64SExt16 { rd: pattern0_0, mem: pattern3_0.clone(), @@ -3147,7 +3147,7 @@ pub fn constructor_emit_sext64_mem( } if pattern1_0 == I32 { let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1911. + // Rule at src/isa/s390x/inst.isle line 1906. let expr0_0 = MInst::Load64SExt32 { rd: pattern0_0, mem: pattern3_0.clone(), @@ -3162,7 +3162,7 @@ pub fn constructor_emit_sext64_mem( pub fn constructor_zext32_mem(ctx: &mut C, arg0: Type, arg1: &MemArg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1915. + // Rule at src/isa/s390x/inst.isle line 1910. let expr0_0: Type = I32; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = constructor_emit_zext32_mem(ctx, expr1_0, pattern0_0, pattern1_0)?; @@ -3174,7 +3174,7 @@ pub fn constructor_zext32_mem(ctx: &mut C, arg0: Type, arg1: &MemArg pub fn constructor_sext32_mem(ctx: &mut C, arg0: Type, arg1: &MemArg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1922. + // Rule at src/isa/s390x/inst.isle line 1917. let expr0_0: Type = I32; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = constructor_emit_sext32_mem(ctx, expr1_0, pattern0_0, pattern1_0)?; @@ -3186,7 +3186,7 @@ pub fn constructor_sext32_mem(ctx: &mut C, arg0: Type, arg1: &MemArg pub fn constructor_zext64_mem(ctx: &mut C, arg0: Type, arg1: &MemArg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1929. + // Rule at src/isa/s390x/inst.isle line 1924. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = constructor_emit_zext64_mem(ctx, expr1_0, pattern0_0, pattern1_0)?; @@ -3198,7 +3198,7 @@ pub fn constructor_zext64_mem(ctx: &mut C, arg0: Type, arg1: &MemArg pub fn constructor_sext64_mem(ctx: &mut C, arg0: Type, arg1: &MemArg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1936. + // Rule at src/isa/s390x/inst.isle line 1931. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = constructor_emit_sext64_mem(ctx, expr1_0, pattern0_0, pattern1_0)?; @@ -3216,7 +3216,7 @@ pub fn constructor_emit_put_in_reg_zext32( let pattern1_0 = arg1; let pattern2_0 = C::value_type(ctx, pattern1_0); if let Some(pattern3_0) = C::u64_from_value(ctx, pattern1_0) { - // Rule at src/isa/s390x/inst.isle line 1944. + // Rule at src/isa/s390x/inst.isle line 1939. let expr0_0 = constructor_ty_ext32(ctx, pattern2_0)?; let expr1_0 = constructor_emit_imm(ctx, expr0_0, pattern0_0, pattern3_0)?; return Some(expr1_0); @@ -3233,7 +3233,7 @@ pub fn constructor_emit_put_in_reg_zext32( { if let &Opcode::Load = &pattern6_0 { if let Some(()) = C::bigendian(ctx, pattern6_2) { - // Rule at src/isa/s390x/inst.isle line 1946. + // Rule at src/isa/s390x/inst.isle line 1941. let expr0_0 = constructor_sink_load(ctx, pattern4_0)?; let expr1_0 = constructor_emit_zext32_mem(ctx, pattern0_0, pattern3_0, &expr0_0)?; @@ -3242,13 +3242,13 @@ pub fn constructor_emit_put_in_reg_zext32( } } } - // Rule at src/isa/s390x/inst.isle line 1948. + // Rule at src/isa/s390x/inst.isle line 1943. let expr0_0 = C::put_in_reg(ctx, pattern1_0); let expr1_0 = constructor_emit_zext32_reg(ctx, pattern0_0, pattern3_0, expr0_0)?; return Some(expr1_0); } if let Some(pattern3_0) = C::ty_32_or_64(ctx, pattern2_0) { - // Rule at src/isa/s390x/inst.isle line 1950. + // Rule at src/isa/s390x/inst.isle line 1945. let expr0_0 = C::put_in_reg(ctx, pattern1_0); let expr1_0 = constructor_emit_mov(ctx, pattern3_0, pattern0_0, expr0_0)?; return Some(expr1_0); @@ -3266,7 +3266,7 @@ pub fn constructor_emit_put_in_reg_sext32( let pattern1_0 = arg1; let pattern2_0 = C::value_type(ctx, pattern1_0); if let Some(pattern3_0) = C::u64_from_signed_value(ctx, pattern1_0) { - // Rule at src/isa/s390x/inst.isle line 1955. + // Rule at src/isa/s390x/inst.isle line 1950. let expr0_0 = constructor_ty_ext32(ctx, pattern2_0)?; let expr1_0 = constructor_emit_imm(ctx, expr0_0, pattern0_0, pattern3_0)?; return Some(expr1_0); @@ -3283,7 +3283,7 @@ pub fn constructor_emit_put_in_reg_sext32( { if let &Opcode::Load = &pattern6_0 { if let Some(()) = C::bigendian(ctx, pattern6_2) { - // Rule at src/isa/s390x/inst.isle line 1957. + // Rule at src/isa/s390x/inst.isle line 1952. let expr0_0 = constructor_sink_load(ctx, pattern4_0)?; let expr1_0 = constructor_emit_sext32_mem(ctx, pattern0_0, pattern3_0, &expr0_0)?; @@ -3292,13 +3292,13 @@ pub fn constructor_emit_put_in_reg_sext32( } } } - // Rule at src/isa/s390x/inst.isle line 1959. + // Rule at src/isa/s390x/inst.isle line 1954. let expr0_0 = C::put_in_reg(ctx, pattern1_0); let expr1_0 = constructor_emit_sext32_reg(ctx, pattern0_0, pattern3_0, expr0_0)?; return Some(expr1_0); } if let Some(pattern3_0) = C::ty_32_or_64(ctx, pattern2_0) { - // Rule at src/isa/s390x/inst.isle line 1961. + // Rule at src/isa/s390x/inst.isle line 1956. let expr0_0 = C::put_in_reg(ctx, pattern1_0); let expr1_0 = constructor_emit_mov(ctx, pattern3_0, pattern0_0, expr0_0)?; return Some(expr1_0); @@ -3316,7 +3316,7 @@ pub fn constructor_emit_put_in_reg_zext64( let pattern1_0 = arg1; let pattern2_0 = C::value_type(ctx, pattern1_0); if let Some(pattern3_0) = C::u64_from_value(ctx, pattern1_0) { - // Rule at src/isa/s390x/inst.isle line 1966. + // Rule at src/isa/s390x/inst.isle line 1961. let expr0_0 = constructor_ty_ext64(ctx, pattern2_0)?; let expr1_0 = constructor_emit_imm(ctx, expr0_0, pattern0_0, pattern3_0)?; return Some(expr1_0); @@ -3333,7 +3333,7 @@ pub fn constructor_emit_put_in_reg_zext64( { if let &Opcode::Load = &pattern6_0 { if let Some(()) = C::bigendian(ctx, pattern6_2) { - // Rule at src/isa/s390x/inst.isle line 1968. + // Rule at src/isa/s390x/inst.isle line 1963. let expr0_0 = constructor_sink_load(ctx, pattern4_0)?; let expr1_0 = constructor_emit_zext64_mem(ctx, pattern0_0, pattern3_0, &expr0_0)?; @@ -3342,13 +3342,13 @@ pub fn constructor_emit_put_in_reg_zext64( } } } - // Rule at src/isa/s390x/inst.isle line 1970. + // Rule at src/isa/s390x/inst.isle line 1965. let expr0_0 = C::put_in_reg(ctx, pattern1_0); let expr1_0 = constructor_emit_zext64_reg(ctx, pattern0_0, pattern3_0, expr0_0)?; return Some(expr1_0); } if let Some(pattern3_0) = C::gpr64_ty(ctx, pattern2_0) { - // Rule at src/isa/s390x/inst.isle line 1972. + // Rule at src/isa/s390x/inst.isle line 1967. let expr0_0 = C::put_in_reg(ctx, pattern1_0); let expr1_0 = constructor_emit_mov(ctx, pattern3_0, pattern0_0, expr0_0)?; return Some(expr1_0); @@ -3366,7 +3366,7 @@ pub fn constructor_emit_put_in_reg_sext64( let pattern1_0 = arg1; let pattern2_0 = C::value_type(ctx, pattern1_0); if let Some(pattern3_0) = C::u64_from_signed_value(ctx, pattern1_0) { - // Rule at src/isa/s390x/inst.isle line 1977. + // Rule at src/isa/s390x/inst.isle line 1972. let expr0_0 = constructor_ty_ext64(ctx, pattern2_0)?; let expr1_0 = constructor_emit_imm(ctx, expr0_0, pattern0_0, pattern3_0)?; return Some(expr1_0); @@ -3383,7 +3383,7 @@ pub fn constructor_emit_put_in_reg_sext64( { if let &Opcode::Load = &pattern6_0 { if let Some(()) = C::bigendian(ctx, pattern6_2) { - // Rule at src/isa/s390x/inst.isle line 1979. + // Rule at src/isa/s390x/inst.isle line 1974. let expr0_0 = constructor_sink_load(ctx, pattern4_0)?; let expr1_0 = constructor_emit_sext64_mem(ctx, pattern0_0, pattern3_0, &expr0_0)?; @@ -3392,13 +3392,13 @@ pub fn constructor_emit_put_in_reg_sext64( } } } - // Rule at src/isa/s390x/inst.isle line 1981. + // Rule at src/isa/s390x/inst.isle line 1976. let expr0_0 = C::put_in_reg(ctx, pattern1_0); let expr1_0 = constructor_emit_sext64_reg(ctx, pattern0_0, pattern3_0, expr0_0)?; return Some(expr1_0); } if let Some(pattern3_0) = C::gpr64_ty(ctx, pattern2_0) { - // Rule at src/isa/s390x/inst.isle line 1983. + // Rule at src/isa/s390x/inst.isle line 1978. let expr0_0 = C::put_in_reg(ctx, pattern1_0); let expr1_0 = constructor_emit_mov(ctx, pattern3_0, pattern0_0, expr0_0)?; return Some(expr1_0); @@ -3411,7 +3411,7 @@ pub fn constructor_put_in_reg_zext32(ctx: &mut C, arg0: Value) -> Op let pattern0_0 = arg0; let pattern1_0 = C::value_type(ctx, pattern0_0); if let Some(pattern2_0) = C::u64_from_value(ctx, pattern0_0) { - // Rule at src/isa/s390x/inst.isle line 1988. + // Rule at src/isa/s390x/inst.isle line 1983. let expr0_0 = constructor_ty_ext32(ctx, pattern1_0)?; let expr1_0 = constructor_imm(ctx, expr0_0, pattern2_0)?; return Some(expr1_0); @@ -3428,7 +3428,7 @@ pub fn constructor_put_in_reg_zext32(ctx: &mut C, arg0: Value) -> Op { if let &Opcode::Load = &pattern5_0 { if let Some(()) = C::bigendian(ctx, pattern5_2) { - // Rule at src/isa/s390x/inst.isle line 1990. + // Rule at src/isa/s390x/inst.isle line 1985. let expr0_0 = constructor_sink_load(ctx, pattern3_0)?; let expr1_0 = constructor_zext32_mem(ctx, pattern2_0, &expr0_0)?; return Some(expr1_0); @@ -3436,13 +3436,13 @@ pub fn constructor_put_in_reg_zext32(ctx: &mut C, arg0: Value) -> Op } } } - // Rule at src/isa/s390x/inst.isle line 1992. + // Rule at src/isa/s390x/inst.isle line 1987. let expr0_0 = C::put_in_reg(ctx, pattern0_0); let expr1_0 = constructor_zext32_reg(ctx, pattern2_0, expr0_0)?; return Some(expr1_0); } if let Some(pattern2_0) = C::ty_32_or_64(ctx, pattern1_0) { - // Rule at src/isa/s390x/inst.isle line 1994. + // Rule at src/isa/s390x/inst.isle line 1989. let expr0_0 = C::put_in_reg(ctx, pattern0_0); return Some(expr0_0); } @@ -3454,7 +3454,7 @@ pub fn constructor_put_in_reg_sext32(ctx: &mut C, arg0: Value) -> Op let pattern0_0 = arg0; let pattern1_0 = C::value_type(ctx, pattern0_0); if let Some(pattern2_0) = C::u64_from_signed_value(ctx, pattern0_0) { - // Rule at src/isa/s390x/inst.isle line 1999. + // Rule at src/isa/s390x/inst.isle line 1994. let expr0_0 = constructor_ty_ext32(ctx, pattern1_0)?; let expr1_0 = constructor_imm(ctx, expr0_0, pattern2_0)?; return Some(expr1_0); @@ -3471,7 +3471,7 @@ pub fn constructor_put_in_reg_sext32(ctx: &mut C, arg0: Value) -> Op { if let &Opcode::Load = &pattern5_0 { if let Some(()) = C::bigendian(ctx, pattern5_2) { - // Rule at src/isa/s390x/inst.isle line 2001. + // Rule at src/isa/s390x/inst.isle line 1996. let expr0_0 = constructor_sink_load(ctx, pattern3_0)?; let expr1_0 = constructor_sext32_mem(ctx, pattern2_0, &expr0_0)?; return Some(expr1_0); @@ -3479,13 +3479,13 @@ pub fn constructor_put_in_reg_sext32(ctx: &mut C, arg0: Value) -> Op } } } - // Rule at src/isa/s390x/inst.isle line 2003. + // Rule at src/isa/s390x/inst.isle line 1998. let expr0_0 = C::put_in_reg(ctx, pattern0_0); let expr1_0 = constructor_sext32_reg(ctx, pattern2_0, expr0_0)?; return Some(expr1_0); } if let Some(pattern2_0) = C::ty_32_or_64(ctx, pattern1_0) { - // Rule at src/isa/s390x/inst.isle line 2005. + // Rule at src/isa/s390x/inst.isle line 2000. let expr0_0 = C::put_in_reg(ctx, pattern0_0); return Some(expr0_0); } @@ -3497,7 +3497,7 @@ pub fn constructor_put_in_reg_zext64(ctx: &mut C, arg0: Value) -> Op let pattern0_0 = arg0; let pattern1_0 = C::value_type(ctx, pattern0_0); if let Some(pattern2_0) = C::u64_from_value(ctx, pattern0_0) { - // Rule at src/isa/s390x/inst.isle line 2010. + // Rule at src/isa/s390x/inst.isle line 2005. let expr0_0 = constructor_ty_ext64(ctx, pattern1_0)?; let expr1_0 = constructor_imm(ctx, expr0_0, pattern2_0)?; return Some(expr1_0); @@ -3514,7 +3514,7 @@ pub fn constructor_put_in_reg_zext64(ctx: &mut C, arg0: Value) -> Op { if let &Opcode::Load = &pattern5_0 { if let Some(()) = C::bigendian(ctx, pattern5_2) { - // Rule at src/isa/s390x/inst.isle line 2012. + // Rule at src/isa/s390x/inst.isle line 2007. let expr0_0 = constructor_sink_load(ctx, pattern3_0)?; let expr1_0 = constructor_zext64_mem(ctx, pattern2_0, &expr0_0)?; return Some(expr1_0); @@ -3522,13 +3522,13 @@ pub fn constructor_put_in_reg_zext64(ctx: &mut C, arg0: Value) -> Op } } } - // Rule at src/isa/s390x/inst.isle line 2014. + // Rule at src/isa/s390x/inst.isle line 2009. let expr0_0 = C::put_in_reg(ctx, pattern0_0); let expr1_0 = constructor_zext64_reg(ctx, pattern2_0, expr0_0)?; return Some(expr1_0); } if let Some(pattern2_0) = C::gpr64_ty(ctx, pattern1_0) { - // Rule at src/isa/s390x/inst.isle line 2016. + // Rule at src/isa/s390x/inst.isle line 2011. let expr0_0 = C::put_in_reg(ctx, pattern0_0); return Some(expr0_0); } @@ -3540,7 +3540,7 @@ pub fn constructor_put_in_reg_sext64(ctx: &mut C, arg0: Value) -> Op let pattern0_0 = arg0; let pattern1_0 = C::value_type(ctx, pattern0_0); if let Some(pattern2_0) = C::u64_from_signed_value(ctx, pattern0_0) { - // Rule at src/isa/s390x/inst.isle line 2021. + // Rule at src/isa/s390x/inst.isle line 2016. let expr0_0 = constructor_ty_ext64(ctx, pattern1_0)?; let expr1_0 = constructor_imm(ctx, expr0_0, pattern2_0)?; return Some(expr1_0); @@ -3557,7 +3557,7 @@ pub fn constructor_put_in_reg_sext64(ctx: &mut C, arg0: Value) -> Op { if let &Opcode::Load = &pattern5_0 { if let Some(()) = C::bigendian(ctx, pattern5_2) { - // Rule at src/isa/s390x/inst.isle line 2023. + // Rule at src/isa/s390x/inst.isle line 2018. let expr0_0 = constructor_sink_load(ctx, pattern3_0)?; let expr1_0 = constructor_sext64_mem(ctx, pattern2_0, &expr0_0)?; return Some(expr1_0); @@ -3565,13 +3565,13 @@ pub fn constructor_put_in_reg_sext64(ctx: &mut C, arg0: Value) -> Op } } } - // Rule at src/isa/s390x/inst.isle line 2025. + // Rule at src/isa/s390x/inst.isle line 2020. let expr0_0 = C::put_in_reg(ctx, pattern0_0); let expr1_0 = constructor_sext64_reg(ctx, pattern2_0, expr0_0)?; return Some(expr1_0); } if let Some(pattern2_0) = C::gpr64_ty(ctx, pattern1_0) { - // Rule at src/isa/s390x/inst.isle line 2027. + // Rule at src/isa/s390x/inst.isle line 2022. let expr0_0 = C::put_in_reg(ctx, pattern0_0); return Some(expr0_0); } @@ -3586,7 +3586,7 @@ pub fn constructor_put_in_regpair_lo_zext32( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2033. + // Rule at src/isa/s390x/inst.isle line 2028. let expr0_0 = constructor_copy_writable_regpair(ctx, pattern1_0)?; let expr1_0 = constructor_writable_regpair_lo(ctx, &expr0_0)?; let expr2_0 = constructor_emit_put_in_reg_zext32(ctx, expr1_0, pattern0_0)?; @@ -3602,7 +3602,7 @@ pub fn constructor_put_in_regpair_lo_sext32( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2041. + // Rule at src/isa/s390x/inst.isle line 2036. let expr0_0 = constructor_copy_writable_regpair(ctx, pattern1_0)?; let expr1_0 = constructor_writable_regpair_lo(ctx, &expr0_0)?; let expr2_0 = constructor_emit_put_in_reg_sext32(ctx, expr1_0, pattern0_0)?; @@ -3618,7 +3618,7 @@ pub fn constructor_put_in_regpair_lo_zext64( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2049. + // Rule at src/isa/s390x/inst.isle line 2044. let expr0_0 = constructor_copy_writable_regpair(ctx, pattern1_0)?; let expr1_0 = constructor_writable_regpair_lo(ctx, &expr0_0)?; let expr2_0 = constructor_emit_put_in_reg_zext64(ctx, expr1_0, pattern0_0)?; @@ -3634,7 +3634,7 @@ pub fn constructor_put_in_regpair_lo_sext64( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2057. + // Rule at src/isa/s390x/inst.isle line 2052. let expr0_0 = constructor_copy_writable_regpair(ctx, pattern1_0)?; let expr1_0 = constructor_writable_regpair_lo(ctx, &expr0_0)?; let expr2_0 = constructor_emit_put_in_reg_sext64(ctx, expr1_0, pattern0_0)?; @@ -3655,7 +3655,7 @@ pub fn constructor_emit_cmov_imm( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 2067. + // Rule at src/isa/s390x/inst.isle line 2062. let expr0_0 = MInst::CMov32SImm16 { rd: pattern2_0, cond: pattern3_0.clone(), @@ -3672,7 +3672,7 @@ pub fn constructor_emit_cmov_imm( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 2070. + // Rule at src/isa/s390x/inst.isle line 2065. let expr0_0 = MInst::CMov64SImm16 { rd: pattern2_0, cond: pattern3_0.clone(), @@ -3700,7 +3700,7 @@ pub fn constructor_cmov_imm( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 2076. + // Rule at src/isa/s390x/inst.isle line 2071. let expr0_0 = constructor_copy_writable_reg(ctx, pattern0_0, pattern3_0)?; let expr1_0 = constructor_emit_cmov_imm(ctx, pattern0_0, expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -3720,7 +3720,7 @@ pub fn constructor_cmov_imm_regpair_lo( let pattern2_0 = arg2; let pattern3_0 = arg3; let pattern4_0 = arg4; - // Rule at src/isa/s390x/inst.isle line 2083. + // Rule at src/isa/s390x/inst.isle line 2078. let expr0_0 = constructor_copy_writable_regpair(ctx, pattern4_0)?; let expr1_0 = constructor_writable_regpair_lo(ctx, &expr0_0)?; let expr2_0 = constructor_emit_cmov_imm(ctx, pattern0_0, expr1_0, pattern2_0, pattern3_0)?; @@ -3743,7 +3743,7 @@ pub fn constructor_cmov_imm_regpair_hi( let pattern2_0 = arg2; let pattern3_0 = arg3; let pattern4_0 = arg4; - // Rule at src/isa/s390x/inst.isle line 2092. + // Rule at src/isa/s390x/inst.isle line 2087. let expr0_0 = constructor_copy_writable_regpair(ctx, pattern4_0)?; let expr1_0 = constructor_writable_regpair_hi(ctx, &expr0_0)?; let expr2_0 = constructor_emit_cmov_imm(ctx, pattern0_0, expr1_0, pattern2_0, pattern3_0)?; @@ -3765,7 +3765,7 @@ pub fn constructor_emit_cmov_reg( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 2106. + // Rule at src/isa/s390x/inst.isle line 2101. let expr0_0 = MInst::FpuCMov32 { rd: pattern2_0, cond: pattern3_0.clone(), @@ -3782,7 +3782,7 @@ pub fn constructor_emit_cmov_reg( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 2109. + // Rule at src/isa/s390x/inst.isle line 2104. let expr0_0 = MInst::FpuCMov64 { rd: pattern2_0, cond: pattern3_0.clone(), @@ -3799,7 +3799,7 @@ pub fn constructor_emit_cmov_reg( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 2100. + // Rule at src/isa/s390x/inst.isle line 2095. let expr0_0 = MInst::CMov32 { rd: pattern2_0, cond: pattern3_0.clone(), @@ -3816,7 +3816,7 @@ pub fn constructor_emit_cmov_reg( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 2103. + // Rule at src/isa/s390x/inst.isle line 2098. let expr0_0 = MInst::CMov64 { rd: pattern2_0, cond: pattern3_0.clone(), @@ -3844,7 +3844,7 @@ pub fn constructor_cmov_reg( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 2115. + // Rule at src/isa/s390x/inst.isle line 2110. let expr0_0 = constructor_copy_writable_reg(ctx, pattern0_0, pattern3_0)?; let expr1_0 = constructor_emit_cmov_reg(ctx, pattern0_0, expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -3865,7 +3865,7 @@ pub fn constructor_trap_if( { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2123. + // Rule at src/isa/s390x/inst.isle line 2118. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = MInst::TrapIf { cond: pattern2_0.clone(), @@ -3891,7 +3891,7 @@ pub fn constructor_icmps_reg_and_trap( let pattern2_0 = arg2; let pattern3_0 = arg3; let pattern4_0 = arg4; - // Rule at src/isa/s390x/inst.isle line 2129. + // Rule at src/isa/s390x/inst.isle line 2124. let expr0_0 = constructor_cmpop_cmps(ctx, pattern0_0)?; let expr1_0 = MInst::CmpTrapRR { op: expr0_0, @@ -3919,7 +3919,7 @@ pub fn constructor_icmps_simm16_and_trap( let pattern2_0 = arg2; let pattern3_0 = arg3; let pattern4_0 = arg4; - // Rule at src/isa/s390x/inst.isle line 2135. + // Rule at src/isa/s390x/inst.isle line 2130. let expr0_0 = constructor_cmpop_cmps(ctx, pattern0_0)?; let expr1_0 = MInst::CmpTrapRSImm16 { op: expr0_0, @@ -3947,7 +3947,7 @@ pub fn constructor_icmpu_reg_and_trap( let pattern2_0 = arg2; let pattern3_0 = arg3; let pattern4_0 = arg4; - // Rule at src/isa/s390x/inst.isle line 2141. + // Rule at src/isa/s390x/inst.isle line 2136. let expr0_0 = constructor_cmpop_cmpu(ctx, pattern0_0)?; let expr1_0 = MInst::CmpTrapRR { op: expr0_0, @@ -3975,7 +3975,7 @@ pub fn constructor_icmpu_uimm16_and_trap( let pattern2_0 = arg2; let pattern3_0 = arg3; let pattern4_0 = arg4; - // Rule at src/isa/s390x/inst.isle line 2147. + // Rule at src/isa/s390x/inst.isle line 2142. let expr0_0 = constructor_cmpop_cmpu(ctx, pattern0_0)?; let expr1_0 = MInst::CmpTrapRUImm16 { op: expr0_0, @@ -3997,7 +3997,7 @@ pub fn constructor_bool( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2160. + // Rule at src/isa/s390x/inst.isle line 2155. let expr0_0 = ProducesBool::ProducesBool { producer: pattern0_0.clone(), cond: pattern1_0.clone(), @@ -4016,7 +4016,7 @@ pub fn constructor_invert_bool( cond: ref pattern1_1, } = pattern0_0 { - // Rule at src/isa/s390x/inst.isle line 2164. + // Rule at src/isa/s390x/inst.isle line 2159. let expr0_0 = C::invert_cond(ctx, &pattern1_1); let expr1_0 = constructor_bool(ctx, &pattern1_0, &expr0_0)?; return Some(expr1_0); @@ -4032,7 +4032,7 @@ pub fn constructor_emit_producer(ctx: &mut C, arg0: &ProducesFlags) result: pattern1_1, } = pattern0_0 { - // Rule at src/isa/s390x/inst.isle line 2173. + // Rule at src/isa/s390x/inst.isle line 2168. let expr0_0 = C::emit(ctx, &pattern1_0); return Some(expr0_0); } @@ -4047,7 +4047,7 @@ pub fn constructor_emit_consumer(ctx: &mut C, arg0: &ConsumesFlags) result: pattern1_1, } = pattern0_0 { - // Rule at src/isa/s390x/inst.isle line 2175. + // Rule at src/isa/s390x/inst.isle line 2170. let expr0_0 = C::emit(ctx, &pattern1_0); return Some(expr0_0); } @@ -4071,7 +4071,7 @@ pub fn constructor_select_bool_reg( { let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 2179. + // Rule at src/isa/s390x/inst.isle line 2174. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = constructor_emit_producer(ctx, &pattern2_0)?; let expr2_0 = constructor_emit_mov(ctx, pattern0_0, expr0_0, pattern4_0)?; @@ -4100,7 +4100,7 @@ pub fn constructor_select_bool_imm( { let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 2188. + // Rule at src/isa/s390x/inst.isle line 2183. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = constructor_emit_producer(ctx, &pattern2_0)?; let expr2_0 = constructor_emit_imm(ctx, pattern0_0, expr0_0, pattern4_0)?; @@ -4121,7 +4121,7 @@ pub fn constructor_lower_bool( let pattern0_0 = arg0; if pattern0_0 == B1 { let pattern2_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2198. + // Rule at src/isa/s390x/inst.isle line 2193. let expr0_0: Type = B1; let expr1_0: i16 = 1; let expr2_0: u64 = 0; @@ -4130,7 +4130,7 @@ pub fn constructor_lower_bool( } if pattern0_0 == B8 { let pattern2_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2199. + // Rule at src/isa/s390x/inst.isle line 2194. let expr0_0: Type = B8; let expr1_0: i16 = -1; let expr2_0: u64 = 0; @@ -4139,7 +4139,7 @@ pub fn constructor_lower_bool( } if pattern0_0 == B16 { let pattern2_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2200. + // Rule at src/isa/s390x/inst.isle line 2195. let expr0_0: Type = B16; let expr1_0: i16 = -1; let expr2_0: u64 = 0; @@ -4148,7 +4148,7 @@ pub fn constructor_lower_bool( } if pattern0_0 == B32 { let pattern2_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2201. + // Rule at src/isa/s390x/inst.isle line 2196. let expr0_0: Type = B32; let expr1_0: i16 = -1; let expr2_0: u64 = 0; @@ -4157,7 +4157,7 @@ pub fn constructor_lower_bool( } if pattern0_0 == B64 { let pattern2_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2202. + // Rule at src/isa/s390x/inst.isle line 2197. let expr0_0: Type = B64; let expr1_0: i16 = -1; let expr2_0: u64 = 0; @@ -4172,7 +4172,7 @@ pub fn constructor_clz_reg(ctx: &mut C, arg0: i16, arg1: Reg) -> Opt let pattern0_0 = arg0; if pattern0_0 == 64 { let pattern2_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2211. + // Rule at src/isa/s390x/inst.isle line 2206. let expr0_0 = constructor_temp_writable_regpair(ctx)?; let expr1_0 = MInst::Flogr { rn: pattern2_0 }; let expr2_0 = C::emit(ctx, &expr1_0); @@ -4180,7 +4180,7 @@ pub fn constructor_clz_reg(ctx: &mut C, arg0: i16, arg1: Reg) -> Opt return Some(expr3_0); } let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2220. + // Rule at src/isa/s390x/inst.isle line 2215. let expr0_0 = constructor_temp_writable_regpair(ctx)?; let expr1_0 = MInst::Flogr { rn: pattern1_0 }; let expr2_0 = C::emit(ctx, &expr1_0); @@ -4201,22 +4201,22 @@ pub fn constructor_clz_reg(ctx: &mut C, arg0: i16, arg1: Reg) -> Opt pub fn constructor_aluop_add(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I8 { - // Rule at src/isa/s390x/inst.isle line 2231. + // Rule at src/isa/s390x/inst.isle line 2226. let expr0_0 = ALUOp::Add32; return Some(expr0_0); } if pattern0_0 == I16 { - // Rule at src/isa/s390x/inst.isle line 2232. + // Rule at src/isa/s390x/inst.isle line 2227. let expr0_0 = ALUOp::Add32; return Some(expr0_0); } if pattern0_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 2233. + // Rule at src/isa/s390x/inst.isle line 2228. let expr0_0 = ALUOp::Add32; return Some(expr0_0); } if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2234. + // Rule at src/isa/s390x/inst.isle line 2229. let expr0_0 = ALUOp::Add64; return Some(expr0_0); } @@ -4227,17 +4227,17 @@ pub fn constructor_aluop_add(ctx: &mut C, arg0: Type) -> Option(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I16 { - // Rule at src/isa/s390x/inst.isle line 2237. + // Rule at src/isa/s390x/inst.isle line 2232. let expr0_0 = ALUOp::Add32Ext16; return Some(expr0_0); } if pattern0_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 2238. + // Rule at src/isa/s390x/inst.isle line 2233. let expr0_0 = ALUOp::Add32Ext16; return Some(expr0_0); } if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2239. + // Rule at src/isa/s390x/inst.isle line 2234. let expr0_0 = ALUOp::Add64Ext16; return Some(expr0_0); } @@ -4248,7 +4248,7 @@ pub fn constructor_aluop_add_sext16(ctx: &mut C, arg0: Type) -> Opti pub fn constructor_aluop_add_sext32(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2242. + // Rule at src/isa/s390x/inst.isle line 2237. let expr0_0 = ALUOp::Add64Ext32; return Some(expr0_0); } @@ -4265,7 +4265,7 @@ pub fn constructor_add_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2245. + // Rule at src/isa/s390x/inst.isle line 2240. let expr0_0 = constructor_aluop_add(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rrr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4281,7 +4281,7 @@ pub fn constructor_add_reg_sext32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2248. + // Rule at src/isa/s390x/inst.isle line 2243. let expr0_0 = constructor_aluop_add_sext32(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4297,7 +4297,7 @@ pub fn constructor_add_simm16( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2251. + // Rule at src/isa/s390x/inst.isle line 2246. let expr0_0 = constructor_aluop_add(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rrsimm16(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4313,7 +4313,7 @@ pub fn constructor_add_simm32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2254. + // Rule at src/isa/s390x/inst.isle line 2249. let expr0_0 = constructor_aluop_add(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rsimm32(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4329,7 +4329,7 @@ pub fn constructor_add_mem( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2257. + // Rule at src/isa/s390x/inst.isle line 2252. let expr0_0 = constructor_aluop_add(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rx(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4345,7 +4345,7 @@ pub fn constructor_add_mem_sext16( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2260. + // Rule at src/isa/s390x/inst.isle line 2255. let expr0_0 = constructor_aluop_add_sext16(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rx(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4361,7 +4361,7 @@ pub fn constructor_add_mem_sext32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2263. + // Rule at src/isa/s390x/inst.isle line 2258. let expr0_0 = constructor_aluop_add_sext32(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rx(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4371,12 +4371,12 @@ pub fn constructor_add_mem_sext32( pub fn constructor_aluop_add_logical(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 2269. + // Rule at src/isa/s390x/inst.isle line 2264. let expr0_0 = ALUOp::AddLogical32; return Some(expr0_0); } if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2270. + // Rule at src/isa/s390x/inst.isle line 2265. let expr0_0 = ALUOp::AddLogical64; return Some(expr0_0); } @@ -4387,7 +4387,7 @@ pub fn constructor_aluop_add_logical(ctx: &mut C, arg0: Type) -> Opt pub fn constructor_aluop_add_logical_zext32(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2273. + // Rule at src/isa/s390x/inst.isle line 2268. let expr0_0 = ALUOp::AddLogical64Ext32; return Some(expr0_0); } @@ -4404,7 +4404,7 @@ pub fn constructor_add_logical_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2276. + // Rule at src/isa/s390x/inst.isle line 2271. let expr0_0 = constructor_aluop_add_logical(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rrr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4420,7 +4420,7 @@ pub fn constructor_add_logical_reg_zext32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2279. + // Rule at src/isa/s390x/inst.isle line 2274. let expr0_0 = constructor_aluop_add_logical_zext32(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4436,7 +4436,7 @@ pub fn constructor_add_logical_zimm32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2282. + // Rule at src/isa/s390x/inst.isle line 2277. let expr0_0 = constructor_aluop_add_logical(ctx, pattern0_0)?; let expr1_0 = constructor_alu_ruimm32(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4452,7 +4452,7 @@ pub fn constructor_add_logical_mem( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2285. + // Rule at src/isa/s390x/inst.isle line 2280. let expr0_0 = constructor_aluop_add_logical(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rx(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4468,7 +4468,7 @@ pub fn constructor_add_logical_mem_zext32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2288. + // Rule at src/isa/s390x/inst.isle line 2283. let expr0_0 = constructor_aluop_add_logical_zext32(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rx(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4478,22 +4478,22 @@ pub fn constructor_add_logical_mem_zext32( pub fn constructor_aluop_sub(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I8 { - // Rule at src/isa/s390x/inst.isle line 2294. + // Rule at src/isa/s390x/inst.isle line 2289. let expr0_0 = ALUOp::Sub32; return Some(expr0_0); } if pattern0_0 == I16 { - // Rule at src/isa/s390x/inst.isle line 2295. + // Rule at src/isa/s390x/inst.isle line 2290. let expr0_0 = ALUOp::Sub32; return Some(expr0_0); } if pattern0_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 2296. + // Rule at src/isa/s390x/inst.isle line 2291. let expr0_0 = ALUOp::Sub32; return Some(expr0_0); } if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2297. + // Rule at src/isa/s390x/inst.isle line 2292. let expr0_0 = ALUOp::Sub64; return Some(expr0_0); } @@ -4504,17 +4504,17 @@ pub fn constructor_aluop_sub(ctx: &mut C, arg0: Type) -> Option(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I16 { - // Rule at src/isa/s390x/inst.isle line 2300. + // Rule at src/isa/s390x/inst.isle line 2295. let expr0_0 = ALUOp::Sub32Ext16; return Some(expr0_0); } if pattern0_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 2301. + // Rule at src/isa/s390x/inst.isle line 2296. let expr0_0 = ALUOp::Sub32Ext16; return Some(expr0_0); } if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2302. + // Rule at src/isa/s390x/inst.isle line 2297. let expr0_0 = ALUOp::Sub64Ext16; return Some(expr0_0); } @@ -4525,7 +4525,7 @@ pub fn constructor_aluop_sub_sext16(ctx: &mut C, arg0: Type) -> Opti pub fn constructor_aluop_sub_sext32(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2305. + // Rule at src/isa/s390x/inst.isle line 2300. let expr0_0 = ALUOp::Sub64Ext32; return Some(expr0_0); } @@ -4542,7 +4542,7 @@ pub fn constructor_sub_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2308. + // Rule at src/isa/s390x/inst.isle line 2303. let expr0_0 = constructor_aluop_sub(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rrr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4558,7 +4558,7 @@ pub fn constructor_sub_reg_sext32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2311. + // Rule at src/isa/s390x/inst.isle line 2306. let expr0_0 = constructor_aluop_sub_sext32(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4574,7 +4574,7 @@ pub fn constructor_sub_mem( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2314. + // Rule at src/isa/s390x/inst.isle line 2309. let expr0_0 = constructor_aluop_sub(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rx(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4590,7 +4590,7 @@ pub fn constructor_sub_mem_sext16( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2317. + // Rule at src/isa/s390x/inst.isle line 2312. let expr0_0 = constructor_aluop_sub_sext16(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rx(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4606,7 +4606,7 @@ pub fn constructor_sub_mem_sext32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2320. + // Rule at src/isa/s390x/inst.isle line 2315. let expr0_0 = constructor_aluop_sub_sext32(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rx(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4616,12 +4616,12 @@ pub fn constructor_sub_mem_sext32( pub fn constructor_aluop_sub_logical(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 2326. + // Rule at src/isa/s390x/inst.isle line 2321. let expr0_0 = ALUOp::SubLogical32; return Some(expr0_0); } if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2327. + // Rule at src/isa/s390x/inst.isle line 2322. let expr0_0 = ALUOp::SubLogical64; return Some(expr0_0); } @@ -4632,7 +4632,7 @@ pub fn constructor_aluop_sub_logical(ctx: &mut C, arg0: Type) -> Opt pub fn constructor_aluop_sub_logical_zext32(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2330. + // Rule at src/isa/s390x/inst.isle line 2325. let expr0_0 = ALUOp::SubLogical64Ext32; return Some(expr0_0); } @@ -4649,7 +4649,7 @@ pub fn constructor_sub_logical_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2333. + // Rule at src/isa/s390x/inst.isle line 2328. let expr0_0 = constructor_aluop_sub_logical(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rrr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4665,7 +4665,7 @@ pub fn constructor_sub_logical_reg_zext32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2336. + // Rule at src/isa/s390x/inst.isle line 2331. let expr0_0 = constructor_aluop_sub_logical_zext32(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4681,7 +4681,7 @@ pub fn constructor_sub_logical_zimm32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2339. + // Rule at src/isa/s390x/inst.isle line 2334. let expr0_0 = constructor_aluop_sub_logical(ctx, pattern0_0)?; let expr1_0 = constructor_alu_ruimm32(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4697,7 +4697,7 @@ pub fn constructor_sub_logical_mem( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2342. + // Rule at src/isa/s390x/inst.isle line 2337. let expr0_0 = constructor_aluop_sub_logical(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rx(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4713,7 +4713,7 @@ pub fn constructor_sub_logical_mem_zext32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2345. + // Rule at src/isa/s390x/inst.isle line 2340. let expr0_0 = constructor_aluop_sub_logical(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rx(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4723,22 +4723,22 @@ pub fn constructor_sub_logical_mem_zext32( pub fn constructor_aluop_mul(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I8 { - // Rule at src/isa/s390x/inst.isle line 2351. + // Rule at src/isa/s390x/inst.isle line 2346. let expr0_0 = ALUOp::Mul32; return Some(expr0_0); } if pattern0_0 == I16 { - // Rule at src/isa/s390x/inst.isle line 2352. + // Rule at src/isa/s390x/inst.isle line 2347. let expr0_0 = ALUOp::Mul32; return Some(expr0_0); } if pattern0_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 2353. + // Rule at src/isa/s390x/inst.isle line 2348. let expr0_0 = ALUOp::Mul32; return Some(expr0_0); } if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2354. + // Rule at src/isa/s390x/inst.isle line 2349. let expr0_0 = ALUOp::Mul64; return Some(expr0_0); } @@ -4749,17 +4749,17 @@ pub fn constructor_aluop_mul(ctx: &mut C, arg0: Type) -> Option(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I16 { - // Rule at src/isa/s390x/inst.isle line 2357. + // Rule at src/isa/s390x/inst.isle line 2352. let expr0_0 = ALUOp::Mul32Ext16; return Some(expr0_0); } if pattern0_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 2358. + // Rule at src/isa/s390x/inst.isle line 2353. let expr0_0 = ALUOp::Mul32Ext16; return Some(expr0_0); } if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2359. + // Rule at src/isa/s390x/inst.isle line 2354. let expr0_0 = ALUOp::Mul64Ext16; return Some(expr0_0); } @@ -4770,7 +4770,7 @@ pub fn constructor_aluop_mul_sext16(ctx: &mut C, arg0: Type) -> Opti pub fn constructor_aluop_mul_sext32(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2362. + // Rule at src/isa/s390x/inst.isle line 2357. let expr0_0 = ALUOp::Mul64Ext32; return Some(expr0_0); } @@ -4787,7 +4787,7 @@ pub fn constructor_mul_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2365. + // Rule at src/isa/s390x/inst.isle line 2360. let expr0_0 = constructor_aluop_mul(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rrr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4803,7 +4803,7 @@ pub fn constructor_mul_reg_sext32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2368. + // Rule at src/isa/s390x/inst.isle line 2363. let expr0_0 = constructor_aluop_mul_sext32(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4819,7 +4819,7 @@ pub fn constructor_mul_simm16( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2371. + // Rule at src/isa/s390x/inst.isle line 2366. let expr0_0 = constructor_aluop_mul(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rsimm16(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4835,7 +4835,7 @@ pub fn constructor_mul_simm32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2374. + // Rule at src/isa/s390x/inst.isle line 2369. let expr0_0 = constructor_aluop_mul(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rsimm32(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4851,7 +4851,7 @@ pub fn constructor_mul_mem( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2377. + // Rule at src/isa/s390x/inst.isle line 2372. let expr0_0 = constructor_aluop_mul(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rx(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4867,7 +4867,7 @@ pub fn constructor_mul_mem_sext16( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2380. + // Rule at src/isa/s390x/inst.isle line 2375. let expr0_0 = constructor_aluop_mul_sext16(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rx(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4883,7 +4883,7 @@ pub fn constructor_mul_mem_sext32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2383. + // Rule at src/isa/s390x/inst.isle line 2378. let expr0_0 = constructor_aluop_mul_sext32(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rx(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4900,14 +4900,14 @@ pub fn constructor_udivmod( if pattern0_0 == I32 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2389. + // Rule at src/isa/s390x/inst.isle line 2384. let expr0_0 = constructor_udivmod32(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if pattern0_0 == I64 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2390. + // Rule at src/isa/s390x/inst.isle line 2385. let expr0_0 = constructor_udivmod64(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } @@ -4925,14 +4925,14 @@ pub fn constructor_sdivmod( if pattern0_0 == I32 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2396. + // Rule at src/isa/s390x/inst.isle line 2391. let expr0_0 = constructor_sdivmod32(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if pattern0_0 == I64 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2397. + // Rule at src/isa/s390x/inst.isle line 2392. let expr0_0 = constructor_sdivmod64(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } @@ -4943,12 +4943,12 @@ pub fn constructor_sdivmod( pub fn constructor_aluop_and(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if let Some(pattern1_0) = C::gpr32_ty(ctx, pattern0_0) { - // Rule at src/isa/s390x/inst.isle line 2403. + // Rule at src/isa/s390x/inst.isle line 2398. let expr0_0 = ALUOp::And32; return Some(expr0_0); } if let Some(pattern1_0) = C::gpr64_ty(ctx, pattern0_0) { - // Rule at src/isa/s390x/inst.isle line 2404. + // Rule at src/isa/s390x/inst.isle line 2399. let expr0_0 = ALUOp::And64; return Some(expr0_0); } @@ -4965,7 +4965,7 @@ pub fn constructor_and_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2407. + // Rule at src/isa/s390x/inst.isle line 2402. let expr0_0 = constructor_aluop_and(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rrr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4981,7 +4981,7 @@ pub fn constructor_and_uimm16shifted( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2410. + // Rule at src/isa/s390x/inst.isle line 2405. let expr0_0 = constructor_aluop_and(ctx, pattern0_0)?; let expr1_0 = constructor_alu_ruimm16shifted(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; @@ -4998,7 +4998,7 @@ pub fn constructor_and_uimm32shifted( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2413. + // Rule at src/isa/s390x/inst.isle line 2408. let expr0_0 = constructor_aluop_and(ctx, pattern0_0)?; let expr1_0 = constructor_alu_ruimm32shifted(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; @@ -5015,7 +5015,7 @@ pub fn constructor_and_mem( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2416. + // Rule at src/isa/s390x/inst.isle line 2411. let expr0_0 = constructor_aluop_and(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rx(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -5025,12 +5025,12 @@ pub fn constructor_and_mem( pub fn constructor_aluop_or(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if let Some(pattern1_0) = C::gpr32_ty(ctx, pattern0_0) { - // Rule at src/isa/s390x/inst.isle line 2422. + // Rule at src/isa/s390x/inst.isle line 2417. let expr0_0 = ALUOp::Orr32; return Some(expr0_0); } if let Some(pattern1_0) = C::gpr64_ty(ctx, pattern0_0) { - // Rule at src/isa/s390x/inst.isle line 2423. + // Rule at src/isa/s390x/inst.isle line 2418. let expr0_0 = ALUOp::Orr64; return Some(expr0_0); } @@ -5047,7 +5047,7 @@ pub fn constructor_or_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2426. + // Rule at src/isa/s390x/inst.isle line 2421. let expr0_0 = constructor_aluop_or(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rrr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -5063,7 +5063,7 @@ pub fn constructor_or_uimm16shifted( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2429. + // Rule at src/isa/s390x/inst.isle line 2424. let expr0_0 = constructor_aluop_or(ctx, pattern0_0)?; let expr1_0 = constructor_alu_ruimm16shifted(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; @@ -5080,7 +5080,7 @@ pub fn constructor_or_uimm32shifted( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2432. + // Rule at src/isa/s390x/inst.isle line 2427. let expr0_0 = constructor_aluop_or(ctx, pattern0_0)?; let expr1_0 = constructor_alu_ruimm32shifted(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; @@ -5097,7 +5097,7 @@ pub fn constructor_or_mem( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2435. + // Rule at src/isa/s390x/inst.isle line 2430. let expr0_0 = constructor_aluop_or(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rx(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -5107,12 +5107,12 @@ pub fn constructor_or_mem( pub fn constructor_aluop_xor(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if let Some(pattern1_0) = C::gpr32_ty(ctx, pattern0_0) { - // Rule at src/isa/s390x/inst.isle line 2441. + // Rule at src/isa/s390x/inst.isle line 2436. let expr0_0 = ALUOp::Xor32; return Some(expr0_0); } if let Some(pattern1_0) = C::gpr64_ty(ctx, pattern0_0) { - // Rule at src/isa/s390x/inst.isle line 2442. + // Rule at src/isa/s390x/inst.isle line 2437. let expr0_0 = ALUOp::Xor64; return Some(expr0_0); } @@ -5129,7 +5129,7 @@ pub fn constructor_xor_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2445. + // Rule at src/isa/s390x/inst.isle line 2440. let expr0_0 = constructor_aluop_xor(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rrr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -5145,7 +5145,7 @@ pub fn constructor_xor_uimm32shifted( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2448. + // Rule at src/isa/s390x/inst.isle line 2443. let expr0_0 = constructor_aluop_xor(ctx, pattern0_0)?; let expr1_0 = constructor_alu_ruimm32shifted(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; @@ -5162,7 +5162,7 @@ pub fn constructor_xor_mem( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2451. + // Rule at src/isa/s390x/inst.isle line 2446. let expr0_0 = constructor_aluop_xor(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rx(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -5173,7 +5173,7 @@ pub fn constructor_not_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> Op let pattern0_0 = arg0; if let Some(pattern1_0) = C::gpr32_ty(ctx, pattern0_0) { let pattern2_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2457. + // Rule at src/isa/s390x/inst.isle line 2452. let expr0_0: u32 = 4294967295; let expr1_0: u8 = 0; let expr2_0 = C::uimm32shifted(ctx, expr0_0, expr1_0); @@ -5182,7 +5182,7 @@ pub fn constructor_not_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> Op } if let Some(pattern1_0) = C::gpr64_ty(ctx, pattern0_0) { let pattern2_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2459. + // Rule at src/isa/s390x/inst.isle line 2454. let expr0_0: u32 = 4294967295; let expr1_0: u8 = 0; let expr2_0 = C::uimm32shifted(ctx, expr0_0, expr1_0); @@ -5200,12 +5200,12 @@ pub fn constructor_not_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> Op pub fn constructor_aluop_and_not(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if let Some(pattern1_0) = C::gpr32_ty(ctx, pattern0_0) { - // Rule at src/isa/s390x/inst.isle line 2468. + // Rule at src/isa/s390x/inst.isle line 2463. let expr0_0 = ALUOp::AndNot32; return Some(expr0_0); } if let Some(pattern1_0) = C::gpr64_ty(ctx, pattern0_0) { - // Rule at src/isa/s390x/inst.isle line 2469. + // Rule at src/isa/s390x/inst.isle line 2464. let expr0_0 = ALUOp::AndNot64; return Some(expr0_0); } @@ -5222,7 +5222,7 @@ pub fn constructor_and_not_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2472. + // Rule at src/isa/s390x/inst.isle line 2467. let expr0_0 = constructor_aluop_and_not(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rrr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -5232,12 +5232,12 @@ pub fn constructor_and_not_reg( pub fn constructor_aluop_or_not(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if let Some(pattern1_0) = C::gpr32_ty(ctx, pattern0_0) { - // Rule at src/isa/s390x/inst.isle line 2478. + // Rule at src/isa/s390x/inst.isle line 2473. let expr0_0 = ALUOp::OrrNot32; return Some(expr0_0); } if let Some(pattern1_0) = C::gpr64_ty(ctx, pattern0_0) { - // Rule at src/isa/s390x/inst.isle line 2479. + // Rule at src/isa/s390x/inst.isle line 2474. let expr0_0 = ALUOp::OrrNot64; return Some(expr0_0); } @@ -5254,7 +5254,7 @@ pub fn constructor_or_not_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2482. + // Rule at src/isa/s390x/inst.isle line 2477. let expr0_0 = constructor_aluop_or_not(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rrr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -5264,12 +5264,12 @@ pub fn constructor_or_not_reg( pub fn constructor_aluop_xor_not(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if let Some(pattern1_0) = C::gpr32_ty(ctx, pattern0_0) { - // Rule at src/isa/s390x/inst.isle line 2488. + // Rule at src/isa/s390x/inst.isle line 2483. let expr0_0 = ALUOp::XorNot32; return Some(expr0_0); } if let Some(pattern1_0) = C::gpr64_ty(ctx, pattern0_0) { - // Rule at src/isa/s390x/inst.isle line 2489. + // Rule at src/isa/s390x/inst.isle line 2484. let expr0_0 = ALUOp::XorNot64; return Some(expr0_0); } @@ -5286,7 +5286,7 @@ pub fn constructor_xor_not_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2492. + // Rule at src/isa/s390x/inst.isle line 2487. let expr0_0 = constructor_aluop_xor_not(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rrr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -5296,12 +5296,12 @@ pub fn constructor_xor_not_reg( pub fn constructor_unaryop_abs(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 2498. + // Rule at src/isa/s390x/inst.isle line 2493. let expr0_0 = UnaryOp::Abs32; return Some(expr0_0); } if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2499. + // Rule at src/isa/s390x/inst.isle line 2494. let expr0_0 = UnaryOp::Abs64; return Some(expr0_0); } @@ -5312,7 +5312,7 @@ pub fn constructor_unaryop_abs(ctx: &mut C, arg0: Type) -> Option(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2502. + // Rule at src/isa/s390x/inst.isle line 2497. let expr0_0 = UnaryOp::Abs64Ext32; return Some(expr0_0); } @@ -5323,7 +5323,7 @@ pub fn constructor_unaryop_abs_sext32(ctx: &mut C, arg0: Type) -> Op pub fn constructor_abs_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2505. + // Rule at src/isa/s390x/inst.isle line 2500. let expr0_0 = constructor_unaryop_abs(ctx, pattern0_0)?; let expr1_0 = constructor_unary_rr(ctx, pattern0_0, &expr0_0, pattern1_0)?; return Some(expr1_0); @@ -5333,7 +5333,7 @@ pub fn constructor_abs_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> Op pub fn constructor_abs_reg_sext32(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2508. + // Rule at src/isa/s390x/inst.isle line 2503. let expr0_0 = constructor_unaryop_abs_sext32(ctx, pattern0_0)?; let expr1_0 = constructor_unary_rr(ctx, pattern0_0, &expr0_0, pattern1_0)?; return Some(expr1_0); @@ -5343,22 +5343,22 @@ pub fn constructor_abs_reg_sext32(ctx: &mut C, arg0: Type, arg1: Reg pub fn constructor_unaryop_neg(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I8 { - // Rule at src/isa/s390x/inst.isle line 2514. + // Rule at src/isa/s390x/inst.isle line 2509. let expr0_0 = UnaryOp::Neg32; return Some(expr0_0); } if pattern0_0 == I16 { - // Rule at src/isa/s390x/inst.isle line 2515. + // Rule at src/isa/s390x/inst.isle line 2510. let expr0_0 = UnaryOp::Neg32; return Some(expr0_0); } if pattern0_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 2516. + // Rule at src/isa/s390x/inst.isle line 2511. let expr0_0 = UnaryOp::Neg32; return Some(expr0_0); } if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2517. + // Rule at src/isa/s390x/inst.isle line 2512. let expr0_0 = UnaryOp::Neg64; return Some(expr0_0); } @@ -5369,7 +5369,7 @@ pub fn constructor_unaryop_neg(ctx: &mut C, arg0: Type) -> Option(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2520. + // Rule at src/isa/s390x/inst.isle line 2515. let expr0_0 = UnaryOp::Neg64Ext32; return Some(expr0_0); } @@ -5380,7 +5380,7 @@ pub fn constructor_unaryop_neg_sext32(ctx: &mut C, arg0: Type) -> Op pub fn constructor_neg_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2523. + // Rule at src/isa/s390x/inst.isle line 2518. let expr0_0 = constructor_unaryop_neg(ctx, pattern0_0)?; let expr1_0 = constructor_unary_rr(ctx, pattern0_0, &expr0_0, pattern1_0)?; return Some(expr1_0); @@ -5390,7 +5390,7 @@ pub fn constructor_neg_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> Op pub fn constructor_neg_reg_sext32(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2526. + // Rule at src/isa/s390x/inst.isle line 2521. let expr0_0 = constructor_unaryop_neg_sext32(ctx, pattern0_0)?; let expr1_0 = constructor_unary_rr(ctx, pattern0_0, &expr0_0, pattern1_0)?; return Some(expr1_0); @@ -5400,12 +5400,12 @@ pub fn constructor_neg_reg_sext32(ctx: &mut C, arg0: Type, arg1: Reg pub fn constructor_shiftop_rot(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 2532. + // Rule at src/isa/s390x/inst.isle line 2527. let expr0_0 = ShiftOp::RotL32; return Some(expr0_0); } if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2533. + // Rule at src/isa/s390x/inst.isle line 2528. let expr0_0 = ShiftOp::RotL64; return Some(expr0_0); } @@ -5422,7 +5422,7 @@ pub fn constructor_rot_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2536. + // Rule at src/isa/s390x/inst.isle line 2531. let expr0_0 = constructor_shiftop_rot(ctx, pattern0_0)?; let expr1_0: u8 = 0; let expr2_0 = constructor_shift_rr(ctx, pattern0_0, &expr0_0, pattern1_0, expr1_0, pattern2_0)?; @@ -5439,7 +5439,7 @@ pub fn constructor_rot_imm( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2540. + // Rule at src/isa/s390x/inst.isle line 2535. let expr0_0 = constructor_shiftop_rot(ctx, pattern0_0)?; let expr1_0 = C::zero_reg(ctx); let expr2_0 = constructor_shift_rr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0, expr1_0)?; @@ -5450,22 +5450,22 @@ pub fn constructor_rot_imm( pub fn constructor_shiftop_lshl(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I8 { - // Rule at src/isa/s390x/inst.isle line 2547. + // Rule at src/isa/s390x/inst.isle line 2542. let expr0_0 = ShiftOp::LShL32; return Some(expr0_0); } if pattern0_0 == I16 { - // Rule at src/isa/s390x/inst.isle line 2548. + // Rule at src/isa/s390x/inst.isle line 2543. let expr0_0 = ShiftOp::LShL32; return Some(expr0_0); } if pattern0_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 2549. + // Rule at src/isa/s390x/inst.isle line 2544. let expr0_0 = ShiftOp::LShL32; return Some(expr0_0); } if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2550. + // Rule at src/isa/s390x/inst.isle line 2545. let expr0_0 = ShiftOp::LShL64; return Some(expr0_0); } @@ -5482,7 +5482,7 @@ pub fn constructor_lshl_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2553. + // Rule at src/isa/s390x/inst.isle line 2548. let expr0_0 = constructor_shiftop_lshl(ctx, pattern0_0)?; let expr1_0: u8 = 0; let expr2_0 = constructor_shift_rr(ctx, pattern0_0, &expr0_0, pattern1_0, expr1_0, pattern2_0)?; @@ -5499,7 +5499,7 @@ pub fn constructor_lshl_imm( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2557. + // Rule at src/isa/s390x/inst.isle line 2552. let expr0_0 = constructor_shiftop_lshl(ctx, pattern0_0)?; let expr1_0 = C::zero_reg(ctx); let expr2_0 = constructor_shift_rr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0, expr1_0)?; @@ -5510,12 +5510,12 @@ pub fn constructor_lshl_imm( pub fn constructor_shiftop_lshr(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 2564. + // Rule at src/isa/s390x/inst.isle line 2559. let expr0_0 = ShiftOp::LShR32; return Some(expr0_0); } if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2565. + // Rule at src/isa/s390x/inst.isle line 2560. let expr0_0 = ShiftOp::LShR64; return Some(expr0_0); } @@ -5532,7 +5532,7 @@ pub fn constructor_lshr_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2568. + // Rule at src/isa/s390x/inst.isle line 2563. let expr0_0 = constructor_shiftop_lshr(ctx, pattern0_0)?; let expr1_0: u8 = 0; let expr2_0 = constructor_shift_rr(ctx, pattern0_0, &expr0_0, pattern1_0, expr1_0, pattern2_0)?; @@ -5549,7 +5549,7 @@ pub fn constructor_lshr_imm( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2572. + // Rule at src/isa/s390x/inst.isle line 2567. let expr0_0 = constructor_shiftop_lshr(ctx, pattern0_0)?; let expr1_0 = C::zero_reg(ctx); let expr2_0 = constructor_shift_rr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0, expr1_0)?; @@ -5560,12 +5560,12 @@ pub fn constructor_lshr_imm( pub fn constructor_shiftop_ashr(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 2579. + // Rule at src/isa/s390x/inst.isle line 2574. let expr0_0 = ShiftOp::AShR32; return Some(expr0_0); } if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2580. + // Rule at src/isa/s390x/inst.isle line 2575. let expr0_0 = ShiftOp::AShR64; return Some(expr0_0); } @@ -5582,7 +5582,7 @@ pub fn constructor_ashr_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2583. + // Rule at src/isa/s390x/inst.isle line 2578. let expr0_0 = constructor_shiftop_ashr(ctx, pattern0_0)?; let expr1_0: u8 = 0; let expr2_0 = constructor_shift_rr(ctx, pattern0_0, &expr0_0, pattern1_0, expr1_0, pattern2_0)?; @@ -5599,7 +5599,7 @@ pub fn constructor_ashr_imm( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2587. + // Rule at src/isa/s390x/inst.isle line 2582. let expr0_0 = constructor_shiftop_ashr(ctx, pattern0_0)?; let expr1_0 = C::zero_reg(ctx); let expr2_0 = constructor_shift_rr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0, expr1_0)?; @@ -5609,7 +5609,7 @@ pub fn constructor_ashr_imm( // Generated as internal constructor for term popcnt_byte. pub fn constructor_popcnt_byte(ctx: &mut C, arg0: Reg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/s390x/inst.isle line 2594. + // Rule at src/isa/s390x/inst.isle line 2589. let expr0_0: Type = I64; let expr1_0 = UnaryOp::PopcntByte; let expr2_0 = constructor_unary_rr(ctx, expr0_0, &expr1_0, pattern0_0)?; @@ -5619,7 +5619,7 @@ pub fn constructor_popcnt_byte(ctx: &mut C, arg0: Reg) -> Option(ctx: &mut C, arg0: Reg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/s390x/inst.isle line 2597. + // Rule at src/isa/s390x/inst.isle line 2592. let expr0_0: Type = I64; let expr1_0 = UnaryOp::PopcntReg; let expr2_0 = constructor_unary_rr(ctx, expr0_0, &expr1_0, pattern0_0)?; @@ -5637,7 +5637,7 @@ pub fn constructor_atomic_rmw_and( if pattern0_0 == I32 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2603. + // Rule at src/isa/s390x/inst.isle line 2598. let expr0_0: Type = I32; let expr1_0 = ALUOp::And32; let expr2_0 = constructor_atomic_rmw_impl(ctx, expr0_0, &expr1_0, pattern2_0, pattern3_0)?; @@ -5646,7 +5646,7 @@ pub fn constructor_atomic_rmw_and( if pattern0_0 == I64 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2604. + // Rule at src/isa/s390x/inst.isle line 2599. let expr0_0: Type = I64; let expr1_0 = ALUOp::And64; let expr2_0 = constructor_atomic_rmw_impl(ctx, expr0_0, &expr1_0, pattern2_0, pattern3_0)?; @@ -5666,7 +5666,7 @@ pub fn constructor_atomic_rmw_or( if pattern0_0 == I32 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2607. + // Rule at src/isa/s390x/inst.isle line 2602. let expr0_0: Type = I32; let expr1_0 = ALUOp::Orr32; let expr2_0 = constructor_atomic_rmw_impl(ctx, expr0_0, &expr1_0, pattern2_0, pattern3_0)?; @@ -5675,7 +5675,7 @@ pub fn constructor_atomic_rmw_or( if pattern0_0 == I64 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2608. + // Rule at src/isa/s390x/inst.isle line 2603. let expr0_0: Type = I64; let expr1_0 = ALUOp::Orr64; let expr2_0 = constructor_atomic_rmw_impl(ctx, expr0_0, &expr1_0, pattern2_0, pattern3_0)?; @@ -5695,7 +5695,7 @@ pub fn constructor_atomic_rmw_xor( if pattern0_0 == I32 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2611. + // Rule at src/isa/s390x/inst.isle line 2606. let expr0_0: Type = I32; let expr1_0 = ALUOp::Xor32; let expr2_0 = constructor_atomic_rmw_impl(ctx, expr0_0, &expr1_0, pattern2_0, pattern3_0)?; @@ -5704,7 +5704,7 @@ pub fn constructor_atomic_rmw_xor( if pattern0_0 == I64 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2612. + // Rule at src/isa/s390x/inst.isle line 2607. let expr0_0: Type = I64; let expr1_0 = ALUOp::Xor64; let expr2_0 = constructor_atomic_rmw_impl(ctx, expr0_0, &expr1_0, pattern2_0, pattern3_0)?; @@ -5724,7 +5724,7 @@ pub fn constructor_atomic_rmw_add( if pattern0_0 == I32 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2615. + // Rule at src/isa/s390x/inst.isle line 2610. let expr0_0: Type = I32; let expr1_0 = ALUOp::Add32; let expr2_0 = constructor_atomic_rmw_impl(ctx, expr0_0, &expr1_0, pattern2_0, pattern3_0)?; @@ -5733,7 +5733,7 @@ pub fn constructor_atomic_rmw_add( if pattern0_0 == I64 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2616. + // Rule at src/isa/s390x/inst.isle line 2611. let expr0_0: Type = I64; let expr1_0 = ALUOp::Add64; let expr2_0 = constructor_atomic_rmw_impl(ctx, expr0_0, &expr1_0, pattern2_0, pattern3_0)?; @@ -5746,12 +5746,12 @@ pub fn constructor_atomic_rmw_add( pub fn constructor_fpuop2_add(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32 { - // Rule at src/isa/s390x/inst.isle line 2622. + // Rule at src/isa/s390x/inst.isle line 2617. let expr0_0 = FPUOp2::Add32; return Some(expr0_0); } if pattern0_0 == F64 { - // Rule at src/isa/s390x/inst.isle line 2623. + // Rule at src/isa/s390x/inst.isle line 2618. let expr0_0 = FPUOp2::Add64; return Some(expr0_0); } @@ -5768,7 +5768,7 @@ pub fn constructor_fadd_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2626. + // Rule at src/isa/s390x/inst.isle line 2621. let expr0_0 = constructor_fpuop2_add(ctx, pattern0_0)?; let expr1_0 = constructor_fpu_rrr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -5778,12 +5778,12 @@ pub fn constructor_fadd_reg( pub fn constructor_fpuop2_sub(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32 { - // Rule at src/isa/s390x/inst.isle line 2632. + // Rule at src/isa/s390x/inst.isle line 2627. let expr0_0 = FPUOp2::Sub32; return Some(expr0_0); } if pattern0_0 == F64 { - // Rule at src/isa/s390x/inst.isle line 2633. + // Rule at src/isa/s390x/inst.isle line 2628. let expr0_0 = FPUOp2::Sub64; return Some(expr0_0); } @@ -5800,7 +5800,7 @@ pub fn constructor_fsub_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2636. + // Rule at src/isa/s390x/inst.isle line 2631. let expr0_0 = constructor_fpuop2_sub(ctx, pattern0_0)?; let expr1_0 = constructor_fpu_rrr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -5810,12 +5810,12 @@ pub fn constructor_fsub_reg( pub fn constructor_fpuop2_mul(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32 { - // Rule at src/isa/s390x/inst.isle line 2642. + // Rule at src/isa/s390x/inst.isle line 2637. let expr0_0 = FPUOp2::Mul32; return Some(expr0_0); } if pattern0_0 == F64 { - // Rule at src/isa/s390x/inst.isle line 2643. + // Rule at src/isa/s390x/inst.isle line 2638. let expr0_0 = FPUOp2::Mul64; return Some(expr0_0); } @@ -5832,7 +5832,7 @@ pub fn constructor_fmul_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2646. + // Rule at src/isa/s390x/inst.isle line 2641. let expr0_0 = constructor_fpuop2_mul(ctx, pattern0_0)?; let expr1_0 = constructor_fpu_rrr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -5842,12 +5842,12 @@ pub fn constructor_fmul_reg( pub fn constructor_fpuop2_div(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32 { - // Rule at src/isa/s390x/inst.isle line 2652. + // Rule at src/isa/s390x/inst.isle line 2647. let expr0_0 = FPUOp2::Div32; return Some(expr0_0); } if pattern0_0 == F64 { - // Rule at src/isa/s390x/inst.isle line 2653. + // Rule at src/isa/s390x/inst.isle line 2648. let expr0_0 = FPUOp2::Div64; return Some(expr0_0); } @@ -5864,7 +5864,7 @@ pub fn constructor_fdiv_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2656. + // Rule at src/isa/s390x/inst.isle line 2651. let expr0_0 = constructor_fpuop2_div(ctx, pattern0_0)?; let expr1_0 = constructor_fpu_rrr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -5874,12 +5874,12 @@ pub fn constructor_fdiv_reg( pub fn constructor_fpuop2_min(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32 { - // Rule at src/isa/s390x/inst.isle line 2662. + // Rule at src/isa/s390x/inst.isle line 2657. let expr0_0 = FPUOp2::Min32; return Some(expr0_0); } if pattern0_0 == F64 { - // Rule at src/isa/s390x/inst.isle line 2663. + // Rule at src/isa/s390x/inst.isle line 2658. let expr0_0 = FPUOp2::Min64; return Some(expr0_0); } @@ -5896,7 +5896,7 @@ pub fn constructor_fmin_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2666. + // Rule at src/isa/s390x/inst.isle line 2661. let expr0_0 = constructor_fpuop2_min(ctx, pattern0_0)?; let expr1_0 = constructor_fpuvec_rrr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -5906,12 +5906,12 @@ pub fn constructor_fmin_reg( pub fn constructor_fpuop2_max(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32 { - // Rule at src/isa/s390x/inst.isle line 2672. + // Rule at src/isa/s390x/inst.isle line 2667. let expr0_0 = FPUOp2::Max32; return Some(expr0_0); } if pattern0_0 == F64 { - // Rule at src/isa/s390x/inst.isle line 2673. + // Rule at src/isa/s390x/inst.isle line 2668. let expr0_0 = FPUOp2::Max64; return Some(expr0_0); } @@ -5928,7 +5928,7 @@ pub fn constructor_fmax_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2676. + // Rule at src/isa/s390x/inst.isle line 2671. let expr0_0 = constructor_fpuop2_max(ctx, pattern0_0)?; let expr1_0 = constructor_fpuvec_rrr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -5938,12 +5938,12 @@ pub fn constructor_fmax_reg( pub fn constructor_fpuop3_fma(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32 { - // Rule at src/isa/s390x/inst.isle line 2682. + // Rule at src/isa/s390x/inst.isle line 2677. let expr0_0 = FPUOp3::MAdd32; return Some(expr0_0); } if pattern0_0 == F64 { - // Rule at src/isa/s390x/inst.isle line 2683. + // Rule at src/isa/s390x/inst.isle line 2678. let expr0_0 = FPUOp3::MAdd64; return Some(expr0_0); } @@ -5962,7 +5962,7 @@ pub fn constructor_fma_reg( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 2686. + // Rule at src/isa/s390x/inst.isle line 2681. let expr0_0 = constructor_fpuop3_fma(ctx, pattern0_0)?; let expr1_0 = constructor_fpu_rrrr( ctx, pattern0_0, &expr0_0, pattern3_0, pattern1_0, pattern2_0, @@ -5974,12 +5974,12 @@ pub fn constructor_fma_reg( pub fn constructor_fpuop1_sqrt(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32 { - // Rule at src/isa/s390x/inst.isle line 2692. + // Rule at src/isa/s390x/inst.isle line 2687. let expr0_0 = FPUOp1::Sqrt32; return Some(expr0_0); } if pattern0_0 == F64 { - // Rule at src/isa/s390x/inst.isle line 2693. + // Rule at src/isa/s390x/inst.isle line 2688. let expr0_0 = FPUOp1::Sqrt64; return Some(expr0_0); } @@ -5990,7 +5990,7 @@ pub fn constructor_fpuop1_sqrt(ctx: &mut C, arg0: Type) -> Option(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2696. + // Rule at src/isa/s390x/inst.isle line 2691. let expr0_0 = constructor_fpuop1_sqrt(ctx, pattern0_0)?; let expr1_0 = constructor_fpu_rr(ctx, pattern0_0, &expr0_0, pattern1_0)?; return Some(expr1_0); @@ -6000,12 +6000,12 @@ pub fn constructor_sqrt_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> O pub fn constructor_fpuop1_neg(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32 { - // Rule at src/isa/s390x/inst.isle line 2702. + // Rule at src/isa/s390x/inst.isle line 2697. let expr0_0 = FPUOp1::Neg32; return Some(expr0_0); } if pattern0_0 == F64 { - // Rule at src/isa/s390x/inst.isle line 2703. + // Rule at src/isa/s390x/inst.isle line 2698. let expr0_0 = FPUOp1::Neg64; return Some(expr0_0); } @@ -6016,7 +6016,7 @@ pub fn constructor_fpuop1_neg(ctx: &mut C, arg0: Type) -> Option(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2706. + // Rule at src/isa/s390x/inst.isle line 2701. let expr0_0 = constructor_fpuop1_neg(ctx, pattern0_0)?; let expr1_0 = constructor_fpu_rr(ctx, pattern0_0, &expr0_0, pattern1_0)?; return Some(expr1_0); @@ -6026,12 +6026,12 @@ pub fn constructor_fneg_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> O pub fn constructor_fpuop1_abs(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32 { - // Rule at src/isa/s390x/inst.isle line 2712. + // Rule at src/isa/s390x/inst.isle line 2707. let expr0_0 = FPUOp1::Abs32; return Some(expr0_0); } if pattern0_0 == F64 { - // Rule at src/isa/s390x/inst.isle line 2713. + // Rule at src/isa/s390x/inst.isle line 2708. let expr0_0 = FPUOp1::Abs64; return Some(expr0_0); } @@ -6042,7 +6042,7 @@ pub fn constructor_fpuop1_abs(ctx: &mut C, arg0: Type) -> Option(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2716. + // Rule at src/isa/s390x/inst.isle line 2711. let expr0_0 = constructor_fpuop1_abs(ctx, pattern0_0)?; let expr1_0 = constructor_fpu_rr(ctx, pattern0_0, &expr0_0, pattern1_0)?; return Some(expr1_0); @@ -6052,12 +6052,12 @@ pub fn constructor_fabs_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> O pub fn constructor_fpuroundmode_ceil(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32 { - // Rule at src/isa/s390x/inst.isle line 2722. + // Rule at src/isa/s390x/inst.isle line 2717. let expr0_0 = FpuRoundMode::Plus32; return Some(expr0_0); } if pattern0_0 == F64 { - // Rule at src/isa/s390x/inst.isle line 2723. + // Rule at src/isa/s390x/inst.isle line 2718. let expr0_0 = FpuRoundMode::Plus64; return Some(expr0_0); } @@ -6068,7 +6068,7 @@ pub fn constructor_fpuroundmode_ceil(ctx: &mut C, arg0: Type) -> Opt pub fn constructor_ceil_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2726. + // Rule at src/isa/s390x/inst.isle line 2721. let expr0_0 = constructor_fpuroundmode_ceil(ctx, pattern0_0)?; let expr1_0 = constructor_fpu_round(ctx, pattern0_0, &expr0_0, pattern1_0)?; return Some(expr1_0); @@ -6078,12 +6078,12 @@ pub fn constructor_ceil_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> O pub fn constructor_fpuroundmode_floor(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32 { - // Rule at src/isa/s390x/inst.isle line 2732. + // Rule at src/isa/s390x/inst.isle line 2727. let expr0_0 = FpuRoundMode::Minus32; return Some(expr0_0); } if pattern0_0 == F64 { - // Rule at src/isa/s390x/inst.isle line 2733. + // Rule at src/isa/s390x/inst.isle line 2728. let expr0_0 = FpuRoundMode::Minus64; return Some(expr0_0); } @@ -6094,7 +6094,7 @@ pub fn constructor_fpuroundmode_floor(ctx: &mut C, arg0: Type) -> Op pub fn constructor_floor_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2736. + // Rule at src/isa/s390x/inst.isle line 2731. let expr0_0 = constructor_fpuroundmode_floor(ctx, pattern0_0)?; let expr1_0 = constructor_fpu_round(ctx, pattern0_0, &expr0_0, pattern1_0)?; return Some(expr1_0); @@ -6104,12 +6104,12 @@ pub fn constructor_floor_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> pub fn constructor_fpuroundmode_trunc(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32 { - // Rule at src/isa/s390x/inst.isle line 2742. + // Rule at src/isa/s390x/inst.isle line 2737. let expr0_0 = FpuRoundMode::Zero32; return Some(expr0_0); } if pattern0_0 == F64 { - // Rule at src/isa/s390x/inst.isle line 2743. + // Rule at src/isa/s390x/inst.isle line 2738. let expr0_0 = FpuRoundMode::Zero64; return Some(expr0_0); } @@ -6120,7 +6120,7 @@ pub fn constructor_fpuroundmode_trunc(ctx: &mut C, arg0: Type) -> Op pub fn constructor_trunc_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2746. + // Rule at src/isa/s390x/inst.isle line 2741. let expr0_0 = constructor_fpuroundmode_trunc(ctx, pattern0_0)?; let expr1_0 = constructor_fpu_round(ctx, pattern0_0, &expr0_0, pattern1_0)?; return Some(expr1_0); @@ -6133,12 +6133,12 @@ pub fn constructor_fpuroundmode_nearest( ) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32 { - // Rule at src/isa/s390x/inst.isle line 2752. + // Rule at src/isa/s390x/inst.isle line 2747. let expr0_0 = FpuRoundMode::Nearest32; return Some(expr0_0); } if pattern0_0 == F64 { - // Rule at src/isa/s390x/inst.isle line 2753. + // Rule at src/isa/s390x/inst.isle line 2748. let expr0_0 = FpuRoundMode::Nearest64; return Some(expr0_0); } @@ -6149,7 +6149,7 @@ pub fn constructor_fpuroundmode_nearest( pub fn constructor_nearest_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2756. + // Rule at src/isa/s390x/inst.isle line 2751. let expr0_0 = constructor_fpuroundmode_nearest(ctx, pattern0_0)?; let expr1_0 = constructor_fpu_round(ctx, pattern0_0, &expr0_0, pattern1_0)?; return Some(expr1_0); @@ -6165,7 +6165,7 @@ pub fn constructor_fpuop1_promote( if pattern0_0 == F64 { let pattern2_0 = arg1; if pattern2_0 == F32 { - // Rule at src/isa/s390x/inst.isle line 2762. + // Rule at src/isa/s390x/inst.isle line 2757. let expr0_0 = FPUOp1::Cvt32To64; return Some(expr0_0); } @@ -6183,7 +6183,7 @@ pub fn constructor_fpromote_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2765. + // Rule at src/isa/s390x/inst.isle line 2760. let expr0_0 = constructor_fpuop1_promote(ctx, pattern0_0, pattern1_0)?; let expr1_0 = constructor_fpu_rr(ctx, pattern0_0, &expr0_0, pattern2_0)?; return Some(expr1_0); @@ -6199,7 +6199,7 @@ pub fn constructor_fpuop1_demote( if pattern0_0 == F32 { let pattern2_0 = arg1; if pattern2_0 == F64 { - // Rule at src/isa/s390x/inst.isle line 2772. + // Rule at src/isa/s390x/inst.isle line 2767. let expr0_0 = FPUOp1::Cvt64To32; return Some(expr0_0); } @@ -6217,7 +6217,7 @@ pub fn constructor_fdemote_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2775. + // Rule at src/isa/s390x/inst.isle line 2770. let expr0_0 = constructor_fpuop1_demote(ctx, pattern0_0, pattern1_0)?; let expr1_0 = constructor_fpu_rr(ctx, pattern0_0, &expr0_0, pattern2_0)?; return Some(expr1_0); @@ -6233,12 +6233,12 @@ pub fn constructor_uint_to_fpu_op( if pattern0_0 == F32 { let pattern2_0 = arg1; if pattern2_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 2782. + // Rule at src/isa/s390x/inst.isle line 2777. let expr0_0 = IntToFpuOp::U32ToF32; return Some(expr0_0); } if pattern2_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2784. + // Rule at src/isa/s390x/inst.isle line 2779. let expr0_0 = IntToFpuOp::U64ToF32; return Some(expr0_0); } @@ -6246,12 +6246,12 @@ pub fn constructor_uint_to_fpu_op( if pattern0_0 == F64 { let pattern2_0 = arg1; if pattern2_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 2783. + // Rule at src/isa/s390x/inst.isle line 2778. let expr0_0 = IntToFpuOp::U32ToF64; return Some(expr0_0); } if pattern2_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2785. + // Rule at src/isa/s390x/inst.isle line 2780. let expr0_0 = IntToFpuOp::U64ToF64; return Some(expr0_0); } @@ -6269,7 +6269,7 @@ pub fn constructor_fcvt_from_uint_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2788. + // Rule at src/isa/s390x/inst.isle line 2783. let expr0_0 = constructor_uint_to_fpu_op(ctx, pattern0_0, pattern1_0)?; let expr1_0 = constructor_int_to_fpu(ctx, pattern0_0, &expr0_0, pattern2_0)?; return Some(expr1_0); @@ -6285,12 +6285,12 @@ pub fn constructor_sint_to_fpu_op( if pattern0_0 == F32 { let pattern2_0 = arg1; if pattern2_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 2795. + // Rule at src/isa/s390x/inst.isle line 2790. let expr0_0 = IntToFpuOp::I32ToF32; return Some(expr0_0); } if pattern2_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2797. + // Rule at src/isa/s390x/inst.isle line 2792. let expr0_0 = IntToFpuOp::I64ToF32; return Some(expr0_0); } @@ -6298,12 +6298,12 @@ pub fn constructor_sint_to_fpu_op( if pattern0_0 == F64 { let pattern2_0 = arg1; if pattern2_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 2796. + // Rule at src/isa/s390x/inst.isle line 2791. let expr0_0 = IntToFpuOp::I32ToF64; return Some(expr0_0); } if pattern2_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2798. + // Rule at src/isa/s390x/inst.isle line 2793. let expr0_0 = IntToFpuOp::I64ToF64; return Some(expr0_0); } @@ -6321,7 +6321,7 @@ pub fn constructor_fcvt_from_sint_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2801. + // Rule at src/isa/s390x/inst.isle line 2796. let expr0_0 = constructor_sint_to_fpu_op(ctx, pattern0_0, pattern1_0)?; let expr1_0 = constructor_int_to_fpu(ctx, pattern0_0, &expr0_0, pattern2_0)?; return Some(expr1_0); @@ -6337,12 +6337,12 @@ pub fn constructor_fpu_to_uint_op( if pattern0_0 == I32 { let pattern2_0 = arg1; if pattern2_0 == F32 { - // Rule at src/isa/s390x/inst.isle line 2808. + // Rule at src/isa/s390x/inst.isle line 2803. let expr0_0 = FpuToIntOp::F32ToU32; return Some(expr0_0); } if pattern2_0 == F64 { - // Rule at src/isa/s390x/inst.isle line 2809. + // Rule at src/isa/s390x/inst.isle line 2804. let expr0_0 = FpuToIntOp::F64ToU32; return Some(expr0_0); } @@ -6350,12 +6350,12 @@ pub fn constructor_fpu_to_uint_op( if pattern0_0 == I64 { let pattern2_0 = arg1; if pattern2_0 == F32 { - // Rule at src/isa/s390x/inst.isle line 2810. + // Rule at src/isa/s390x/inst.isle line 2805. let expr0_0 = FpuToIntOp::F32ToU64; return Some(expr0_0); } if pattern2_0 == F64 { - // Rule at src/isa/s390x/inst.isle line 2811. + // Rule at src/isa/s390x/inst.isle line 2806. let expr0_0 = FpuToIntOp::F64ToU64; return Some(expr0_0); } @@ -6373,7 +6373,7 @@ pub fn constructor_fcvt_to_uint_reg_with_flags( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2814. + // Rule at src/isa/s390x/inst.isle line 2809. let expr0_0 = constructor_fpu_to_uint_op(ctx, pattern0_0, pattern1_0)?; let expr1_0 = constructor_fpu_to_int(ctx, pattern0_0, &expr0_0, pattern2_0)?; return Some(expr1_0); @@ -6389,7 +6389,7 @@ pub fn constructor_fcvt_to_uint_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2818. + // Rule at src/isa/s390x/inst.isle line 2813. let expr0_0 = constructor_fcvt_to_uint_reg_with_flags(ctx, pattern0_0, pattern1_0, pattern2_0)?; let expr1_0 = constructor_drop_flags(ctx, &expr0_0)?; return Some(expr1_0); @@ -6405,12 +6405,12 @@ pub fn constructor_fpu_to_sint_op( if pattern0_0 == I32 { let pattern2_0 = arg1; if pattern2_0 == F32 { - // Rule at src/isa/s390x/inst.isle line 2825. + // Rule at src/isa/s390x/inst.isle line 2820. let expr0_0 = FpuToIntOp::F32ToI32; return Some(expr0_0); } if pattern2_0 == F64 { - // Rule at src/isa/s390x/inst.isle line 2826. + // Rule at src/isa/s390x/inst.isle line 2821. let expr0_0 = FpuToIntOp::F64ToI32; return Some(expr0_0); } @@ -6418,12 +6418,12 @@ pub fn constructor_fpu_to_sint_op( if pattern0_0 == I64 { let pattern2_0 = arg1; if pattern2_0 == F32 { - // Rule at src/isa/s390x/inst.isle line 2827. + // Rule at src/isa/s390x/inst.isle line 2822. let expr0_0 = FpuToIntOp::F32ToI64; return Some(expr0_0); } if pattern2_0 == F64 { - // Rule at src/isa/s390x/inst.isle line 2828. + // Rule at src/isa/s390x/inst.isle line 2823. let expr0_0 = FpuToIntOp::F64ToI64; return Some(expr0_0); } @@ -6441,7 +6441,7 @@ pub fn constructor_fcvt_to_sint_reg_with_flags( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2831. + // Rule at src/isa/s390x/inst.isle line 2826. let expr0_0 = constructor_fpu_to_sint_op(ctx, pattern0_0, pattern1_0)?; let expr1_0 = constructor_fpu_to_int(ctx, pattern0_0, &expr0_0, pattern2_0)?; return Some(expr1_0); @@ -6457,7 +6457,7 @@ pub fn constructor_fcvt_to_sint_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2835. + // Rule at src/isa/s390x/inst.isle line 2830. let expr0_0 = constructor_fcvt_to_sint_reg_with_flags(ctx, pattern0_0, pattern1_0, pattern2_0)?; let expr1_0 = constructor_drop_flags(ctx, &expr0_0)?; return Some(expr1_0); @@ -6467,12 +6467,12 @@ pub fn constructor_fcvt_to_sint_reg( pub fn constructor_cmpop_cmps(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 2842. + // Rule at src/isa/s390x/inst.isle line 2837. let expr0_0 = CmpOp::CmpS32; return Some(expr0_0); } if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2843. + // Rule at src/isa/s390x/inst.isle line 2838. let expr0_0 = CmpOp::CmpS64; return Some(expr0_0); } @@ -6483,12 +6483,12 @@ pub fn constructor_cmpop_cmps(ctx: &mut C, arg0: Type) -> Option(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 2846. + // Rule at src/isa/s390x/inst.isle line 2841. let expr0_0 = CmpOp::CmpS32Ext16; return Some(expr0_0); } if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2847. + // Rule at src/isa/s390x/inst.isle line 2842. let expr0_0 = CmpOp::CmpS64Ext16; return Some(expr0_0); } @@ -6499,7 +6499,7 @@ pub fn constructor_cmpop_cmps_sext16(ctx: &mut C, arg0: Type) -> Opt pub fn constructor_cmpop_cmps_sext32(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2850. + // Rule at src/isa/s390x/inst.isle line 2845. let expr0_0 = CmpOp::CmpS64Ext32; return Some(expr0_0); } @@ -6516,7 +6516,7 @@ pub fn constructor_icmps_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2853. + // Rule at src/isa/s390x/inst.isle line 2848. let expr0_0 = constructor_cmpop_cmps(ctx, pattern0_0)?; let expr1_0 = constructor_cmp_rr(ctx, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -6532,7 +6532,7 @@ pub fn constructor_icmps_reg_sext32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2856. + // Rule at src/isa/s390x/inst.isle line 2851. let expr0_0 = constructor_cmpop_cmps_sext32(ctx, pattern0_0)?; let expr1_0 = constructor_cmp_rr(ctx, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -6548,7 +6548,7 @@ pub fn constructor_icmps_simm16( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2859. + // Rule at src/isa/s390x/inst.isle line 2854. let expr0_0 = constructor_cmpop_cmps(ctx, pattern0_0)?; let expr1_0 = constructor_cmp_rsimm16(ctx, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -6564,7 +6564,7 @@ pub fn constructor_icmps_simm32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2862. + // Rule at src/isa/s390x/inst.isle line 2857. let expr0_0 = constructor_cmpop_cmps(ctx, pattern0_0)?; let expr1_0 = constructor_cmp_rsimm32(ctx, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -6580,7 +6580,7 @@ pub fn constructor_icmps_mem( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2865. + // Rule at src/isa/s390x/inst.isle line 2860. let expr0_0 = constructor_cmpop_cmps(ctx, pattern0_0)?; let expr1_0 = constructor_cmp_rx(ctx, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -6596,7 +6596,7 @@ pub fn constructor_icmps_mem_sext16( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2868. + // Rule at src/isa/s390x/inst.isle line 2863. let expr0_0 = constructor_cmpop_cmps_sext16(ctx, pattern0_0)?; let expr1_0 = constructor_cmp_rx(ctx, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -6612,7 +6612,7 @@ pub fn constructor_icmps_mem_sext32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2871. + // Rule at src/isa/s390x/inst.isle line 2866. let expr0_0 = constructor_cmpop_cmps_sext32(ctx, pattern0_0)?; let expr1_0 = constructor_cmp_rx(ctx, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -6622,12 +6622,12 @@ pub fn constructor_icmps_mem_sext32( pub fn constructor_cmpop_cmpu(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 2877. + // Rule at src/isa/s390x/inst.isle line 2872. let expr0_0 = CmpOp::CmpL32; return Some(expr0_0); } if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2878. + // Rule at src/isa/s390x/inst.isle line 2873. let expr0_0 = CmpOp::CmpL64; return Some(expr0_0); } @@ -6638,12 +6638,12 @@ pub fn constructor_cmpop_cmpu(ctx: &mut C, arg0: Type) -> Option(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 2881. + // Rule at src/isa/s390x/inst.isle line 2876. let expr0_0 = CmpOp::CmpL32Ext16; return Some(expr0_0); } if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2882. + // Rule at src/isa/s390x/inst.isle line 2877. let expr0_0 = CmpOp::CmpL64Ext16; return Some(expr0_0); } @@ -6654,7 +6654,7 @@ pub fn constructor_cmpop_cmpu_zext16(ctx: &mut C, arg0: Type) -> Opt pub fn constructor_cmpop_cmpu_zext32(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2885. + // Rule at src/isa/s390x/inst.isle line 2880. let expr0_0 = CmpOp::CmpL64Ext32; return Some(expr0_0); } @@ -6671,7 +6671,7 @@ pub fn constructor_icmpu_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2888. + // Rule at src/isa/s390x/inst.isle line 2883. let expr0_0 = constructor_cmpop_cmpu(ctx, pattern0_0)?; let expr1_0 = constructor_cmp_rr(ctx, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -6687,7 +6687,7 @@ pub fn constructor_icmpu_reg_zext32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2891. + // Rule at src/isa/s390x/inst.isle line 2886. let expr0_0 = constructor_cmpop_cmpu_zext32(ctx, pattern0_0)?; let expr1_0 = constructor_cmp_rr(ctx, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -6703,7 +6703,7 @@ pub fn constructor_icmpu_uimm32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2894. + // Rule at src/isa/s390x/inst.isle line 2889. let expr0_0 = constructor_cmpop_cmpu(ctx, pattern0_0)?; let expr1_0 = constructor_cmp_ruimm32(ctx, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -6719,7 +6719,7 @@ pub fn constructor_icmpu_mem( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2897. + // Rule at src/isa/s390x/inst.isle line 2892. let expr0_0 = constructor_cmpop_cmpu(ctx, pattern0_0)?; let expr1_0 = constructor_cmp_rx(ctx, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -6735,7 +6735,7 @@ pub fn constructor_icmpu_mem_zext16( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2900. + // Rule at src/isa/s390x/inst.isle line 2895. let expr0_0 = constructor_cmpop_cmpu_zext16(ctx, pattern0_0)?; let expr1_0 = constructor_cmp_rx(ctx, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -6751,7 +6751,7 @@ pub fn constructor_icmpu_mem_zext32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2903. + // Rule at src/isa/s390x/inst.isle line 2898. let expr0_0 = constructor_cmpop_cmpu_zext32(ctx, pattern0_0)?; let expr1_0 = constructor_cmp_rx(ctx, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -6768,14 +6768,14 @@ pub fn constructor_fcmp_reg( if pattern0_0 == F32 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2909. + // Rule at src/isa/s390x/inst.isle line 2904. let expr0_0 = constructor_fpu_cmp32(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if pattern0_0 == F64 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2910. + // Rule at src/isa/s390x/inst.isle line 2905. let expr0_0 = constructor_fpu_cmp64(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } diff --git a/cranelift/codegen/src/isa/x64/encoding/rex.rs b/cranelift/codegen/src/isa/x64/encoding/rex.rs index 17e5408cc1b9..509309205df5 100644 --- a/cranelift/codegen/src/isa/x64/encoding/rex.rs +++ b/cranelift/codegen/src/isa/x64/encoding/rex.rs @@ -47,7 +47,8 @@ pub(crate) fn encode_sib(shift: u8, enc_index: u8, enc_base: u8) -> u8 { /// Get the encoding number of a GPR. #[inline(always)] -pub(crate) fn int_reg_enc(reg: Reg) -> u8 { +pub(crate) fn int_reg_enc(reg: impl Into) -> u8 { + let reg = reg.into(); debug_assert!(reg.is_real()); debug_assert_eq!(reg.get_class(), RegClass::I64); reg.get_hw_encoding() @@ -55,7 +56,8 @@ pub(crate) fn int_reg_enc(reg: Reg) -> u8 { /// Get the encoding number of any register. #[inline(always)] -pub(crate) fn reg_enc(reg: Reg) -> u8 { +pub(crate) fn reg_enc(reg: impl Into) -> u8 { + let reg = reg.into(); debug_assert!(reg.is_real()); reg.get_hw_encoding() } diff --git a/cranelift/codegen/src/isa/x64/inst.isle b/cranelift/codegen/src/isa/x64/inst.isle index a126750872fa..4ba053caef37 100644 --- a/cranelift/codegen/src/isa/x64/inst.isle +++ b/cranelift/codegen/src/isa/x64/inst.isle @@ -2,82 +2,486 @@ ;;;; `MInst` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -(type MInst extern - (enum (Nop (len u8)) - (Ud2 (trap_code TrapCode)) - (AluRmiR (size OperandSize) - (op AluRmiROpcode) - (src1 Reg) - (src2 RegMemImm) - (dst WritableReg)) - (MulHi (size OperandSize) - (signed bool) - (src1 Reg) - (src2 RegMem) - (dst_lo WritableReg) - (dst_hi WritableReg)) - (XmmRmR (op SseOpcode) - (src1 Reg) - (src2 RegMem) - (dst WritableReg)) - (XmmUnaryRmR (op SseOpcode) - (src RegMem) - (dst WritableReg)) - (XmmUnaryRmREvex (op Avx512Opcode) - (src RegMem) - (dst WritableReg)) - (XmmRmiReg (opcode SseOpcode) - (src1 Reg) - (src2 RegMemImm) +;; Don't build `MInst` variants directly, in general. Instead, use the +;; instruction-emitting helpers defined further down. + +(type MInst nodebug + (enum + ;; Nops of various sizes, including zero. + (Nop (len u8)) + + ;; ========================================= + ;; Integer instructions. + + ;; Integer arithmetic/bit-twiddling. + (AluRmiR (size OperandSize) ;; 4 or 8 + (op AluRmiROpcode) + (src1 Reg) + (src2 RegMemImm) + (dst WritableReg)) + + ;; Instructions on general-purpose registers that only read src and + ;; defines dst (dst is not modified). `bsr`, etc. + (UnaryRmR (size OperandSize) ;; 2, 4, or 8 + (op UnaryRmROpcode) + (src GprMem) + (dst WritableGpr)) + + ;; Bitwise not. + (Not (size OperandSize) ;; 1, 2, 4, or 8 + (src Gpr) + (dst WritableGpr)) + + ;; Integer negation. + (Neg (size OperandSize) ;; 1, 2, 4, or 8 + (src Gpr) + (dst WritableGpr)) + + ;; Integer quotient and remainder: (div idiv) $rax $rdx (reg addr) + (Div (size OperandSize) ;; 1, 2, 4, or 8 + (signed bool) + (divisor RegMem) + (dividend Reg) + (dst_quotient WritableReg) + (dst_remainder WritableReg)) + + ;; The high (and low) bits of a (un)signed multiply: `RDX:RAX := RAX * + ;; rhs`. + (MulHi (size OperandSize) + (signed bool) + (src1 Reg) + (src2 RegMem) + (dst_lo WritableReg) + (dst_hi WritableReg)) + + ;; A synthetic sequence to implement the right inline checks for + ;; remainder and division, assuming the dividend is in %rax. + ;; + ;; Puts the result back into %rax if is_div, %rdx if !is_div, to mimic + ;; what the div instruction does. + ;; + ;; The generated code sequence is described in the emit's function match + ;; arm for this instruction. + ;; + ;; Note: %rdx is marked as modified by this instruction, to avoid an + ;; early clobber problem with the temporary and divisor registers. Make + ;; sure to zero %rdx right before this instruction, or you might run into + ;; regalloc failures where %rdx is live before its first def! + (CheckedDivOrRemSeq (kind DivOrRemKind) + (size OperandSize) + (dividend Reg) + ;; The divisor operand. Note it's marked as modified + ;; so that it gets assigned a register different from + ;; the temporary. + (divisor WritableReg) + (dst_quotient WritableReg) + (dst_remainder WritableReg) + (tmp OptionWritableReg)) + + ;; Do a sign-extend based on the sign of the value in rax into rdx: (cwd + ;; cdq cqo) or al into ah: (cbw) + (SignExtendData (size OperandSize) ;; 1, 2, 4, or 8 + (src Reg) (dst WritableReg)) - (XmmRmRImm (op SseOpcode) - (src1 Reg) - (src2 RegMem) - (dst WritableReg) - (imm u8) - (size OperandSize)) - (XmmUninitializedValue (dst WritableReg)) - (CmpRmiR (size OperandSize) - (opcode CmpOpcode) - (src RegMemImm) - (dst Reg)) - (Imm (dst_size OperandSize) - (simm64 u64) + + ;; Constant materialization: (imm32 imm64) reg. + ;; + ;; Either: movl $imm32, %reg32 or movabsq $imm64, %reg32. + (Imm (dst_size OperandSize) ;; 4 or 8 + (simm64 u64) + (dst WritableReg)) + + ;; GPR to GPR move: mov (64 32) reg reg. + (MovRR (size OperandSize) ;; 4 or 8 + (src Reg) + (dst WritableReg)) + + ;; Zero-extended loads, except for 64 bits: movz (bl bq wl wq lq) addr + ;; reg. + ;; + ;; Note that the lq variant doesn't really exist since the default + ;; zero-extend rule makes it unnecessary. For that case we emit the + ;; equivalent "movl AM, reg32". + (MovzxRmR (ext_mode ExtMode) + (src RegMem) (dst WritableReg)) - (ShiftR (size OperandSize) - (kind ShiftKind) - (src Reg) - (num_bits Imm8Reg) - (dst WritableReg)) - (MovzxRmR (ext_mode ExtMode) - (src RegMem) - (dst WritableReg)) - (MovsxRmR (ext_mode ExtMode) - (src RegMem) - (dst WritableReg)) - (Mov64MR (src SyntheticAmode) - (dst WritableReg)) - (Cmove (size OperandSize) - (cc CC) - (consequent RegMem) - (alternative Reg) - (dst WritableReg)) - (XmmRmREvex (op Avx512Opcode) - (src1 RegMem) - (src2 Reg) - (dst WritableReg)) - (GprToXmm (op SseOpcode) - (src RegMem) - (dst WritableReg) - (src_size OperandSize)) - (Not (size OperandSize) - (src Reg) + + ;; A plain 64-bit integer load, since MovZX_RM_R can't represent that. + (Mov64MR (src SyntheticAmode) + (dst WritableReg)) + + ;; Loads the memory address of addr into dst. + (LoadEffectiveAddress (addr SyntheticAmode) + (dst WritableGpr)) + + ;; Sign-extended loads and moves: movs (bl bq wl wq lq) addr reg. + (MovsxRmR (ext_mode ExtMode) + (src RegMem) (dst WritableReg)) - (Neg (size OperandSize) + + ;; Integer stores: mov (b w l q) reg addr. + (MovRM (size OperandSize) ;; 1, 2, 4, or 8 + (src Reg) + (dst SyntheticAmode)) + + ;; Arithmetic shifts: (shl shr sar) (b w l q) imm reg. + (ShiftR (size OperandSize) ;; 1, 2, 4, or 8 + (kind ShiftKind) + (src Reg) + ;; shift count: `Imm8Reg::Imm8(0 .. #bits-in-type - 1)` or + ;; `Imm8Reg::Reg(r)` where `r` get's move mitosis'd into `%cl`. + (num_bits Imm8Reg) + (dst WritableReg)) + + ;; Arithmetic SIMD shifts. + (XmmRmiReg (opcode SseOpcode) + (src1 Xmm) + (src2 XmmMemImm) + (dst WritableXmm)) + + ;; Integer comparisons/tests: cmp or test (b w l q) (reg addr imm) reg. + (CmpRmiR (size OperandSize) ;; 1, 2, 4, or 8 + (opcode CmpOpcode) + (src RegMemImm) + (dst Reg)) + + ;; Materializes the requested condition code in the destinaton reg. + (Setcc (cc CC) + (dst WritableReg)) + + ;; Integer conditional move. + ;; + ;; Overwrites the destination register. + (Cmove (size OperandSize) + (cc CC) + (consequent RegMem) + (alternative Reg) + (dst WritableReg)) + + ;; ========================================= + ;; Stack manipulation. + + ;; pushq (reg addr imm) + (Push64 (src RegMemImm)) + + ;; popq reg + (Pop64 (dst WritableReg)) + + ;; ========================================= + ;; Floating-point operations. + + ;; XMM (scalar or vector) binary op: (add sub and or xor mul adc? sbb?) + ;; (32 64) (reg addr) reg + (XmmRmR (op SseOpcode) + (src1 Xmm) + (src2 XmmMem) + (dst WritableXmm)) + + ;; XMM (scalar or vector) binary op that relies on the EVEX prefix. + (XmmRmREvex (op Avx512Opcode) + (src1 XmmMem) + (src2 Xmm) + (dst WritableXmm)) + + ;; XMM (scalar or vector) unary op: mov between XMM registers (32 64) + ;; (reg addr) reg, sqrt, etc. + ;; + ;; This differs from XMM_RM_R in that the dst register of XmmUnaryRmR is + ;; not used in the computation of the instruction dst value and so does + ;; not have to be a previously valid value. This is characteristic of mov + ;; instructions. + (XmmUnaryRmR (op SseOpcode) + (src XmmMem) + (dst WritableXmm)) + + ;; XMM (scalar or vector) unary op that relies on the EVEX prefix. + (XmmUnaryRmREvex (op Avx512Opcode) + (src XmmMem) + (dst WritableXmm)) + + ;; XMM (scalar or vector) unary op (from xmm to reg/mem): stores, movd, + ;; movq + (XmmMovRM (op SseOpcode) (src Reg) + (dst SyntheticAmode)) + + ;; XMM (vector) unary op (to move a constant value into an xmm register): + ;; movups + (XmmLoadConst (src VCodeConstant) + (dst WritableReg) + (ty Type)) + + ;; XMM (scalar) unary op (from xmm to integer reg): movd, movq, + ;; cvtts{s,d}2si + (XmmToGpr (op SseOpcode) + (src Xmm) + (dst WritableGpr) + (dst_size OperandSize)) + + ;; XMM (scalar) unary op (from integer to float reg): movd, movq, + ;; cvtsi2s{s,d} + (GprToXmm (op SseOpcode) + (src RegMem) + (dst WritableXmm) + (src_size OperandSize)) + + ;; Converts an unsigned int64 to a float32/float64. + (CvtUint64ToFloatSeq (dst_size OperandSize) ;; 4 or 8 + ;; A copy of the source register, fed by + ;; lowering. It is marked as modified during + ;; register allocation to make sure that the + ;; temporary registers differ from the src register, + ;; since both registers are live at the same time in + ;; the generated code sequence. + (src WritableGpr) + (dst WritableXmm) + (tmp_gpr1 WritableGpr) + (tmp_gpr2 WritableGpr)) + + ;; Converts a scalar xmm to a signed int32/int64. + (CvtFloatToSintSeq (dst_size OperandSize) + (src_size OperandSize) + (is_saturating bool) + ;; A copy of the source register, fed by + ;; lowering. It is marked as modified during + ;; register allocation to make sure that the + ;; temporary registers differ from the src register, + ;; since both registers are live at the same time in + ;; the generated code sequence. + (src WritableXmm) + (dst WritableGpr) + (tmp_gpr WritableGpr) + (tmp_xmm WritableXmm)) + + ;; Converts a scalar xmm to an unsigned int32/int64. + (CvtFloatToUintSeq (dst_size OperandSize) + (src_size OperandSize) + (is_saturating bool) + ;; A copy of the source register, fed by + ;; lowering. It is marked as modified during + ;; register allocation to make sure that the + ;; temporary registers differ from the src register, + ;; since both registers are live at the same time in + ;; the generated code sequence. + (src WritableXmm) + (dst WritableGpr) + (tmp_gpr WritableGpr) + (tmp_xmm WritableXmm)) + + ;; A sequence to compute min/max with the proper NaN semantics for xmm + ;; registers. + (XmmMinMaxSeq (size OperandSize) + (is_min bool) + (lhs Reg) + (rhs_dst WritableReg)) + + ;; XMM (scalar) conditional move. + ;; + ;; Overwrites the destination register if cc is set. + (XmmCmove (size OperandSize) + (cc CC) + (src RegMem) (dst WritableReg)) - (LoadEffectiveAddress (addr SyntheticAmode) - (dst WritableReg)))) + + ;; Float comparisons/tests: cmp (b w l q) (reg addr imm) reg. + (XmmCmpRmR (op SseOpcode) + (src RegMem) + (dst Reg)) + + ;; A binary XMM instruction with an 8-bit immediate: e.g. cmp (ps pd) imm + ;; (reg addr) reg + ;; + ;; Note: this has to use `Reg*`, not `Xmm*`, operands because it is used + ;; in various lane insertion and extraction instructions that move + ;; between XMMs and GPRs. + (XmmRmRImm (op SseOpcode) + (src1 Reg) + (src2 RegMem) + (dst WritableReg) + (imm u8) + (size OperandSize)) + + ;; ========================================= + ;; Control flow instructions. + + ;; Direct call: call simm32. + (CallKnown (dest ExternalName) + (uses VecReg) + (defs VecWritableReg) + (opcode Opcode)) + + ;; Indirect call: callq (reg mem) + (CallUnknown (dest RegMem) + (uses VecReg) + (defs VecWritableReg) + (opcode Opcode)) + + ;; Return. + (Ret) + + ;; A placeholder instruction, generating no code, meaning that a function + ;; epilogue must be inserted there. + (EpiloguePlaceholder) + + ;; Jump to a known target: jmp simm32. + (JmpKnown (dst MachLabel)) + + + ;; One-way conditional branch: jcond cond target. + ;; + ;; This instruction is useful when we have conditional jumps depending on + ;; more than two conditions, see for instance the lowering of Brz/brnz + ;; with Fcmp inputs. + ;; + ;; A note of caution: in contexts where the branch target is another + ;; block, this has to be the same successor as the one specified in the + ;; terminator branch of the current block. Otherwise, this might confuse + ;; register allocation by creating new invisible edges. + (JmpIf (cc CC) + (taken MachLabel)) + + ;; Two-way conditional branch: jcond cond target target. + ;; + ;; Emitted as a compound sequence; the MachBuffer will shrink it as + ;; appropriate. + (JmpCond (cc CC) + (taken MachLabel) + (not_taken MachLabel)) + + ;; Jump-table sequence, as one compound instruction (see note in lower.rs + ;; for rationale). + ;; + ;; The generated code sequence is described in the emit's function match + ;; arm for this instruction. + ;; + ;; See comment in lowering about the temporaries signedness. + (JmpTableSeq (idx Reg) + (tmp1 WritableReg) + (tmp2 WritableReg) + (default_target MachLabel) + (targets VecMachLabel) + (targets_for_term VecMachLabel)) + + ;; Indirect jump: jmpq (reg mem). + (JmpUnknown (target RegMem)) + + ;; Traps if the condition code is set. + (TrapIf (cc CC) + (trap_code TrapCode)) + + ;; A debug trap. + (Hlt) + + ;; An instruction that will always trigger the illegal instruction + ;; exception. + (Ud2 (trap_code TrapCode)) + + ;; Loads an external symbol in a register, with a relocation: + ;; + ;; movq $name@GOTPCREL(%rip), dst if PIC is enabled, or + ;; movabsq $name, dst otherwise. + (LoadExtName (dst WritableReg) + (name BoxExternalName) + (offset i64)) + + ;; ========================================= + ;; Instructions pertaining to atomic memory accesses. + + ;; A standard (native) `lock cmpxchg src, (amode)`, with register + ;; conventions: + ;; + ;; `mem` (read) address + ;; `replacement` (read) replacement value + ;; %rax (modified) in: expected value, out: value that was actually at `dst` + ;; %rflags is written. Do not assume anything about it after the instruction. + ;; + ;; The instruction "succeeded" iff the lowest `ty` bits of %rax + ;; afterwards are the same as they were before. + (LockCmpxchg (ty Type) ;; I8, I16, I32, or I64 + (replacement Reg) + (expected Reg) + (mem SyntheticAmode) + (dst_old WritableReg)) + + ;; A synthetic instruction, based on a loop around a native `lock + ;; cmpxchg` instruction. + ;; + ;; This atomically modifies a value in memory and returns the old value. + ;; The sequence consists of an initial "normal" load from `dst`, followed + ;; by a loop which computes the new value and tries to compare-and-swap + ;; ("CAS") it into `dst`, using the native instruction `lock + ;; cmpxchg{b,w,l,q}` . The loop iterates until the CAS is successful. + ;; If there is no contention, there will be only one pass through the + ;; loop body. The sequence does *not* perform any explicit memory fence + ;; instructions (mfence/sfence/lfence). + ;; + ;; Note that the transaction is atomic in the sense that, as observed by + ;; some other thread, `dst` either has the initial or final value, but no + ;; other. It isn't atomic in the sense of guaranteeing that no other + ;; thread writes to `dst` in between the initial load and the CAS -- but + ;; that would cause the CAS to fail unless the other thread's last write + ;; before the CAS wrote the same value that was already there. In other + ;; words, this implementation suffers (unavoidably) from the A-B-A + ;; problem. + ;; + ;; This instruction sequence has fixed register uses as follows: + ;; + ;; %r9 (read) address + ;; %r10 (read) second operand for `op` + ;; %r11 (written) scratch reg; value afterwards has no meaning + ;; %rax (written) the old value at %r9 + ;; %rflags is written. Do not assume anything about it after the instruction. + (AtomicRmwSeq (ty Type) ;; I8, I16, I32, or I64 + (op AtomicRmwOp) + (address Reg) + (operand Reg) + (temp WritableReg) + (dst_old WritableReg)) + + ;; A memory fence (mfence, lfence or sfence). + (Fence (kind FenceKind)) + + ;; ========================================= + ;; Meta-instructions generating no code. + + ;; Marker, no-op in generated code: SP "virtual offset" is adjusted. + ;; + ;; This controls how `MemArg::NominalSPOffset` args are lowered. + (VirtualSPOffsetAdj (offset i64)) + + ;; Provides a way to tell the register allocator that the upcoming + ;; sequence of instructions will overwrite `dst` so it should be + ;; considered as a `def`; use this with care. + ;; + ;; This is useful when we have a sequence of instructions whose register + ;; usages are nominally `mod`s, but such that the combination of + ;; operations creates a result that is independent of the initial + ;; register value. It's thus semantically a `def`, not a `mod`, when all + ;; the instructions are taken together, so we want to ensure the register + ;; is defined (its live-range starts) prior to the sequence to keep + ;; analyses happy. + ;; + ;; One alternative would be a compound instruction that somehow + ;; encapsulates the others and reports its own `def`s/`use`s/`mod`s; this + ;; adds complexity (the instruction list is no longer flat) and requires + ;; knowledge about semantics and initial-value independence anyway. + (XmmUninitializedValue (dst WritableXmm)) + + ;; A call to the `ElfTlsGetAddr` libcall. Returns address of TLS symbol + ;; in `rax`. + (ElfTlsGetAddr (symbol ExternalName)) + + ;; A Mach-O TLS symbol access. Returns address of the TLS symbol in + ;; `rax`. + (MachOTlsGetAddr (symbol ExternalName)) + + ;; A definition of a value label. + (ValueLabelMarker (reg Reg) + (label ValueLabel)) + + ;; An unwind pseudoinstruction describing the state of the machine at + ;; this program point. + (Unwind (inst UnwindInst)))) (type OperandSize extern (enum Size8 @@ -85,6 +489,13 @@ Size32 Size64)) +(type VCodeConstant (primitive VCodeConstant)) + +(type FenceKind extern + (enum MFence + LFence + SFence)) + ;; Get the `OperandSize` for a given `Type`, rounding smaller types up to 32 bits. (decl operand_size_of_type_32_64 (Type) OperandSize) (extern constructor operand_size_of_type_32_64 operand_size_of_type_32_64) @@ -112,6 +523,19 @@ And8 Or8)) +(type UnaryRmROpcode extern + (enum Bsr + Bsf + Lzcnt + Tzcnt + Popcnt)) + +(type DivOrRemKind extern + (enum SignedDiv + UnsignedDiv + SignedRem + UnsignedRem)) + (type SseOpcode extern (enum Addps Addpd @@ -330,7 +754,7 @@ (type Amode extern (enum)) -(decl amode_imm_reg_reg_shift (u32 Reg Reg u8) Amode) +(decl amode_imm_reg_reg_shift (u32 Gpr Gpr u8) Amode) (extern constructor amode_imm_reg_reg_shift amode_imm_reg_reg_shift) (decl amode_to_synthetic_amode (Amode) SyntheticAmode) @@ -397,13 +821,176 @@ (decl encode_fcmp_imm (FcmpImm) u8) (extern constructor encode_fcmp_imm encode_fcmp_imm) +;;;; Newtypes for Different Register Classes ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(type Gpr (primitive Gpr)) +(type WritableGpr (primitive WritableGpr)) +(type GprMem extern (enum)) +(type GprMemImm extern (enum)) + +(type Xmm (primitive Xmm)) +(type WritableXmm (primitive WritableXmm)) +(type XmmMem extern (enum)) +(type XmmMemImm extern (enum)) + +;; Convert a `WritableGpr` to a `WritableReg`. +(decl writable_gpr_to_reg (WritableGpr) WritableReg) +(extern constructor writable_gpr_to_reg writable_gpr_to_reg) + +;; Convert a `WritableXmm` to a `WritableReg`. +(decl writable_xmm_to_reg (WritableXmm) WritableReg) +(extern constructor writable_xmm_to_reg writable_xmm_to_reg) + +;; Convert a `WritableReg` to a `WritableXmm`. +(decl writable_reg_to_xmm (WritableReg) WritableXmm) +(extern constructor writable_reg_to_xmm writable_reg_to_xmm) + +;; Convert a `WritableXmm` to an `Xmm`. +(decl writable_xmm_to_xmm (WritableXmm) Xmm) +(extern constructor writable_xmm_to_xmm writable_xmm_to_xmm) + +;; Convert a `WritableGpr` to an `Gpr`. +(decl writable_gpr_to_gpr (WritableGpr) Gpr) +(extern constructor writable_gpr_to_gpr writable_gpr_to_gpr) + +;; Convert an `Gpr` to a `Reg`. +(decl gpr_to_reg (Gpr) Reg) +(extern constructor gpr_to_reg gpr_to_reg) + +;; Convert an `Xmm` to a `Reg`. +(decl xmm_to_reg (Xmm) Reg) +(extern constructor xmm_to_reg xmm_to_reg) + +;; Convert an `Xmm` into an `XmmMemImm`. +(decl xmm_to_xmm_mem_imm (Xmm) XmmMemImm) +(extern constructor xmm_to_xmm_mem_imm xmm_to_xmm_mem_imm) + +;; Allocate a new temporary GPR register. +(decl temp_writable_gpr () WritableGpr) +(extern constructor temp_writable_gpr temp_writable_gpr) + +;; Allocate a new temporary XMM register. +(decl temp_writable_xmm () WritableXmm) +(extern constructor temp_writable_xmm temp_writable_xmm) + +;; Construct a new `XmmMem` from the given `RegMem`. +;; +;; Asserts that the `RegMem`'s register, if any, is an XMM register. +(decl xmm_mem_new (RegMem) XmmMem) +(extern constructor xmm_mem_new xmm_mem_new) + +;; Construct a new `GprMemImm` from the given `RegMemImm`. +;; +;; Asserts that the `RegMemImm`'s register, if any, is an GPR register. +(decl gpr_mem_imm_new (RegMemImm) GprMemImm) +(extern constructor gpr_mem_imm_new gpr_mem_imm_new) + +;; Construct a new `XmmMemImm` from the given `RegMemImm`. +;; +;; Asserts that the `RegMemImm`'s register, if any, is an XMM register. +(decl xmm_mem_imm_new (RegMemImm) XmmMemImm) +(extern constructor xmm_mem_imm_new xmm_mem_imm_new) + +;; Construct a new `XmmMem` from an `Xmm`. +(decl xmm_to_xmm_mem (Xmm) XmmMem) +(extern constructor xmm_to_xmm_mem xmm_to_xmm_mem) + +;; Construct a new `XmmMem` from an `RegMem`. +(decl xmm_mem_to_reg_mem (XmmMem) RegMem) +(extern constructor xmm_mem_to_reg_mem xmm_mem_to_reg_mem) + +;; Convert a `GprMem` to a `RegMem`. +(decl gpr_mem_to_reg_mem (GprMem) RegMem) +(extern constructor gpr_mem_to_reg_mem gpr_mem_to_reg_mem) + +;; Construct a new `Xmm` from a `Reg`. +;; +;; Asserts that the register is a XMM. +(decl xmm_new (Reg) Xmm) +(extern constructor xmm_new xmm_new) + +;; Construct a new `Gpr` from a `Reg`. +;; +;; Asserts that the register is a GPR. +(decl gpr_new (Reg) Gpr) +(extern constructor gpr_new gpr_new) + +;; Construct a new `GprMem` from a `RegMem`. +;; +;; Asserts that the `RegMem`'s register, if any, is a GPR. +(decl gpr_mem_new (RegMem) GprMem) +(extern constructor gpr_mem_new gpr_mem_new) + +;; Construct a `GprMem` from a `Reg`. +;; +;; Asserts that the `Reg` is a GPR. +(decl reg_to_gpr_mem (Reg) GprMem) +(extern constructor reg_to_gpr_mem reg_to_gpr_mem) + +;; Put a value into a GPR. +;; +;; Asserts that the value goes into a GPR. +(decl put_in_gpr (Value) Gpr) +(rule (put_in_gpr val) + (gpr_new (put_in_reg val))) + +;; Put a value into a `GprMem`. +;; +;; Asserts that the value goes into a GPR. +(decl put_in_gpr_mem (Value) GprMem) +(rule (put_in_gpr_mem val) + (gpr_mem_new (put_in_reg_mem val))) + +;; Put a value into a `GprMemImm`. +;; +;; Asserts that the value goes into a GPR. +(decl put_in_gpr_mem_imm (Value) GprMemImm) +(rule (put_in_gpr_mem_imm val) + (gpr_mem_imm_new (put_in_reg_mem_imm val))) + +;; Put a value into a XMM. +;; +;; Asserts that the value goes into a XMM. +(decl put_in_xmm (Value) Xmm) +(rule (put_in_xmm val) + (xmm_new (put_in_reg val))) + +;; Put a value into a `XmmMem`. +;; +;; Asserts that the value goes into a XMM. +(decl put_in_xmm_mem (Value) XmmMem) +(rule (put_in_xmm_mem val) + (xmm_mem_new (put_in_reg_mem val))) + +;; Put a value into a `XmmMemImm`. +;; +;; Asserts that the value goes into a XMM. +(decl put_in_xmm_mem_imm (Value) XmmMemImm) +(rule (put_in_xmm_mem_imm val) + (xmm_mem_imm_new (put_in_reg_mem_imm val))) + +;; Construct a `ValueRegs` out of a single GPR register. +(decl value_gpr (Gpr) ValueRegs) +(rule (value_gpr x) + (value_reg (gpr_to_reg x))) + +;; Construct a `ValueRegs` out of two GPR registers. +(decl value_gprs (Gpr Gpr) ValueRegs) +(rule (value_gprs x y) + (value_regs (gpr_to_reg x) (gpr_to_reg y))) + +;; Construct a `ValueRegs` out of a single XMM register. +(decl value_xmm (Xmm) ValueRegs) +(rule (value_xmm x) + (value_reg (xmm_to_reg x))) + ;;;; Helpers for Getting Particular Physical Registers ;;;;;;;;;;;;;;;;;;;;;;;;; ;; ;; These should only be used for legalization purposes, when we can't otherwise ;; rely on something like `Inst::mov_mitosis` to put an operand into the ;; appropriate physical register for whatever reason. -(decl xmm0 () WritableReg) +(decl xmm0 () WritableXmm) (extern constructor xmm0 xmm0) ;;;; Helpers for Querying Enabled ISA Extensions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -507,8 +1094,8 @@ (rule (sse_xor_op $F64X2) (SseOpcode.Xorpd)) (rule (sse_xor_op (multi_lane _bits _lanes)) (SseOpcode.Pxor)) -;; Performs an xor operation of the two operands specified -(decl sse_xor (Type Reg RegMem) Reg) +;; Performs an xor operation of the two operands specified. +(decl sse_xor (Type Xmm XmmMem) Xmm) (rule (sse_xor ty x y) (xmm_rm_r ty (sse_xor_op ty) x y)) ;; Determine the appropriate operation to compare two vectors of the specified @@ -532,45 +1119,46 @@ ;; then it runs the risk of comparing NaN against NaN and not actually producing ;; an all-ones mask. By using integer comparision operations we're guaranteeed ;; that everything is equal to itself. -(decl vector_all_ones (Type) Reg) +(decl vector_all_ones (Type) Xmm) (rule (vector_all_ones ty) - (let ((wr WritableReg (temp_writable_reg ty)) - (r Reg (writable_reg_to_reg wr)) + (let ((wr WritableXmm (temp_writable_xmm)) + (r Xmm (writable_xmm_to_xmm wr)) (_ Unit (emit (MInst.XmmRmR (sse_cmp_op $I32X4) r - (RegMem.Reg r) + (xmm_to_xmm_mem r) wr)))) r)) ;; Helper for creating an SSE register holding an `i64x2` from two `i64` values. -(decl make_i64x2_from_lanes (RegMem RegMem) Reg) +(decl make_i64x2_from_lanes (GprMem GprMem) Xmm) (rule (make_i64x2_from_lanes lo hi) - (let ((dst_w WritableReg (temp_writable_reg $I64X2)) - (dst_r Reg (writable_reg_to_reg dst_w)) - (_0 Unit (emit (MInst.XmmUninitializedValue dst_w))) + (let ((dst_xmm_w WritableXmm (temp_writable_xmm)) + (dst_reg_w WritableReg (writable_xmm_to_reg dst_xmm_w)) + (dst_xmm_r Xmm (writable_xmm_to_xmm dst_xmm_w)) + (dst_reg_r Reg (xmm_to_reg dst_xmm_r)) + (_0 Unit (emit (MInst.XmmUninitializedValue dst_xmm_w))) (_1 Unit (emit (MInst.XmmRmRImm (SseOpcode.Pinsrd) - dst_r - lo - dst_w + dst_reg_r + (gpr_mem_to_reg_mem lo) + dst_reg_w 0 (OperandSize.Size64)))) (_2 Unit (emit (MInst.XmmRmRImm (SseOpcode.Pinsrd) - dst_r - hi - dst_w + dst_reg_r + (gpr_mem_to_reg_mem hi) + dst_reg_w 1 (OperandSize.Size64))))) - dst_r)) + dst_xmm_r)) ;; Move a `RegMemImm.Reg` operand to an XMM register, if necessary. -(decl reg_mem_imm_to_xmm (RegMemImm) RegMemImm) -(rule (reg_mem_imm_to_xmm rmi @ (RegMemImm.Mem _)) rmi) -(rule (reg_mem_imm_to_xmm rmi @ (RegMemImm.Imm _)) rmi) -(rule (reg_mem_imm_to_xmm (RegMemImm.Reg r)) - (RegMemImm.Reg (gpr_to_xmm $I8X16 - (SseOpcode.Movd) - (RegMem.Reg r) - (OperandSize.Size32)))) +(decl mov_rmi_to_xmm (RegMemImm) XmmMemImm) +(rule (mov_rmi_to_xmm rmi @ (RegMemImm.Mem _)) (xmm_mem_imm_new rmi)) +(rule (mov_rmi_to_xmm rmi @ (RegMemImm.Imm _)) (xmm_mem_imm_new rmi)) +(rule (mov_rmi_to_xmm (RegMemImm.Reg r)) + (xmm_to_xmm_mem_imm (gpr_to_xmm (SseOpcode.Movd) + (reg_to_gpr_mem r) + (OperandSize.Size32)))) ;;;; Helpers for Emitting Loads ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -588,24 +1176,24 @@ (writable_reg_to_reg dst))) (rule (x64_load $F32 addr _ext_kind) - (xmm_unary_rm_r (SseOpcode.Movss) - (synthetic_amode_to_reg_mem addr))) + (xmm_to_reg (xmm_unary_rm_r (SseOpcode.Movss) + (xmm_mem_new (synthetic_amode_to_reg_mem addr))))) (rule (x64_load $F64 addr _ext_kind) - (xmm_unary_rm_r (SseOpcode.Movsd) - (synthetic_amode_to_reg_mem addr))) + (xmm_to_reg (xmm_unary_rm_r (SseOpcode.Movsd) + (xmm_mem_new (synthetic_amode_to_reg_mem addr))))) (rule (x64_load $F32X4 addr _ext_kind) - (xmm_unary_rm_r (SseOpcode.Movups) - (synthetic_amode_to_reg_mem addr))) + (xmm_to_reg (xmm_unary_rm_r (SseOpcode.Movups) + (xmm_mem_new (synthetic_amode_to_reg_mem addr))))) (rule (x64_load $F64X2 addr _ext_kind) - (xmm_unary_rm_r (SseOpcode.Movupd) - (synthetic_amode_to_reg_mem addr))) + (xmm_to_reg (xmm_unary_rm_r (SseOpcode.Movupd) + (xmm_mem_new (synthetic_amode_to_reg_mem addr))))) (rule (x64_load (multi_lane _bits _lanes) addr _ext_kind) - (xmm_unary_rm_r (SseOpcode.Movdqu) - (synthetic_amode_to_reg_mem addr))) + (xmm_to_reg (xmm_unary_rm_r (SseOpcode.Movdqu) + (xmm_mem_new (synthetic_amode_to_reg_mem addr))))) ;;;; Instruction Constructors ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; @@ -735,11 +1323,15 @@ ;; `f32` immediates. (rule (imm $F32 bits) - (gpr_to_xmm $F32 (SseOpcode.Movd) (RegMem.Reg (imm $I32 bits)) (OperandSize.Size32))) + (xmm_to_reg (gpr_to_xmm (SseOpcode.Movd) + (gpr_mem_new (RegMem.Reg (imm $I32 bits))) + (OperandSize.Size32)))) ;; `f64` immediates. (rule (imm $F64 bits) - (gpr_to_xmm $F64 (SseOpcode.Movq) (RegMem.Reg (imm $I64 bits)) (OperandSize.Size64))) + (xmm_to_reg (gpr_to_xmm (SseOpcode.Movq) + (gpr_mem_new (RegMem.Reg (imm $I64 bits))) + (OperandSize.Size64)))) (decl nonzero_u64_fits_in_u32 (u64) u64) (extern extractor nonzero_u64_fits_in_u32 nonzero_u64_fits_in_u32) @@ -766,35 +1358,35 @@ ;; Special case for zero immediates with vector types, they turn into an xor ;; specific to the vector type. (rule (imm ty @ (multi_lane _bits _lanes) 0) - (let ((wr WritableReg (temp_writable_reg ty)) - (r Reg (writable_reg_to_reg wr)) + (let ((wr WritableXmm (temp_writable_xmm)) + (r Xmm (writable_xmm_to_xmm wr)) (_ Unit (emit (MInst.XmmRmR (sse_xor_op ty) r - (RegMem.Reg r) + (xmm_to_xmm_mem r) wr)))) - r)) + (xmm_to_reg r))) ;; Special case for `f32` zero immediates to use `xorps`. (rule (imm $F32 0) - (let ((wr WritableReg (temp_writable_reg $F32)) - (r Reg (writable_reg_to_reg wr)) + (let ((wr WritableXmm (temp_writable_xmm)) + (r Xmm (writable_xmm_to_xmm wr)) (_ Unit (emit (MInst.XmmRmR (SseOpcode.Xorps) r - (RegMem.Reg r) + (xmm_to_xmm_mem r) wr)))) - r)) + (xmm_to_reg r))) ;; TODO: use cmpeqps for all 1s ;; Special case for `f64` zero immediates to use `xorpd`. (rule (imm $F64 0) - (let ((wr WritableReg (temp_writable_reg $F64)) - (r Reg (writable_reg_to_reg wr)) + (let ((wr WritableXmm (temp_writable_xmm)) + (r Xmm (writable_xmm_to_xmm wr)) (_ Unit (emit (MInst.XmmRmR (SseOpcode.Xorpd) r - (RegMem.Reg r) + (xmm_to_xmm_mem r) wr)))) - r)) + (xmm_to_reg r))) ;; TODO: use cmpeqpd for all 1s @@ -877,199 +1469,199 @@ (writable_reg_to_reg dst))) ;; Helper for creating `MInst.XmmRmR` instructions. -(decl xmm_rm_r (Type SseOpcode Reg RegMem) Reg) +(decl xmm_rm_r (Type SseOpcode Xmm XmmMem) Xmm) (rule (xmm_rm_r ty op src1 src2) - (let ((dst WritableReg (temp_writable_reg ty)) + (let ((dst WritableXmm (temp_writable_xmm)) (_ Unit (emit (MInst.XmmRmR op src1 src2 dst)))) - (writable_reg_to_reg dst))) + (writable_xmm_to_xmm dst))) ;; Helper for creating `paddb` instructions. -(decl paddb (Reg RegMem) Reg) +(decl paddb (Xmm XmmMem) Xmm) (rule (paddb src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Paddb) src1 src2)) ;; Helper for creating `paddw` instructions. -(decl paddw (Reg RegMem) Reg) +(decl paddw (Xmm XmmMem) Xmm) (rule (paddw src1 src2) (xmm_rm_r $I16X8 (SseOpcode.Paddw) src1 src2)) ;; Helper for creating `paddd` instructions. -(decl paddd (Reg RegMem) Reg) +(decl paddd (Xmm XmmMem) Xmm) (rule (paddd src1 src2) (xmm_rm_r $I32X4 (SseOpcode.Paddd) src1 src2)) ;; Helper for creating `paddq` instructions. -(decl paddq (Reg RegMem) Reg) +(decl paddq (Xmm XmmMem) Xmm) (rule (paddq src1 src2) (xmm_rm_r $I64X2 (SseOpcode.Paddq) src1 src2)) ;; Helper for creating `paddsb` instructions. -(decl paddsb (Reg RegMem) Reg) +(decl paddsb (Xmm XmmMem) Xmm) (rule (paddsb src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Paddsb) src1 src2)) ;; Helper for creating `paddsw` instructions. -(decl paddsw (Reg RegMem) Reg) +(decl paddsw (Xmm XmmMem) Xmm) (rule (paddsw src1 src2) (xmm_rm_r $I16X8 (SseOpcode.Paddsw) src1 src2)) ;; Helper for creating `paddusb` instructions. -(decl paddusb (Reg RegMem) Reg) +(decl paddusb (Xmm XmmMem) Xmm) (rule (paddusb src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Paddusb) src1 src2)) ;; Helper for creating `paddusw` instructions. -(decl paddusw (Reg RegMem) Reg) +(decl paddusw (Xmm XmmMem) Xmm) (rule (paddusw src1 src2) (xmm_rm_r $I16X8 (SseOpcode.Paddusw) src1 src2)) ;; Helper for creating `psubb` instructions. -(decl psubb (Reg RegMem) Reg) +(decl psubb (Xmm XmmMem) Xmm) (rule (psubb src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Psubb) src1 src2)) ;; Helper for creating `psubw` instructions. -(decl psubw (Reg RegMem) Reg) +(decl psubw (Xmm XmmMem) Xmm) (rule (psubw src1 src2) (xmm_rm_r $I16X8 (SseOpcode.Psubw) src1 src2)) ;; Helper for creating `psubd` instructions. -(decl psubd (Reg RegMem) Reg) +(decl psubd (Xmm XmmMem) Xmm) (rule (psubd src1 src2) (xmm_rm_r $I32X4 (SseOpcode.Psubd) src1 src2)) ;; Helper for creating `psubq` instructions. -(decl psubq (Reg RegMem) Reg) +(decl psubq (Xmm XmmMem) Xmm) (rule (psubq src1 src2) (xmm_rm_r $I64X2 (SseOpcode.Psubq) src1 src2)) ;; Helper for creating `psubsb` instructions. -(decl psubsb (Reg RegMem) Reg) +(decl psubsb (Xmm XmmMem) Xmm) (rule (psubsb src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Psubsb) src1 src2)) ;; Helper for creating `psubsw` instructions. -(decl psubsw (Reg RegMem) Reg) +(decl psubsw (Xmm XmmMem) Xmm) (rule (psubsw src1 src2) (xmm_rm_r $I16X8 (SseOpcode.Psubsw) src1 src2)) ;; Helper for creating `psubusb` instructions. -(decl psubusb (Reg RegMem) Reg) +(decl psubusb (Xmm XmmMem) Xmm) (rule (psubusb src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Psubusb) src1 src2)) ;; Helper for creating `psubusw` instructions. -(decl psubusw (Reg RegMem) Reg) +(decl psubusw (Xmm XmmMem) Xmm) (rule (psubusw src1 src2) (xmm_rm_r $I16X8 (SseOpcode.Psubusw) src1 src2)) ;; Helper for creating `pavgb` instructions. -(decl pavgb (Reg RegMem) Reg) +(decl pavgb (Xmm XmmMem) Xmm) (rule (pavgb src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Pavgb) src1 src2)) ;; Helper for creating `pavgw` instructions. -(decl pavgw (Reg RegMem) Reg) +(decl pavgw (Xmm XmmMem) Xmm) (rule (pavgw src1 src2) (xmm_rm_r $I16X8 (SseOpcode.Pavgw) src1 src2)) ;; Helper for creating `pand` instructions. -(decl pand (Reg RegMem) Reg) +(decl pand (Xmm XmmMem) Xmm) (rule (pand src1 src2) (xmm_rm_r $F32X4 (SseOpcode.Pand) src1 src2)) ;; Helper for creating `andps` instructions. -(decl andps (Reg RegMem) Reg) +(decl andps (Xmm XmmMem) Xmm) (rule (andps src1 src2) (xmm_rm_r $F32X4 (SseOpcode.Andps) src1 src2)) ;; Helper for creating `andpd` instructions. -(decl andpd (Reg RegMem) Reg) +(decl andpd (Xmm XmmMem) Xmm) (rule (andpd src1 src2) (xmm_rm_r $F64X2 (SseOpcode.Andpd) src1 src2)) ;; Helper for creating `por` instructions. -(decl por (Reg RegMem) Reg) +(decl por (Xmm XmmMem) Xmm) (rule (por src1 src2) (xmm_rm_r $F32X4 (SseOpcode.Por) src1 src2)) ;; Helper for creating `orps` instructions. -(decl orps (Reg RegMem) Reg) +(decl orps (Xmm XmmMem) Xmm) (rule (orps src1 src2) (xmm_rm_r $F32X4 (SseOpcode.Orps) src1 src2)) ;; Helper for creating `orpd` instructions. -(decl orpd (Reg RegMem) Reg) +(decl orpd (Xmm XmmMem) Xmm) (rule (orpd src1 src2) (xmm_rm_r $F64X2 (SseOpcode.Orpd) src1 src2)) ;; Helper for creating `pxor` instructions. -(decl pxor (Reg RegMem) Reg) +(decl pxor (Xmm XmmMem) Xmm) (rule (pxor src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Pxor) src1 src2)) ;; Helper for creating `xorps` instructions. -(decl xorps (Reg RegMem) Reg) +(decl xorps (Xmm XmmMem) Xmm) (rule (xorps src1 src2) (xmm_rm_r $F32X4 (SseOpcode.Xorps) src1 src2)) ;; Helper for creating `xorpd` instructions. -(decl xorpd (Reg RegMem) Reg) +(decl xorpd (Xmm XmmMem) Xmm) (rule (xorpd src1 src2) (xmm_rm_r $F64X2 (SseOpcode.Xorpd) src1 src2)) ;; Helper for creating `pmullw` instructions. -(decl pmullw (Reg RegMem) Reg) +(decl pmullw (Xmm XmmMem) Xmm) (rule (pmullw src1 src2) (xmm_rm_r $I16X8 (SseOpcode.Pmullw) src1 src2)) ;; Helper for creating `pmulld` instructions. -(decl pmulld (Reg RegMem) Reg) +(decl pmulld (Xmm XmmMem) Xmm) (rule (pmulld src1 src2) (xmm_rm_r $I16X8 (SseOpcode.Pmulld) src1 src2)) ;; Helper for creating `pmulhw` instructions. -(decl pmulhw (Reg RegMem) Reg) +(decl pmulhw (Xmm XmmMem) Xmm) (rule (pmulhw src1 src2) (xmm_rm_r $I16X8 (SseOpcode.Pmulhw) src1 src2)) ;; Helper for creating `pmulhuw` instructions. -(decl pmulhuw (Reg RegMem) Reg) +(decl pmulhuw (Xmm XmmMem) Xmm) (rule (pmulhuw src1 src2) (xmm_rm_r $I16X8 (SseOpcode.Pmulhuw) src1 src2)) ;; Helper for creating `pmuldq` instructions. -(decl pmuldq (Reg RegMem) Reg) +(decl pmuldq (Xmm XmmMem) Xmm) (rule (pmuldq src1 src2) (xmm_rm_r $I16X8 (SseOpcode.Pmuldq) src1 src2)) ;; Helper for creating `pmuludq` instructions. -(decl pmuludq (Reg RegMem) Reg) +(decl pmuludq (Xmm XmmMem) Xmm) (rule (pmuludq src1 src2) (xmm_rm_r $I64X2 (SseOpcode.Pmuludq) src1 src2)) ;; Helper for creating `punpckhwd` instructions. -(decl punpckhwd (Reg RegMem) Reg) +(decl punpckhwd (Xmm XmmMem) Xmm) (rule (punpckhwd src1 src2) (xmm_rm_r $I16X8 (SseOpcode.Punpckhwd) src1 src2)) ;; Helper for creating `punpcklwd` instructions. -(decl punpcklwd (Reg RegMem) Reg) +(decl punpcklwd (Xmm XmmMem) Xmm) (rule (punpcklwd src1 src2) (xmm_rm_r $I16X8 (SseOpcode.Punpcklwd) src1 src2)) ;; Helper for creating `andnps` instructions. -(decl andnps (Reg RegMem) Reg) +(decl andnps (Xmm XmmMem) Xmm) (rule (andnps src1 src2) (xmm_rm_r $F32X4 (SseOpcode.Andnps) src1 src2)) ;; Helper for creating `andnpd` instructions. -(decl andnpd (Reg RegMem) Reg) +(decl andnpd (Xmm XmmMem) Xmm) (rule (andnpd src1 src2) (xmm_rm_r $F64X2 (SseOpcode.Andnpd) src1 src2)) ;; Helper for creating `pandn` instructions. -(decl pandn (Reg RegMem) Reg) +(decl pandn (Xmm XmmMem) Xmm) (rule (pandn src1 src2) (xmm_rm_r $F64X2 (SseOpcode.Pandn) src1 src2)) @@ -1084,367 +1676,387 @@ (rule (sse_mov_op (multi_lane _bits _lanes)) (SseOpcode.Movdqa)) ;; Helper for creating `blendvp{d,s}` and `pblendvb` instructions. -(decl sse_blend (Type RegMem RegMem Reg) Reg) +(decl sse_blend (Type XmmMem XmmMem Xmm) Xmm) (rule (sse_blend ty mask src1 src2) ;; Move the mask into `xmm0`, as blend instructions implicitly operate on ;; that register. (This kind of thing would normally happen inside of ;; `Inst::mov_mitosis`, but has to happen here, where we still have the ;; mask register, because the mask is implicit and doesn't appear in the ;; `Inst` itself.) - (let ((mask2 WritableReg (xmm0)) - (_ Unit (emit (MInst.XmmUnaryRmR (sse_mov_op ty) mask mask2)))) + (let ((mask2 WritableXmm (xmm0)) + (_ Unit (emit (MInst.XmmUnaryRmR (sse_mov_op ty) + mask + mask2)))) (xmm_rm_r ty (sse_blend_op ty) src2 src1))) ;; Helper for creating `blendvpd` instructions. -(decl blendvpd (Reg RegMem Reg) Reg) +(decl blendvpd (Xmm XmmMem Xmm) Xmm) (rule (blendvpd src1 src2 mask) ;; Move the mask into `xmm0`, as `blendvpd` implicitly operates on that ;; register. (This kind of thing would normally happen inside of ;; `Inst::mov_mitosis`, but has to happen here, where we still have the ;; mask register, because the mask is implicit and doesn't appear in the ;; `Inst` itself.) - (let ((mask2 WritableReg (xmm0)) - (_ Unit (emit (MInst.XmmUnaryRmR (SseOpcode.Movapd) (RegMem.Reg mask) mask2)))) + (let ((mask2 WritableXmm (xmm0)) + (_ Unit (emit (MInst.XmmUnaryRmR (SseOpcode.Movapd) + (xmm_to_xmm_mem mask) + mask2)))) (xmm_rm_r $F64X2 (SseOpcode.Blendvpd) src1 src2))) ;; Helper for creating `movsd` instructions. -(decl movsd (Reg RegMem) Reg) +(decl movsd (Xmm XmmMem) Xmm) (rule (movsd src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Movsd) src1 src2)) ;; Helper for creating `movlhps` instructions. -(decl movlhps (Reg RegMem) Reg) +(decl movlhps (Xmm XmmMem) Xmm) (rule (movlhps src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Movlhps) src1 src2)) ;; Helper for creating `pmaxsb` instructions. -(decl pmaxsb (Reg RegMem) Reg) +(decl pmaxsb (Xmm XmmMem) Xmm) (rule (pmaxsb src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Pmaxsb) src1 src2)) ;; Helper for creating `pmaxsw` instructions. -(decl pmaxsw (Reg RegMem) Reg) +(decl pmaxsw (Xmm XmmMem) Xmm) (rule (pmaxsw src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Pmaxsw) src1 src2)) ;; Helper for creating `pmaxsd` instructions. -(decl pmaxsd (Reg RegMem) Reg) +(decl pmaxsd (Xmm XmmMem) Xmm) (rule (pmaxsd src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Pmaxsd) src1 src2)) ;; Helper for creating `pminsb` instructions. -(decl pminsb (Reg RegMem) Reg) +(decl pminsb (Xmm XmmMem) Xmm) (rule (pminsb src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Pminsb) src1 src2)) ;; Helper for creating `pminsw` instructions. -(decl pminsw (Reg RegMem) Reg) +(decl pminsw (Xmm XmmMem) Xmm) (rule (pminsw src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Pminsw) src1 src2)) ;; Helper for creating `pminsd` instructions. -(decl pminsd (Reg RegMem) Reg) +(decl pminsd (Xmm XmmMem) Xmm) (rule (pminsd src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Pminsd) src1 src2)) ;; Helper for creating `pmaxub` instructions. -(decl pmaxub (Reg RegMem) Reg) +(decl pmaxub (Xmm XmmMem) Xmm) (rule (pmaxub src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Pmaxub) src1 src2)) ;; Helper for creating `pmaxuw` instructions. -(decl pmaxuw (Reg RegMem) Reg) +(decl pmaxuw (Xmm XmmMem) Xmm) (rule (pmaxuw src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Pmaxuw) src1 src2)) ;; Helper for creating `pmaxud` instructions. -(decl pmaxud (Reg RegMem) Reg) +(decl pmaxud (Xmm XmmMem) Xmm) (rule (pmaxud src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Pmaxud) src1 src2)) ;; Helper for creating `pminub` instructions. -(decl pminub (Reg RegMem) Reg) +(decl pminub (Xmm XmmMem) Xmm) (rule (pminub src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Pminub) src1 src2)) ;; Helper for creating `pminuw` instructions. -(decl pminuw (Reg RegMem) Reg) +(decl pminuw (Xmm XmmMem) Xmm) (rule (pminuw src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Pminuw) src1 src2)) ;; Helper for creating `pminud` instructions. -(decl pminud (Reg RegMem) Reg) +(decl pminud (Xmm XmmMem) Xmm) (rule (pminud src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Pminud) src1 src2)) ;; Helper for creating `punpcklbw` instructions. -(decl punpcklbw (Reg RegMem) Reg) +(decl punpcklbw (Xmm XmmMem) Xmm) (rule (punpcklbw src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Punpcklbw) src1 src2)) ;; Helper for creating `punpckhbw` instructions. -(decl punpckhbw (Reg RegMem) Reg) +(decl punpckhbw (Xmm XmmMem) Xmm) (rule (punpckhbw src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Punpckhbw) src1 src2)) ;; Helper for creating `packsswb` instructions. -(decl packsswb (Reg RegMem) Reg) +(decl packsswb (Xmm XmmMem) Xmm) (rule (packsswb src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Packsswb) src1 src2)) ;; Helper for creating `MInst.XmmRmRImm` instructions. -(decl xmm_rm_r_imm (SseOpcode Reg RegMem u8 OperandSize) Reg) +(decl xmm_rm_r_imm (SseOpcode Reg RegMem u8 OperandSize) Xmm) (rule (xmm_rm_r_imm op src1 src2 imm size) - (let ((dst WritableReg (temp_writable_reg $I8X16)) + (let ((dst WritableXmm (temp_writable_xmm)) (_ Unit (emit (MInst.XmmRmRImm op src1 src2 - dst + (writable_xmm_to_reg dst) imm size)))) - (writable_reg_to_reg dst))) + (writable_xmm_to_xmm dst))) ;; Helper for creating `palignr` instructions. -(decl palignr (Reg RegMem u8 OperandSize) Reg) +(decl palignr (Xmm XmmMem u8 OperandSize) Xmm) (rule (palignr src1 src2 imm size) (xmm_rm_r_imm (SseOpcode.Palignr) - src1 - src2 + (xmm_to_reg src1) + (xmm_mem_to_reg_mem src2) imm size)) +;; Helper for creating `cmpps` instructions. +(decl cmpps (Xmm XmmMem FcmpImm) Xmm) +(rule (cmpps src1 src2 imm) + (xmm_rm_r_imm (SseOpcode.Cmpps) + (xmm_to_reg src1) + (xmm_mem_to_reg_mem src2) + (encode_fcmp_imm imm) + (OperandSize.Size32))) + +;; Helper for creating `pinsrb` instructions. +(decl pinsrb (Xmm GprMem u8) Xmm) +(rule (pinsrb src1 src2 lane) + (xmm_rm_r_imm (SseOpcode.Pinsrb) + (xmm_to_reg src1) + (gpr_mem_to_reg_mem src2) + lane + (OperandSize.Size32))) + +;; Helper for creating `pinsrw` instructions. +(decl pinsrw (Xmm GprMem u8) Xmm) +(rule (pinsrw src1 src2 lane) + (xmm_rm_r_imm (SseOpcode.Pinsrw) + (xmm_to_reg src1) + (gpr_mem_to_reg_mem src2) + lane + (OperandSize.Size32))) + +;; Helper for creating `pinsrd` instructions. +(decl pinsrd (Xmm GprMem u8 OperandSize) Xmm) +(rule (pinsrd src1 src2 lane size) + (xmm_rm_r_imm (SseOpcode.Pinsrd) + (xmm_to_reg src1) + (gpr_mem_to_reg_mem src2) + lane + size)) + +;; Helper for creating `insertps` instructions. +(decl insertps (Xmm XmmMem u8) Xmm) +(rule (insertps src1 src2 lane) + (xmm_rm_r_imm (SseOpcode.Insertps) + (xmm_to_reg src1) + (xmm_mem_to_reg_mem src2) + lane + (OperandSize.Size32))) + ;; Helper for creating `pshufd` instructions. -(decl pshufd (RegMem u8 OperandSize) Reg) +(decl pshufd (XmmMem u8 OperandSize) Xmm) (rule (pshufd src imm size) - (let ((w_dst WritableReg (temp_writable_reg $I8X16)) - (dst Reg (writable_reg_to_reg w_dst)) + (let ((w_dst WritableXmm (temp_writable_xmm)) + (dst Xmm (writable_xmm_to_xmm w_dst)) (_ Unit (emit (MInst.XmmRmRImm (SseOpcode.Pshufd) - dst - src - w_dst + (xmm_to_reg dst) + (xmm_mem_to_reg_mem src) + (writable_xmm_to_reg w_dst) imm size)))) dst)) ;; Helper for creating `MInst.XmmUnaryRmR` instructions. -(decl xmm_unary_rm_r (SseOpcode RegMem) Reg) +(decl xmm_unary_rm_r (SseOpcode XmmMem) Xmm) (rule (xmm_unary_rm_r op src) - (let ((dst WritableReg (temp_writable_reg $I8X16)) + (let ((dst WritableXmm (temp_writable_xmm)) (_ Unit (emit (MInst.XmmUnaryRmR op src dst)))) - (writable_reg_to_reg dst))) + (writable_xmm_to_xmm dst))) ;; Helper for creating `pmovsxbw` instructions. -(decl pmovsxbw (RegMem) Reg) +(decl pmovsxbw (XmmMem) Xmm) (rule (pmovsxbw src) (xmm_unary_rm_r (SseOpcode.Pmovsxbw) src)) ;; Helper for creating `pmovzxbw` instructions. -(decl pmovzxbw (RegMem) Reg) +(decl pmovzxbw (XmmMem) Xmm) (rule (pmovzxbw src) (xmm_unary_rm_r (SseOpcode.Pmovzxbw) src)) ;; Helper for creating `pabsb` instructions. -(decl pabsb (RegMem) Reg) +(decl pabsb (XmmMem) Xmm) (rule (pabsb src) (xmm_unary_rm_r (SseOpcode.Pabsb) src)) ;; Helper for creating `pabsw` instructions. -(decl pabsw (RegMem) Reg) +(decl pabsw (XmmMem) Xmm) (rule (pabsw src) (xmm_unary_rm_r (SseOpcode.Pabsw) src)) ;; Helper for creating `pabsd` instructions. -(decl pabsd (RegMem) Reg) +(decl pabsd (XmmMem) Xmm) (rule (pabsd src) (xmm_unary_rm_r (SseOpcode.Pabsd) src)) ;; Helper for creating `MInst.XmmUnaryRmREvex` instructions. -(decl xmm_unary_rm_r_evex (Avx512Opcode RegMem) Reg) +(decl xmm_unary_rm_r_evex (Avx512Opcode XmmMem) Xmm) (rule (xmm_unary_rm_r_evex op src) - (let ((dst WritableReg (temp_writable_reg $I8X16)) + (let ((dst WritableXmm (temp_writable_xmm)) (_ Unit (emit (MInst.XmmUnaryRmREvex op src dst)))) - (writable_reg_to_reg dst))) + (writable_xmm_to_xmm dst))) ;; Helper for creating `vpabsq` instructions. -(decl vpabsq (RegMem) Reg) +(decl vpabsq (XmmMem) Xmm) (rule (vpabsq src) (xmm_unary_rm_r_evex (Avx512Opcode.Vpabsq) src)) ;; Helper for creating `MInst.XmmRmREvex` instructions. -(decl xmm_rm_r_evex (Avx512Opcode RegMem Reg) Reg) +(decl xmm_rm_r_evex (Avx512Opcode XmmMem Xmm) Xmm) (rule (xmm_rm_r_evex op src1 src2) - (let ((dst WritableReg (temp_writable_reg $I8X16)) + (let ((dst WritableXmm (temp_writable_xmm)) (_ Unit (emit (MInst.XmmRmREvex op src1 src2 dst)))) - (writable_reg_to_reg dst))) + (writable_xmm_to_xmm dst))) ;; Helper for creating `vpmullq` instructions. ;; ;; Requires AVX-512 vl and dq. -(decl vpmullq (RegMem Reg) Reg) +(decl vpmullq (XmmMem Xmm) Xmm) (rule (vpmullq src1 src2) (xmm_rm_r_evex (Avx512Opcode.Vpmullq) src1 src2)) -;; Helper for creating `MInst.XmmRmiReg` instructions. -(decl xmm_rmi_reg (SseOpcode Reg RegMemImm) Reg) -(rule (xmm_rmi_reg op src1 src2) - (let ((dst WritableReg (temp_writable_reg $I8X16)) +;; Helper for creating `MInst.MulHi` instructions. +;; +;; Returns the (lo, hi) register halves of the multiplication. +(decl mul_hi (Type bool Reg RegMem) ValueRegs) +(rule (mul_hi ty signed src1 src2) + (let ((dst_lo WritableReg (temp_writable_reg ty)) + (dst_hi WritableReg (temp_writable_reg ty)) + (size OperandSize (operand_size_of_type_32_64 ty)) + (_ Unit (emit (MInst.MulHi size + signed + src1 + src2 + dst_lo + dst_hi)))) + (value_regs (writable_reg_to_reg dst_lo) + (writable_reg_to_reg dst_hi)))) + +;; Helper for creating `mul` instructions that return both the lower and +;; (unsigned) higher halves of the result. +(decl mulhi_u (Type Reg RegMem) ValueRegs) +(rule (mulhi_u ty src1 src2) + (mul_hi ty $false src1 src2)) + +;; Helper for creating `MInst.XmmRmiXmm` instructions. +(decl xmm_rmi_xmm (SseOpcode Xmm XmmMemImm) Xmm) +(rule (xmm_rmi_xmm op src1 src2) + (let ((dst WritableXmm (temp_writable_xmm)) (_ Unit (emit (MInst.XmmRmiReg op src1 src2 dst)))) - (writable_reg_to_reg dst))) + (writable_xmm_to_xmm dst))) ;; Helper for creating `psllw` instructions. -(decl psllw (Reg RegMemImm) Reg) +(decl psllw (Xmm XmmMemImm) Xmm) (rule (psllw src1 src2) - (xmm_rmi_reg (SseOpcode.Psllw) src1 src2)) + (xmm_rmi_xmm (SseOpcode.Psllw) src1 src2)) ;; Helper for creating `pslld` instructions. -(decl pslld (Reg RegMemImm) Reg) +(decl pslld (Xmm XmmMemImm) Xmm) (rule (pslld src1 src2) - (xmm_rmi_reg (SseOpcode.Pslld) src1 src2)) + (xmm_rmi_xmm (SseOpcode.Pslld) src1 src2)) ;; Helper for creating `psllq` instructions. -(decl psllq (Reg RegMemImm) Reg) +(decl psllq (Xmm XmmMemImm) Xmm) (rule (psllq src1 src2) - (xmm_rmi_reg (SseOpcode.Psllq) src1 src2)) + (xmm_rmi_xmm (SseOpcode.Psllq) src1 src2)) ;; Helper for creating `psrlw` instructions. -(decl psrlw (Reg RegMemImm) Reg) +(decl psrlw (Xmm XmmMemImm) Xmm) (rule (psrlw src1 src2) - (xmm_rmi_reg (SseOpcode.Psrlw) src1 src2)) + (xmm_rmi_xmm (SseOpcode.Psrlw) src1 src2)) ;; Helper for creating `psrld` instructions. -(decl psrld (Reg RegMemImm) Reg) +(decl psrld (Xmm XmmMemImm) Xmm) (rule (psrld src1 src2) - (xmm_rmi_reg (SseOpcode.Psrld) src1 src2)) + (xmm_rmi_xmm (SseOpcode.Psrld) src1 src2)) ;; Helper for creating `psrlq` instructions. -(decl psrlq (Reg RegMemImm) Reg) +(decl psrlq (Xmm XmmMemImm) Xmm) (rule (psrlq src1 src2) - (xmm_rmi_reg (SseOpcode.Psrlq) src1 src2)) + (xmm_rmi_xmm (SseOpcode.Psrlq) src1 src2)) ;; Helper for creating `psraw` instructions. -(decl psraw (Reg RegMemImm) Reg) +(decl psraw (Xmm XmmMemImm) Xmm) (rule (psraw src1 src2) - (xmm_rmi_reg (SseOpcode.Psraw) src1 src2)) + (xmm_rmi_xmm (SseOpcode.Psraw) src1 src2)) ;; Helper for creating `psrad` instructions. -(decl psrad (Reg RegMemImm) Reg) +(decl psrad (Xmm XmmMemImm) Xmm) (rule (psrad src1 src2) - (xmm_rmi_reg (SseOpcode.Psrad) src1 src2)) - -;; Helper for creating `MInst.MulHi` instructions. -;; -;; Returns the (lo, hi) register halves of the multiplication. -(decl mul_hi (Type bool Reg RegMem) ValueRegs) -(rule (mul_hi ty signed src1 src2) - (let ((dst_lo WritableReg (temp_writable_reg ty)) - (dst_hi WritableReg (temp_writable_reg ty)) - (size OperandSize (operand_size_of_type_32_64 ty)) - (_ Unit (emit (MInst.MulHi size - signed - src1 - src2 - dst_lo - dst_hi)))) - (value_regs (writable_reg_to_reg dst_lo) - (writable_reg_to_reg dst_hi)))) + (xmm_rmi_xmm (SseOpcode.Psrad) src1 src2)) -;; Helper for creating `mul` instructions that return both the lower and -;; (unsigned) higher halves of the result. -(decl mulhi_u (Type Reg RegMem) ValueRegs) -(rule (mulhi_u ty src1 src2) - (mul_hi ty $false src1 src2)) - -;; Helper for creating `cmpps` instructions. -(decl cmpps (Reg RegMem FcmpImm) Reg) -(rule (cmpps src1 src2 imm) - (xmm_rm_r_imm (SseOpcode.Cmpps) - src1 - src2 - (encode_fcmp_imm imm) - (OperandSize.Size32))) +;; Helper for creating `pextrd` instructions. +(decl pextrd (Type Xmm u8) Gpr) +(rule (pextrd ty src lane) + (let ((w_dst WritableGpr (temp_writable_gpr)) + (r_dst Gpr (writable_gpr_to_gpr w_dst)) + (_ Unit (emit (MInst.XmmRmRImm (SseOpcode.Pextrd) + (gpr_to_reg r_dst) + (RegMem.Reg (xmm_to_reg src)) + (writable_gpr_to_reg w_dst) + lane + (operand_size_of_type_32_64 (lane_type ty)))))) + r_dst)) ;; Helper for creating `cmppd` instructions. ;; ;; Note that `Size32` is intentional despite this being used for 64-bit ;; operations, since this presumably induces the correct encoding of the ;; instruction. -(decl cmppd (Reg RegMem FcmpImm) Reg) +(decl cmppd (Xmm XmmMem FcmpImm) Xmm) (rule (cmppd src1 src2 imm) (xmm_rm_r_imm (SseOpcode.Cmppd) - src1 - src2 + (xmm_to_reg src1) + (xmm_mem_to_reg_mem src2) (encode_fcmp_imm imm) (OperandSize.Size32))) ;; Helper for creating `MInst.GprToXmm` instructions. -(decl gpr_to_xmm (Type SseOpcode RegMem OperandSize) Reg) -(rule (gpr_to_xmm ty op src size) - (let ((dst WritableReg (temp_writable_reg ty)) - (_ Unit (emit (MInst.GprToXmm op src dst size)))) - (writable_reg_to_reg dst))) - -;; Helper for creating `pinsrb` instructions. -(decl pinsrb (Reg RegMem u8) Reg) -(rule (pinsrb src1 src2 lane) - (xmm_rm_r_imm (SseOpcode.Pinsrb) src1 src2 lane (OperandSize.Size32))) - -;; Helper for creating `pinsrw` instructions. -(decl pinsrw (Reg RegMem u8) Reg) -(rule (pinsrw src1 src2 lane) - (xmm_rm_r_imm (SseOpcode.Pinsrw) src1 src2 lane (OperandSize.Size32))) - -;; Helper for creating `pinsrd` instructions. -(decl pinsrd (Reg RegMem u8 OperandSize) Reg) -(rule (pinsrd src1 src2 lane size) - (xmm_rm_r_imm (SseOpcode.Pinsrd) src1 src2 lane size)) - -;; Helper for creating `insertps` instructions. -(decl insertps (Reg RegMem u8) Reg) -(rule (insertps src1 src2 lane) - (xmm_rm_r_imm (SseOpcode.Insertps) src1 src2 lane (OperandSize.Size32))) - -;; Helper for creating `pextrd` instructions. -(decl pextrd (Type Reg u8) Reg) -(rule (pextrd ty src lane) - (let ((w_dst WritableReg (temp_writable_reg ty)) - (r_dst Reg (writable_reg_to_reg w_dst)) - (_ Unit (emit (MInst.XmmRmRImm (SseOpcode.Pextrd) - r_dst - (RegMem.Reg src) - w_dst - lane - (operand_size_of_type_32_64 (lane_type ty)))))) - r_dst)) +(decl gpr_to_xmm (SseOpcode GprMem OperandSize) Xmm) +(rule (gpr_to_xmm op src size) + (let ((dst WritableXmm (temp_writable_xmm)) + (_ Unit (emit (MInst.GprToXmm op (gpr_mem_to_reg_mem src) dst size)))) + (writable_xmm_to_xmm dst))) ;; Helper for creating `not` instructions. -(decl not (Type Reg) Reg) +(decl not (Type Gpr) Gpr) (rule (not ty src) - (let ((dst WritableReg (temp_writable_reg ty)) + (let ((dst WritableGpr (temp_writable_gpr)) (size OperandSize (operand_size_of_type_32_64 ty)) (_ Unit (emit (MInst.Not size src dst)))) - (writable_reg_to_reg dst))) + (writable_gpr_to_gpr dst))) ;; Helper for creating `neg` instructions. -(decl neg (Type Reg) Reg) +(decl neg (Type Gpr) Gpr) (rule (neg ty src) - (let ((dst WritableReg (temp_writable_reg ty)) + (let ((dst WritableGpr (temp_writable_gpr)) (size OperandSize (operand_size_of_type_32_64 ty)) (_ Unit (emit (MInst.Neg size src dst)))) - (writable_reg_to_reg dst))) + (writable_gpr_to_gpr dst))) -(decl lea (SyntheticAmode) Reg) +(decl lea (SyntheticAmode) Gpr) (rule (lea addr) - (let ((dst WritableReg (temp_writable_reg $I64)) + (let ((dst WritableGpr (temp_writable_gpr)) (_ Unit (emit (MInst.LoadEffectiveAddress addr dst)))) - (writable_reg_to_reg dst))) + (writable_gpr_to_gpr dst))) ;; Helper for creating `ud2` instructions. (decl ud2 (TrapCode) SideEffectNoResult) diff --git a/cranelift/codegen/src/isa/x64/inst/args.rs b/cranelift/codegen/src/isa/x64/inst/args.rs index 9b9cf2fd0ca9..e27dc3130ec6 100644 --- a/cranelift/codegen/src/isa/x64/inst/args.rs +++ b/cranelift/codegen/src/isa/x64/inst/args.rs @@ -13,6 +13,309 @@ use smallvec::{smallvec, SmallVec}; use std::fmt; use std::string::String; +/// An extenstion trait for converting `Writable{Xmm,Gpr}` to `Writable`. +pub trait ToWritableReg { + fn to_writable_reg(&self) -> Writable; +} + +/// An extension trait for converting `Writable` to `Writable{Xmm,Gpr}`. +pub trait FromWritableReg: Sized { + fn from_writable_reg(w: Writable) -> Option; +} + +/// An extension trait for mapping register uses on `{Xmm,Gpr}`. +pub trait MapUseExt { + fn map_use(&mut self, mapper: &RM) + where + RM: RegMapper; +} + +/// An extension trait for mapping register mods and defs on +/// `Writable{Xmm,Gpr}`. +pub trait MapDefModExt { + fn map_def(&mut self, mapper: &RM) + where + RM: RegMapper; + + fn map_mod(&mut self, mapper: &RM) + where + RM: RegMapper; +} + +/// A macro for defining a newtype of `Reg` that enforces some invariant about +/// the wrapped `Reg` (such as that it is of a particular register class). +macro_rules! newtype_of_reg { + ( + $newtype_reg:ident, + $newtype_writable_reg:ident, + $newtype_reg_mem:ident, + $newtype_reg_mem_imm:ident, + |$check_reg:ident| $check:expr + ) => { + /// A newtype wrapper around `Reg`. + #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] + pub struct $newtype_reg(Reg); + + impl PartialEq for $newtype_reg { + fn eq(&self, other: &Reg) -> bool { + self.0 == *other + } + } + + impl From<$newtype_reg> for Reg { + fn from(r: $newtype_reg) -> Self { + r.0 + } + } + + impl PrettyPrint for $newtype_reg { + fn show_rru(&self, mb_rru: Option<&RealRegUniverse>) -> String { + self.0.show_rru(mb_rru) + } + } + + impl $newtype_reg { + /// Create this newtype from the given register, or return `None` if the register + /// is not a valid instance of this newtype. + pub fn new($check_reg: Reg) -> Option { + if $check { + Some(Self($check_reg)) + } else { + None + } + } + + /// Get this newtype's underlying `Reg`. + pub fn to_reg(self) -> Reg { + self.0 + } + } + + // Convenience impl so that people working with this newtype can use it + // "just like" a plain `Reg`. + // + // NB: We cannot implement `DerefMut` because that would let people do + // nasty stuff like `*my_gpr.deref_mut() = some_xmm_reg`, breaking the + // invariants that `Gpr` provides. + impl std::ops::Deref for $newtype_reg { + type Target = Reg; + + fn deref(&self) -> &Reg { + &self.0 + } + } + + impl MapUseExt for $newtype_reg { + fn map_use(&mut self, mapper: &RM) + where + RM: RegMapper, + { + let mut reg = self.0; + mapper.map_use(&mut reg); + debug_assert!({ + let $check_reg = reg; + $check + }); + *self = $newtype_reg(reg); + } + } + + pub type $newtype_writable_reg = Writable<$newtype_reg>; + + impl ToWritableReg for $newtype_writable_reg { + fn to_writable_reg(&self) -> Writable { + Writable::from_reg(self.to_reg().to_reg()) + } + } + + impl FromWritableReg for $newtype_writable_reg { + fn from_writable_reg(w: Writable) -> Option { + Some(Writable::from_reg($newtype_reg::new(w.to_reg())?)) + } + } + + impl MapDefModExt for $newtype_writable_reg { + fn map_def(&mut self, mapper: &RM) + where + RM: RegMapper, + { + let mut reg = self.to_writable_reg(); + mapper.map_def(&mut reg); + debug_assert!({ + let $check_reg = reg.to_reg(); + $check + }); + *self = Writable::from_reg($newtype_reg(reg.to_reg())); + } + + fn map_mod(&mut self, mapper: &RM) + where + RM: RegMapper, + { + let mut reg = self.to_writable_reg(); + mapper.map_mod(&mut reg); + debug_assert!({ + let $check_reg = reg.to_reg(); + $check + }); + *self = Writable::from_reg($newtype_reg(reg.to_reg())); + } + } + + /// A newtype wrapper around `RegMem` for general-purpose registers. + #[derive(Clone, Debug)] + pub struct $newtype_reg_mem(RegMem); + + impl From<$newtype_reg_mem> for RegMem { + fn from(rm: $newtype_reg_mem) -> Self { + rm.0 + } + } + + impl From<$newtype_reg> for $newtype_reg_mem { + fn from(r: $newtype_reg) -> Self { + $newtype_reg_mem(RegMem::reg(r.into())) + } + } + + impl $newtype_reg_mem { + /// Construct a `RegMem` newtype from the given `RegMem`, or return + /// `None` if the `RegMem` is not a valid instance of this `RegMem` + /// newtype. + pub fn new(rm: RegMem) -> Option { + match rm { + RegMem::Mem { addr: _ } => Some(Self(rm)), + RegMem::Reg { reg: $check_reg } if $check => Some(Self(rm)), + RegMem::Reg { reg: _ } => None, + } + } + + /// Convert this newtype into its underlying `RegMem`. + pub fn to_reg_mem(self) -> RegMem { + self.0 + } + + #[allow(dead_code)] // Used by some newtypes and not others. + pub fn map_uses(&mut self, mapper: &RM) + where + RM: RegMapper, + { + self.0.map_uses(mapper); + debug_assert!(match self.0 { + RegMem::Reg { reg: $check_reg } => $check, + _ => true, + }); + } + + #[allow(dead_code)] // Used by some newtypes and not others. + pub fn map_as_def(&mut self, mapper: &RM) + where + RM: RegMapper, + { + self.0.map_as_def(mapper); + debug_assert!(match self.0 { + RegMem::Reg { reg: $check_reg } => $check, + _ => true, + }); + } + } + + impl PrettyPrint for $newtype_reg_mem { + fn show_rru(&self, mb_rru: Option<&RealRegUniverse>) -> String { + self.0.show_rru(mb_rru) + } + } + + impl PrettyPrintSized for $newtype_reg_mem { + fn show_rru_sized(&self, mb_rru: Option<&RealRegUniverse>, size: u8) -> String { + self.0.show_rru_sized(mb_rru, size) + } + } + + /// A newtype wrapper around `RegMemImm`. + #[derive(Clone, Debug)] + pub struct $newtype_reg_mem_imm(RegMemImm); + + impl From<$newtype_reg_mem_imm> for RegMemImm { + fn from(rmi: $newtype_reg_mem_imm) -> RegMemImm { + rmi.0 + } + } + + impl From<$newtype_reg> for $newtype_reg_mem_imm { + fn from(r: $newtype_reg) -> Self { + $newtype_reg_mem_imm(RegMemImm::reg(r.into())) + } + } + + impl $newtype_reg_mem_imm { + /// Construct this newtype from the given `RegMemImm`, or return + /// `None` if the `RegMemImm` is not a valid instance of this + /// newtype. + pub fn new(rmi: RegMemImm) -> Option { + match rmi { + RegMemImm::Imm { .. } => Some(Self(rmi)), + RegMemImm::Mem { addr: _ } => Some(Self(rmi)), + RegMemImm::Reg { reg: $check_reg } if $check => Some(Self(rmi)), + RegMemImm::Reg { reg: _ } => None, + } + } + + /// Convert this newtype into its underlying `RegMemImm`. + #[allow(dead_code)] // Used by some newtypes and not others. + pub fn to_reg_mem_imm(self) -> RegMemImm { + self.0 + } + + #[allow(dead_code)] // Used by some newtypes and not others. + pub fn map_uses(&mut self, mapper: &RM) + where + RM: RegMapper, + { + self.0.map_uses(mapper); + debug_assert!(match self.0 { + RegMemImm::Reg { reg: $check_reg } => $check, + _ => true, + }); + } + + #[allow(dead_code)] // Used by some newtypes and not others. + pub fn map_as_def(&mut self, mapper: &RM) + where + RM: RegMapper, + { + self.0.map_as_def(mapper); + debug_assert!(match self.0 { + RegMemImm::Reg { reg: $check_reg } => $check, + _ => true, + }); + } + } + + impl PrettyPrint for $newtype_reg_mem_imm { + fn show_rru(&self, mb_rru: Option<&RealRegUniverse>) -> String { + self.0.show_rru(mb_rru) + } + } + + impl PrettyPrintSized for $newtype_reg_mem_imm { + fn show_rru_sized(&self, mb_rru: Option<&RealRegUniverse>, size: u8) -> String { + self.0.show_rru_sized(mb_rru, size) + } + } + }; +} + +// Define a newtype of `Reg` for general-purpose registers. +newtype_of_reg!(Gpr, WritableGpr, GprMem, GprMemImm, |reg| { + reg.get_class() == RegClass::I64 +}); + +// Define a newtype of `Reg` for XMM registers. +newtype_of_reg!(Xmm, WritableXmm, XmmMem, XmmMemImm, |reg| { + reg.get_class() == RegClass::V128 +}); + /// A possible addressing mode (amode) that can be used in instructions. /// These denote a 64-bit value only. #[derive(Clone, Debug)] @@ -27,8 +330,8 @@ pub enum Amode { /// sign-extend-32-to-64(Immediate) + Register1 + (Register2 << Shift) ImmRegRegShift { simm32: u32, - base: Reg, - index: Reg, + base: Gpr, + index: Gpr, shift: u8, /* 0 .. 3 only */ flags: MemFlags, }, @@ -48,7 +351,7 @@ impl Amode { } } - pub(crate) fn imm_reg_reg_shift(simm32: u32, base: Reg, index: Reg, shift: u8) -> Self { + pub(crate) fn imm_reg_reg_shift(simm32: u32, base: Gpr, index: Gpr, shift: u8) -> Self { debug_assert!(base.get_class() == RegClass::I64); debug_assert!(index.get_class() == RegClass::I64); debug_assert!(shift <= 3); @@ -96,8 +399,8 @@ impl Amode { collector.add_use(*base); } Amode::ImmRegRegShift { base, index, .. } => { - collector.add_use(*base); - collector.add_use(*index); + collector.add_use(base.to_reg()); + collector.add_use(index.to_reg()); } Amode::RipRelative { .. } => { // RIP isn't involved in regalloc. @@ -225,7 +528,7 @@ impl PrettyPrint for SyntheticAmode { /// denote an 8, 16, 32 or 64 bit value. For the Immediate form, in the 8- and 16-bit case, only /// the lower 8 or 16 bits of `simm32` is relevant. In the 64-bit case, the value denoted by /// `simm32` is its sign-extension out to 64 bits. -#[derive(Clone)] +#[derive(Clone, Debug)] pub enum RegMemImm { Reg { reg: Reg }, Mem { addr: SyntheticAmode }, diff --git a/cranelift/codegen/src/isa/x64/inst/emit.rs b/cranelift/codegen/src/isa/x64/inst/emit.rs index c4dc3f3d7a5d..bda2ddc0a241 100644 --- a/cranelift/codegen/src/isa/x64/inst/emit.rs +++ b/cranelift/codegen/src/isa/x64/inst/emit.rs @@ -298,14 +298,14 @@ pub(crate) fn emit( Popcnt => (0x0fb8, 2), }; - match src { + match src.clone().into() { RegMem::Reg { reg: src } => emit_std_reg_reg( sink, prefix, opcode, num_opcodes, - dst.to_reg(), - *src, + dst.to_reg().to_reg(), + src, rex_flags, ), RegMem::Mem { addr: src } => { @@ -317,7 +317,7 @@ pub(crate) fn emit( prefix, opcode, num_opcodes, - dst.to_reg(), + dst.to_reg().to_reg(), &amode, rex_flags, ); @@ -327,7 +327,7 @@ pub(crate) fn emit( Inst::Not { size, src, dst } => { debug_assert_eq!(*src, dst.to_reg()); - let rex_flags = RexFlags::from((*size, dst.to_reg())); + let rex_flags = RexFlags::from((*size, dst.to_writable_reg().to_reg())); let (opcode, prefix) = match size { OperandSize::Size8 => (0xF6, LegacyPrefixes::None), OperandSize::Size16 => (0xF7, LegacyPrefixes::_66), @@ -342,7 +342,7 @@ pub(crate) fn emit( Inst::Neg { size, src, dst } => { debug_assert_eq!(*src, dst.to_reg()); - let rex_flags = RexFlags::from((*size, dst.to_reg())); + let rex_flags = RexFlags::from((*size, dst.to_writable_reg().to_reg())); let (opcode, prefix) = match size { OperandSize::Size8 => (0xF6, LegacyPrefixes::None), OperandSize::Size16 => (0xF7, LegacyPrefixes::_66), @@ -728,7 +728,7 @@ pub(crate) fn emit( LegacyPrefixes::None, 0x8D, 1, - dst.to_reg(), + dst.to_reg().to_reg(), &amode, RexFlags::set_w(), ); @@ -884,6 +884,7 @@ pub(crate) fn emit( debug_assert_eq!(*src1, dst.to_reg()); let rex = RexFlags::clear_w(); let prefix = LegacyPrefixes::_66; + let src2 = src2.clone().to_reg_mem_imm(); if let RegMemImm::Imm { simm32 } = src2 { let (opcode_bytes, reg_digit) = match opcode { SseOpcode::Psllw => (0x0F71, 6), @@ -898,7 +899,7 @@ pub(crate) fn emit( }; let dst_enc = reg_enc(dst.to_reg()); emit_std_enc_enc(sink, prefix, opcode_bytes, 2, reg_digit, dst_enc, rex); - let imm = (*simm32) + let imm = (simm32) .try_into() .expect("the immediate must be convertible to a u8"); sink.put1(imm); @@ -917,7 +918,15 @@ pub(crate) fn emit( match src2 { RegMemImm::Reg { reg } => { - emit_std_reg_reg(sink, prefix, opcode_bytes, 2, dst.to_reg(), *reg, rex); + emit_std_reg_reg( + sink, + prefix, + opcode_bytes, + 2, + dst.to_reg().to_reg(), + reg, + rex, + ); } RegMemImm::Mem { addr } => { let addr = &addr.finalize(state, sink); @@ -928,7 +937,7 @@ pub(crate) fn emit( prefix, opcode_bytes, 2, - dst.to_reg(), + dst.to_reg().to_reg(), addr, rex, ); @@ -1335,7 +1344,12 @@ pub(crate) fn emit( // might be negative; use a sign-extension. let inst = Inst::movsx_rm_r( ExtMode::LQ, - RegMem::mem(Amode::imm_reg_reg_shift(0, tmp1.to_reg(), tmp2.to_reg(), 2)), + RegMem::mem(Amode::imm_reg_reg_shift( + 0, + Gpr::new(tmp1.to_reg()).unwrap(), + Gpr::new(tmp2.to_reg()).unwrap(), + 2, + )), *tmp2, ); inst.emit(sink, info, state); @@ -1424,15 +1438,15 @@ pub(crate) fn emit( _ => unimplemented!("Opcode {:?} not implemented", op), }; - match src_e { + match src_e.clone().to_reg_mem() { RegMem::Reg { reg: reg_e } => { emit_std_reg_reg( sink, prefix, opcode, num_opcodes, - reg_g.to_reg(), - *reg_e, + reg_g.to_reg().to_reg(), + reg_e, rex, ); } @@ -1445,7 +1459,7 @@ pub(crate) fn emit( prefix, opcode, num_opcodes, - reg_g.to_reg(), + reg_g.to_reg().to_reg(), addr, rex, ); @@ -1460,7 +1474,7 @@ pub(crate) fn emit( Avx512Opcode::Vpopcntb => (LegacyPrefixes::_66, OpcodeMap::_0F38, false, 0x54), _ => unimplemented!("Opcode {:?} not implemented", op), }; - match src { + match src.clone().to_reg_mem() { RegMem::Reg { reg: src } => EvexInstruction::new() .length(EvexVectorLength::V128) .prefix(prefix) @@ -1587,9 +1601,17 @@ pub(crate) fn emit( _ => unimplemented!("Opcode {:?} not implemented", op), }; - match src_e { + match src_e.clone().to_reg_mem() { RegMem::Reg { reg: reg_e } => { - emit_std_reg_reg(sink, prefix, opcode, length, reg_g.to_reg(), *reg_e, rex); + emit_std_reg_reg( + sink, + prefix, + opcode, + length, + reg_g.to_reg().to_reg(), + reg_e, + rex, + ); } RegMem::Mem { addr } => { let addr = &addr.finalize(state, sink); @@ -1600,7 +1622,7 @@ pub(crate) fn emit( prefix, opcode, length, - reg_g.to_reg(), + reg_g.to_reg().to_reg(), addr, rex, ); @@ -1619,7 +1641,7 @@ pub(crate) fn emit( Avx512Opcode::Vpmullq => (true, 0x40), _ => unimplemented!("Opcode {:?} not implemented", op), }; - match src1 { + match src1.clone().to_reg_mem() { RegMem::Reg { reg: src } => EvexInstruction::new() .length(EvexVectorLength::V128) .prefix(LegacyPrefixes::_66) @@ -1845,9 +1867,9 @@ pub(crate) fn emit( }; let rex = RexFlags::from(*dst_size); let (src, dst) = if dst_first { - (dst.to_reg(), *src) + (dst.to_reg().to_reg(), src.to_reg()) } else { - (*src, dst.to_reg()) + (src.to_reg(), dst.to_reg().to_reg()) }; emit_std_reg_reg(sink, prefix, opcode, 2, src, dst, rex); @@ -1870,7 +1892,15 @@ pub(crate) fn emit( let rex = RexFlags::from(*src_size); match src_e { RegMem::Reg { reg: reg_e } => { - emit_std_reg_reg(sink, prefix, opcode, 2, reg_g.to_reg(), *reg_e, rex); + emit_std_reg_reg( + sink, + prefix, + opcode, + 2, + reg_g.to_reg().to_reg(), + *reg_e, + rex, + ); } RegMem::Mem { addr } => { let addr = &addr.finalize(state, sink); @@ -1881,7 +1911,7 @@ pub(crate) fn emit( prefix, opcode, 2, - reg_g.to_reg(), + reg_g.to_reg().to_reg(), addr, rex, ); @@ -1950,7 +1980,11 @@ pub(crate) fn emit( // If x seen as a signed int64 is not negative, a signed-conversion will do the right // thing. // TODO use tst src, src here. - let inst = Inst::cmp_rmi_r(OperandSize::Size64, RegMemImm::imm(0), src.to_reg()); + let inst = Inst::cmp_rmi_r( + OperandSize::Size64, + RegMemImm::imm(0), + src.to_reg().to_reg(), + ); inst.emit(sink, info, state); one_way_jmp(sink, CC::L, handle_negative); @@ -1961,8 +1995,8 @@ pub(crate) fn emit( sink, info, state, - src.to_reg(), - *dst, + src.to_reg().to_reg(), + dst.to_writable_reg(), *dst_size == OperandSize::Size64, ); @@ -1973,7 +2007,11 @@ pub(crate) fn emit( // Divide x by two to get it in range for the signed conversion, keep the LSB, and // scale it back up on the FP side. - let inst = Inst::gen_move(*tmp_gpr1, src.to_reg(), types::I64); + let inst = Inst::gen_move( + tmp_gpr1.to_writable_reg(), + src.to_reg().to_reg(), + types::I64, + ); inst.emit(sink, info, state); // tmp_gpr1 := src >> 1 @@ -1981,26 +2019,30 @@ pub(crate) fn emit( OperandSize::Size64, ShiftKind::ShiftRightLogical, Some(1), - *tmp_gpr1, + tmp_gpr1.to_writable_reg(), ); inst.emit(sink, info, state); - let inst = Inst::gen_move(*tmp_gpr2, src.to_reg(), types::I64); + let inst = Inst::gen_move( + tmp_gpr2.to_writable_reg(), + src.to_reg().to_reg(), + types::I64, + ); inst.emit(sink, info, state); let inst = Inst::alu_rmi_r( OperandSize::Size64, AluRmiROpcode::And, RegMemImm::imm(1), - *tmp_gpr2, + tmp_gpr2.to_writable_reg(), ); inst.emit(sink, info, state); let inst = Inst::alu_rmi_r( OperandSize::Size64, AluRmiROpcode::Or, - RegMemImm::reg(tmp_gpr1.to_reg()), - *tmp_gpr2, + RegMemImm::reg(tmp_gpr1.to_reg().to_reg()), + tmp_gpr2.to_writable_reg(), ); inst.emit(sink, info, state); @@ -2008,8 +2050,8 @@ pub(crate) fn emit( sink, info, state, - tmp_gpr2.to_reg(), - *dst, + tmp_gpr2.to_reg().to_reg(), + dst.to_writable_reg(), *dst_size == OperandSize::Size64, ); @@ -2018,7 +2060,11 @@ pub(crate) fn emit( } else { SseOpcode::Addss }; - let inst = Inst::xmm_rm_r(add_op, RegMem::reg(dst.to_reg()), *dst); + let inst = Inst::xmm_rm_r( + add_op, + RegMem::reg(dst.to_reg().to_reg()), + dst.to_writable_reg(), + ); inst.emit(sink, info, state); sink.bind_label(done); @@ -2091,18 +2137,18 @@ pub(crate) fn emit( let not_nan = sink.get_label(); // The truncation. - let inst = Inst::xmm_to_gpr(trunc_op, src, *dst, *dst_size); + let inst = Inst::xmm_to_gpr(trunc_op, src.to_reg(), dst.to_writable_reg(), *dst_size); inst.emit(sink, info, state); // Compare against 1, in case of overflow the dst operand was INT_MIN. - let inst = Inst::cmp_rmi_r(*dst_size, RegMemImm::imm(1), dst.to_reg()); + let inst = Inst::cmp_rmi_r(*dst_size, RegMemImm::imm(1), dst.to_reg().to_reg()); inst.emit(sink, info, state); one_way_jmp(sink, CC::NO, done); // no overflow => done // Check for NaN. - let inst = Inst::xmm_cmp_rm_r(cmp_op, RegMem::reg(src), src); + let inst = Inst::xmm_cmp_rm_r(cmp_op, RegMem::reg(src.to_reg()), src.to_reg()); inst.emit(sink, info, state); one_way_jmp(sink, CC::NP, not_nan); // go to not_nan if not a NaN @@ -2112,8 +2158,8 @@ pub(crate) fn emit( let inst = Inst::alu_rmi_r( *dst_size, AluRmiROpcode::Xor, - RegMemImm::reg(dst.to_reg()), - *dst, + RegMemImm::reg(dst.to_reg().to_reg()), + dst.to_writable_reg(), ); inst.emit(sink, info, state); @@ -2125,11 +2171,18 @@ pub(crate) fn emit( // If the input was positive, saturate to INT_MAX. // Zero out tmp_xmm. - let inst = - Inst::xmm_rm_r(SseOpcode::Xorpd, RegMem::reg(tmp_xmm.to_reg()), *tmp_xmm); + let inst = Inst::xmm_rm_r( + SseOpcode::Xorpd, + RegMem::reg(tmp_xmm.to_reg().to_reg()), + tmp_xmm.to_writable_reg(), + ); inst.emit(sink, info, state); - let inst = Inst::xmm_cmp_rm_r(cmp_op, RegMem::reg(src), tmp_xmm.to_reg()); + let inst = Inst::xmm_cmp_rm_r( + cmp_op, + RegMem::reg(src.to_reg()), + tmp_xmm.to_reg().to_reg(), + ); inst.emit(sink, info, state); // Jump if >= to done. @@ -2137,10 +2190,14 @@ pub(crate) fn emit( // Otherwise, put INT_MAX. if *dst_size == OperandSize::Size64 { - let inst = Inst::imm(OperandSize::Size64, 0x7fffffffffffffff, *dst); + let inst = Inst::imm( + OperandSize::Size64, + 0x7fffffffffffffff, + dst.to_writable_reg(), + ); inst.emit(sink, info, state); } else { - let inst = Inst::imm(OperandSize::Size32, 0x7fffffff, *dst); + let inst = Inst::imm(OperandSize::Size32, 0x7fffffff, dst.to_writable_reg()); inst.emit(sink, info, state); } } else { @@ -2162,7 +2219,8 @@ pub(crate) fn emit( match *src_size { OperandSize::Size32 => { let cst = Ieee32::pow2(output_bits - 1).neg().bits(); - let inst = Inst::imm(OperandSize::Size32, cst as u64, *tmp_gpr); + let inst = + Inst::imm(OperandSize::Size32, cst as u64, tmp_gpr.to_writable_reg()); inst.emit(sink, info, state); } OperandSize::Size64 => { @@ -2174,17 +2232,26 @@ pub(crate) fn emit( } else { Ieee64::pow2(output_bits - 1).neg() }; - let inst = Inst::imm(OperandSize::Size64, cst.bits(), *tmp_gpr); + let inst = + Inst::imm(OperandSize::Size64, cst.bits(), tmp_gpr.to_writable_reg()); inst.emit(sink, info, state); } _ => unreachable!(), } - let inst = - Inst::gpr_to_xmm(cast_op, RegMem::reg(tmp_gpr.to_reg()), *src_size, *tmp_xmm); + let inst = Inst::gpr_to_xmm( + cast_op, + RegMem::reg(tmp_gpr.to_reg().to_reg()), + *src_size, + tmp_xmm.to_writable_reg(), + ); inst.emit(sink, info, state); - let inst = Inst::xmm_cmp_rm_r(cmp_op, RegMem::reg(tmp_xmm.to_reg()), src); + let inst = Inst::xmm_cmp_rm_r( + cmp_op, + RegMem::reg(tmp_xmm.to_reg().to_reg()), + src.to_reg(), + ); inst.emit(sink, info, state); // jump over trap if src >= or > threshold @@ -2198,11 +2265,18 @@ pub(crate) fn emit( sink.bind_label(check_positive); // Zero out the tmp_xmm register. - let inst = - Inst::xmm_rm_r(SseOpcode::Xorpd, RegMem::reg(tmp_xmm.to_reg()), *tmp_xmm); + let inst = Inst::xmm_rm_r( + SseOpcode::Xorpd, + RegMem::reg(tmp_xmm.to_reg().to_reg()), + tmp_xmm.to_writable_reg(), + ); inst.emit(sink, info, state); - let inst = Inst::xmm_cmp_rm_r(cmp_op, RegMem::reg(src), tmp_xmm.to_reg()); + let inst = Inst::xmm_cmp_rm_r( + cmp_op, + RegMem::reg(src.to_reg()), + tmp_xmm.to_reg().to_reg(), + ); inst.emit(sink, info, state); one_way_jmp(sink, CC::NB, done); // jump over trap if 0 >= src @@ -2282,14 +2356,22 @@ pub(crate) fn emit( _ => unreachable!(), }; - let inst = Inst::imm(*src_size, cst, *tmp_gpr); + let inst = Inst::imm(*src_size, cst, tmp_gpr.to_writable_reg()); inst.emit(sink, info, state); - let inst = - Inst::gpr_to_xmm(cast_op, RegMem::reg(tmp_gpr.to_reg()), *src_size, *tmp_xmm); + let inst = Inst::gpr_to_xmm( + cast_op, + RegMem::reg(tmp_gpr.to_reg().to_reg()), + *src_size, + tmp_xmm.to_writable_reg(), + ); inst.emit(sink, info, state); - let inst = Inst::xmm_cmp_rm_r(cmp_op, RegMem::reg(tmp_xmm.to_reg()), src.to_reg()); + let inst = Inst::xmm_cmp_rm_r( + cmp_op, + RegMem::reg(tmp_xmm.to_reg().to_reg()), + src.to_reg().to_reg(), + ); inst.emit(sink, info, state); let handle_large = sink.get_label(); @@ -2303,8 +2385,8 @@ pub(crate) fn emit( let inst = Inst::alu_rmi_r( *dst_size, AluRmiROpcode::Xor, - RegMemImm::reg(dst.to_reg()), - *dst, + RegMemImm::reg(dst.to_reg().to_reg()), + dst.to_writable_reg(), ); inst.emit(sink, info, state); @@ -2321,10 +2403,15 @@ pub(crate) fn emit( // Actual truncation for small inputs: if the result is not positive, then we had an // overflow. - let inst = Inst::xmm_to_gpr(trunc_op, src.to_reg(), *dst, *dst_size); + let inst = Inst::xmm_to_gpr( + trunc_op, + src.to_reg().to_reg(), + dst.to_writable_reg(), + *dst_size, + ); inst.emit(sink, info, state); - let inst = Inst::cmp_rmi_r(*dst_size, RegMemImm::imm(0), dst.to_reg()); + let inst = Inst::cmp_rmi_r(*dst_size, RegMemImm::imm(0), dst.to_reg().to_reg()); inst.emit(sink, info, state); one_way_jmp(sink, CC::NL, done); // if dst >= 0, jump to done @@ -2335,8 +2422,8 @@ pub(crate) fn emit( let inst = Inst::alu_rmi_r( *dst_size, AluRmiROpcode::Xor, - RegMemImm::reg(dst.to_reg()), - *dst, + RegMemImm::reg(dst.to_reg().to_reg()), + dst.to_writable_reg(), ); inst.emit(sink, info, state); @@ -2352,13 +2439,22 @@ pub(crate) fn emit( sink.bind_label(handle_large); - let inst = Inst::xmm_rm_r(sub_op, RegMem::reg(tmp_xmm.to_reg()), *src); + let inst = Inst::xmm_rm_r( + sub_op, + RegMem::reg(tmp_xmm.to_reg().to_reg()), + src.to_writable_reg(), + ); inst.emit(sink, info, state); - let inst = Inst::xmm_to_gpr(trunc_op, src.to_reg(), *dst, *dst_size); + let inst = Inst::xmm_to_gpr( + trunc_op, + src.to_reg().to_reg(), + dst.to_writable_reg(), + *dst_size, + ); inst.emit(sink, info, state); - let inst = Inst::cmp_rmi_r(*dst_size, RegMemImm::imm(0), dst.to_reg()); + let inst = Inst::cmp_rmi_r(*dst_size, RegMemImm::imm(0), dst.to_reg().to_reg()); inst.emit(sink, info, state); let next_is_large = sink.get_label(); @@ -2374,7 +2470,7 @@ pub(crate) fn emit( } else { u32::max_value() as u64 }, - *dst, + dst.to_writable_reg(), ); inst.emit(sink, info, state); @@ -2388,14 +2484,14 @@ pub(crate) fn emit( sink.bind_label(next_is_large); if *dst_size == OperandSize::Size64 { - let inst = Inst::imm(OperandSize::Size64, 1 << 63, *tmp_gpr); + let inst = Inst::imm(OperandSize::Size64, 1 << 63, tmp_gpr.to_writable_reg()); inst.emit(sink, info, state); let inst = Inst::alu_rmi_r( OperandSize::Size64, AluRmiROpcode::Add, - RegMemImm::reg(tmp_gpr.to_reg()), - *dst, + RegMemImm::reg(tmp_gpr.to_reg().to_reg()), + dst.to_writable_reg(), ); inst.emit(sink, info, state); } else { @@ -2403,7 +2499,7 @@ pub(crate) fn emit( OperandSize::Size32, AluRmiROpcode::Add, RegMemImm::imm(1 << 31), - *dst, + dst.to_writable_reg(), ); inst.emit(sink, info, state); } diff --git a/cranelift/codegen/src/isa/x64/inst/mod.rs b/cranelift/codegen/src/isa/x64/inst/mod.rs index 6f00370ea09d..bd7c039b9436 100644 --- a/cranelift/codegen/src/isa/x64/inst/mod.rs +++ b/cranelift/codegen/src/isa/x64/inst/mod.rs @@ -2,13 +2,11 @@ use crate::binemit::{Addend, CodeOffset, Reloc, StackMap}; use crate::ir::{types, ExternalName, Opcode, SourceLoc, TrapCode, Type, ValueLabel}; -use crate::isa::unwind::UnwindInst; use crate::isa::x64::abi::X64ABIMachineSpec; use crate::isa::x64::settings as x64_settings; use crate::isa::CallConv; use crate::machinst::*; use crate::{settings, CodegenError, CodegenResult}; -use alloc::boxed::Box; use alloc::vec::Vec; use regalloc::{ PrettyPrint, PrettyPrintSized, RealRegUniverse, Reg, RegClass, RegUsageCollector, SpillSlot, @@ -31,510 +29,8 @@ use regs::show_ireg_sized; //============================================================================= // Instructions (top level): definition -// Don't build these directly. Instead use the Inst:: functions to create them. - -/// Instructions. -#[derive(Clone)] -pub enum Inst { - /// Nops of various sizes, including zero. - Nop { len: u8 }, - - // ===================================== - // Integer instructions. - /// Integer arithmetic/bit-twiddling: (add sub and or xor mul adc? sbb?) (32 64) (reg addr imm) reg - AluRmiR { - size: OperandSize, // 4 or 8 - op: AluRmiROpcode, - src1: Reg, - src2: RegMemImm, - dst: Writable, - }, - - /// Instructions on GPR that only read src and defines dst (dst is not modified): bsr, etc. - UnaryRmR { - size: OperandSize, // 2, 4 or 8 - op: UnaryRmROpcode, - src: RegMem, - dst: Writable, - }, - - /// Bitwise not - Not { - size: OperandSize, // 1, 2, 4 or 8 - src: Reg, - dst: Writable, - }, - - /// Integer negation - Neg { - size: OperandSize, // 1, 2, 4 or 8 - src: Reg, - dst: Writable, - }, - - /// Integer quotient and remainder: (div idiv) $rax $rdx (reg addr) - Div { - size: OperandSize, // 1, 2, 4 or 8 - signed: bool, - divisor: RegMem, - dividend: Reg, - dst_quotient: Writable, - dst_remainder: Writable, - }, - - /// The high bits (RDX) of a (un)signed multiply: RDX:RAX := RAX * rhs. - MulHi { - size: OperandSize, // 2, 4, or 8 - signed: bool, - src1: Reg, - src2: RegMem, - dst_lo: Writable, - dst_hi: Writable, - }, - - /// A synthetic sequence to implement the right inline checks for remainder and division, - /// assuming the dividend is in %rax. - /// - /// Puts the result back into %rax if is_div, %rdx if !is_div, to mimic what the div - /// instruction does. - /// - /// The generated code sequence is described in the emit's function match arm for this - /// instruction. - /// - /// Note: %rdx is marked as modified by this instruction, to avoid an early clobber problem - /// with the temporary and divisor registers. Make sure to zero %rdx right before this - /// instruction, or you might run into regalloc failures where %rdx is live before its first - /// def! - CheckedDivOrRemSeq { - kind: DivOrRemKind, - size: OperandSize, - dividend: Reg, - /// The divisor operand. Note it's marked as modified so that it gets assigned a register - /// different from the temporary. - divisor: Writable, - dst_quotient: Writable, - dst_remainder: Writable, - tmp: Option>, - }, - - /// Do a sign-extend based on the sign of the value in rax into rdx: (cwd cdq cqo) - /// or al into ah: (cbw) - SignExtendData { - size: OperandSize, // 1, 2, 4 or 8 - src: Reg, - dst: Writable, - }, - - /// Constant materialization: (imm32 imm64) reg. - /// - /// Either: movl $imm32, %reg32 or movabsq $imm64, %reg32. - Imm { - dst_size: OperandSize, // 4 or 8 - simm64: u64, - dst: Writable, - }, - - /// GPR to GPR move: mov (64 32) reg reg. - MovRR { - size: OperandSize, // 4 or 8 - src: Reg, - dst: Writable, - }, - - /// Zero-extended loads, except for 64 bits: movz (bl bq wl wq lq) addr reg. - /// Note that the lq variant doesn't really exist since the default zero-extend rule makes it - /// unnecessary. For that case we emit the equivalent "movl AM, reg32". - MovzxRmR { - ext_mode: ExtMode, - src: RegMem, - dst: Writable, - }, - - /// A plain 64-bit integer load, since MovZX_RM_R can't represent that. - Mov64MR { - src: SyntheticAmode, - dst: Writable, - }, - - /// Loads the memory address of addr into dst. - LoadEffectiveAddress { - addr: SyntheticAmode, - dst: Writable, - }, - - /// Sign-extended loads and moves: movs (bl bq wl wq lq) addr reg. - MovsxRmR { - ext_mode: ExtMode, - src: RegMem, - dst: Writable, - }, - - /// Integer stores: mov (b w l q) reg addr. - MovRM { - size: OperandSize, // 1, 2, 4 or 8. - src: Reg, - dst: SyntheticAmode, - }, - - /// Arithmetic shifts: (shl shr sar) (b w l q) imm reg. - ShiftR { - size: OperandSize, // 1, 2, 4 or 8 - kind: ShiftKind, - src: Reg, - /// shift count: Some(0 .. #bits-in-type - 1), or None to mean "%cl". - num_bits: Imm8Reg, - dst: Writable, - }, - - /// Arithmetic SIMD shifts. - XmmRmiReg { - opcode: SseOpcode, - src1: Reg, - src2: RegMemImm, - dst: Writable, - }, - - /// Integer comparisons/tests: cmp or test (b w l q) (reg addr imm) reg. - CmpRmiR { - size: OperandSize, // 1, 2, 4 or 8 - opcode: CmpOpcode, - src: RegMemImm, - dst: Reg, - }, - - /// Materializes the requested condition code in the destination reg. - Setcc { cc: CC, dst: Writable }, - - /// Integer conditional move. - /// Overwrites the destination register. - Cmove { - size: OperandSize, // 2, 4, or 8 - cc: CC, - consequent: RegMem, - alternative: Reg, - dst: Writable, - }, - - // ===================================== - // Stack manipulation. - /// pushq (reg addr imm) - Push64 { src: RegMemImm }, - - /// popq reg - Pop64 { dst: Writable }, - - // ===================================== - // Floating-point operations. - /// XMM (scalar or vector) binary op: (add sub and or xor mul adc? sbb?) (32 64) (reg addr) reg - XmmRmR { - op: SseOpcode, - src1: Reg, - src2: RegMem, - dst: Writable, - }, - - XmmRmREvex { - op: Avx512Opcode, - src1: RegMem, - src2: Reg, - dst: Writable, - }, - - /// XMM (scalar or vector) unary op: mov between XMM registers (32 64) (reg addr) reg, sqrt, - /// etc. - /// - /// This differs from XMM_RM_R in that the dst register of XmmUnaryRmR is not used in the - /// computation of the instruction dst value and so does not have to be a previously valid - /// value. This is characteristic of mov instructions. - XmmUnaryRmR { - op: SseOpcode, - src: RegMem, - dst: Writable, - }, - - XmmUnaryRmREvex { - op: Avx512Opcode, - src: RegMem, - dst: Writable, - }, - - /// XMM (scalar or vector) unary op (from xmm to reg/mem): stores, movd, movq - XmmMovRM { - op: SseOpcode, - src: Reg, - dst: SyntheticAmode, - }, - - /// XMM (vector) unary op (to move a constant value into an xmm register): movups - XmmLoadConst { - src: VCodeConstant, - dst: Writable, - ty: Type, - }, - - /// XMM (scalar) unary op (from xmm to integer reg): movd, movq, cvtts{s,d}2si - XmmToGpr { - op: SseOpcode, - src: Reg, - dst: Writable, - dst_size: OperandSize, - }, - - /// XMM (scalar) unary op (from integer to float reg): movd, movq, cvtsi2s{s,d} - GprToXmm { - op: SseOpcode, - src: RegMem, - dst: Writable, - src_size: OperandSize, - }, - - /// Converts an unsigned int64 to a float32/float64. - CvtUint64ToFloatSeq { - dst_size: OperandSize, // 4 or 8 - /// A copy of the source register, fed by lowering. It is marked as modified during - /// register allocation to make sure that the temporary registers differ from the src - /// register, since both registers are live at the same time in the generated code - /// sequence. - src: Writable, - dst: Writable, - tmp_gpr1: Writable, - tmp_gpr2: Writable, - }, - - /// Converts a scalar xmm to a signed int32/int64. - CvtFloatToSintSeq { - dst_size: OperandSize, - src_size: OperandSize, - is_saturating: bool, - /// A copy of the source register, fed by lowering. It is marked as modified during - /// register allocation to make sure that the temporary xmm register differs from the src - /// register, since both registers are live at the same time in the generated code - /// sequence. - src: Writable, - dst: Writable, - tmp_gpr: Writable, - tmp_xmm: Writable, - }, - - /// Converts a scalar xmm to an unsigned int32/int64. - CvtFloatToUintSeq { - src_size: OperandSize, - dst_size: OperandSize, - is_saturating: bool, - /// A copy of the source register, fed by lowering, reused as a temporary. It is marked as - /// modified during register allocation to make sure that the temporary xmm register - /// differs from the src register, since both registers are live at the same time in the - /// generated code sequence. - src: Writable, - dst: Writable, - tmp_gpr: Writable, - tmp_xmm: Writable, - }, - - /// A sequence to compute min/max with the proper NaN semantics for xmm registers. - XmmMinMaxSeq { - size: OperandSize, - is_min: bool, - lhs: Reg, - rhs_dst: Writable, - }, - - /// XMM (scalar) conditional move. - /// Overwrites the destination register if cc is set. - XmmCmove { - size: OperandSize, // 4 or 8 - cc: CC, - src: RegMem, - dst: Writable, - }, - - /// Float comparisons/tests: cmp (b w l q) (reg addr imm) reg. - XmmCmpRmR { - op: SseOpcode, - src: RegMem, - dst: Reg, - }, - - /// A binary XMM instruction with an 8-bit immediate: e.g. cmp (ps pd) imm (reg addr) reg - XmmRmRImm { - op: SseOpcode, - src1: Reg, - src2: RegMem, - dst: Writable, - imm: u8, - size: OperandSize, // 4 or 8 - }, - - // ===================================== - // Control flow instructions. - /// Direct call: call simm32. - CallKnown { - dest: ExternalName, - uses: Vec, - defs: Vec>, - opcode: Opcode, - }, - - /// Indirect call: callq (reg mem). - CallUnknown { - dest: RegMem, - uses: Vec, - defs: Vec>, - opcode: Opcode, - }, - - /// Return. - Ret, - - /// A placeholder instruction, generating no code, meaning that a function epilogue must be - /// inserted there. - EpiloguePlaceholder, - - /// Jump to a known target: jmp simm32. - JmpKnown { dst: MachLabel }, - - /// One-way conditional branch: jcond cond target. - /// - /// This instruction is useful when we have conditional jumps depending on more than two - /// conditions, see for instance the lowering of Brz/brnz with Fcmp inputs. - /// - /// A note of caution: in contexts where the branch target is another block, this has to be the - /// same successor as the one specified in the terminator branch of the current block. - /// Otherwise, this might confuse register allocation by creating new invisible edges. - JmpIf { cc: CC, taken: MachLabel }, - - /// Two-way conditional branch: jcond cond target target. - /// Emitted as a compound sequence; the MachBuffer will shrink it as appropriate. - JmpCond { - cc: CC, - taken: MachLabel, - not_taken: MachLabel, - }, - - /// Jump-table sequence, as one compound instruction (see note in lower.rs for rationale). - /// The generated code sequence is described in the emit's function match arm for this - /// instruction. - /// See comment in lowering about the temporaries signedness. - JmpTableSeq { - idx: Reg, - tmp1: Writable, - tmp2: Writable, - default_target: MachLabel, - targets: Vec, - targets_for_term: Vec, - }, - - /// Indirect jump: jmpq (reg mem). - JmpUnknown { target: RegMem }, - - /// Traps if the condition code is set. - TrapIf { cc: CC, trap_code: TrapCode }, - - /// A debug trap. - Hlt, - - /// An instruction that will always trigger the illegal instruction exception. - Ud2 { trap_code: TrapCode }, - - /// Loads an external symbol in a register, with a relocation: - /// - /// movq $name@GOTPCREL(%rip), dst if PIC is enabled, or - /// movabsq $name, dst otherwise. - LoadExtName { - dst: Writable, - name: Box, - offset: i64, - }, - - // ===================================== - // Instructions pertaining to atomic memory accesses. - /// A standard (native) `lock cmpxchg src, (amode)`, with register conventions: - /// - /// `mem` (read) address - /// `replacement` (read) replacement value - /// %rax (modified) in: expected value, out: value that was actually at `dst` - /// %rflags is written. Do not assume anything about it after the instruction. - /// - /// The instruction "succeeded" iff the lowest `ty` bits of %rax afterwards are the same as - /// they were before. - LockCmpxchg { - ty: Type, // I8, I16, I32 or I64 - replacement: Reg, - expected: Reg, - mem: SyntheticAmode, - dst_old: Writable, - }, - - /// A synthetic instruction, based on a loop around a native `lock cmpxchg` instruction. - /// This atomically modifies a value in memory and returns the old value. The sequence - /// consists of an initial "normal" load from `dst`, followed by a loop which computes the - /// new value and tries to compare-and-swap ("CAS") it into `dst`, using the native - /// instruction `lock cmpxchg{b,w,l,q}` . The loop iterates until the CAS is successful. - /// If there is no contention, there will be only one pass through the loop body. The - /// sequence does *not* perform any explicit memory fence instructions - /// (mfence/sfence/lfence). - /// - /// Note that the transaction is atomic in the sense that, as observed by some other thread, - /// `dst` either has the initial or final value, but no other. It isn't atomic in the sense - /// of guaranteeing that no other thread writes to `dst` in between the initial load and the - /// CAS -- but that would cause the CAS to fail unless the other thread's last write before - /// the CAS wrote the same value that was already there. In other words, this - /// implementation suffers (unavoidably) from the A-B-A problem. - /// - /// This instruction sequence has fixed register uses as follows: - /// - /// %r9 (read) address - /// %r10 (read) second operand for `op` - /// %r11 (written) scratch reg; value afterwards has no meaning - /// %rax (written) the old value at %r9 - /// %rflags is written. Do not assume anything about it after the instruction. - AtomicRmwSeq { - ty: Type, // I8, I16, I32 or I64 - op: inst_common::AtomicRmwOp, - address: Reg, - operand: Reg, - temp: Writable, - dst_old: Writable, - }, - - /// A memory fence (mfence, lfence or sfence). - Fence { kind: FenceKind }, - - // ===================================== - // Meta-instructions generating no code. - /// Marker, no-op in generated code: SP "virtual offset" is adjusted. This - /// controls how MemArg::NominalSPOffset args are lowered. - VirtualSPOffsetAdj { offset: i64 }, - - /// Provides a way to tell the register allocator that the upcoming sequence of instructions - /// will overwrite `dst` so it should be considered as a `def`; use this with care. - /// - /// This is useful when we have a sequence of instructions whose register usages are nominally - /// `mod`s, but such that the combination of operations creates a result that is independent of - /// the initial register value. It's thus semantically a `def`, not a `mod`, when all the - /// instructions are taken together, so we want to ensure the register is defined (its - /// live-range starts) prior to the sequence to keep analyses happy. - /// - /// One alternative would be a compound instruction that somehow encapsulates the others and - /// reports its own `def`s/`use`s/`mod`s; this adds complexity (the instruction list is no - /// longer flat) and requires knowledge about semantics and initial-value independence anyway. - XmmUninitializedValue { dst: Writable }, - - /// A call to the `ElfTlsGetAddr` libcall. Returns address - /// of TLS symbol in rax. - ElfTlsGetAddr { symbol: ExternalName }, - - /// A Mach-O TLS symbol access. Returns address of the TLS - /// symbol in rax. - MachOTlsGetAddr { symbol: ExternalName }, - - /// A definition of a value label. - ValueLabelMarker { reg: Reg, label: ValueLabel }, - - /// An unwind pseudoinstruction describing the state of the - /// machine at this program point. - Unwind { inst: UnwindInst }, -} +// `Inst` is defined inside ISLE as `MInst`. We publicly re-export it here. +pub use super::lower::isle::generated_code::MInst as Inst; pub(crate) fn low32_will_sign_extend_to_64(x: u64) -> bool { let xs = x as i64; @@ -656,15 +152,20 @@ impl Inst { OperandSize::Size32, OperandSize::Size64 ])); - Self::UnaryRmR { size, op, src, dst } + Self::UnaryRmR { + size, + op, + src: GprMem::new(src).unwrap(), + dst: WritableGpr::from_writable_reg(dst).unwrap(), + } } pub(crate) fn not(size: OperandSize, src: Writable) -> Inst { debug_assert_eq!(src.to_reg().get_class(), RegClass::I64); Inst::Not { size, - src: src.to_reg(), - dst: src, + src: Gpr::new(src.to_reg()).unwrap(), + dst: WritableGpr::from_writable_reg(src).unwrap(), } } @@ -753,7 +254,11 @@ impl Inst { pub(crate) fn xmm_mov(op: SseOpcode, src: RegMem, dst: Writable) -> Inst { src.assert_regclass_is(RegClass::V128); debug_assert!(dst.to_reg().get_class() == RegClass::V128); - Inst::XmmUnaryRmR { op, src, dst } + Inst::XmmUnaryRmR { + op, + src: XmmMem::new(src).unwrap(), + dst: WritableXmm::from_writable_reg(dst).unwrap(), + } } pub(crate) fn xmm_load_const(src: VCodeConstant, dst: Writable, ty: Type) -> Inst { @@ -766,13 +271,21 @@ impl Inst { pub(crate) fn xmm_unary_rm_r(op: SseOpcode, src: RegMem, dst: Writable) -> Inst { src.assert_regclass_is(RegClass::V128); debug_assert!(dst.to_reg().get_class() == RegClass::V128); - Inst::XmmUnaryRmR { op, src, dst } + Inst::XmmUnaryRmR { + op, + src: XmmMem::new(src).unwrap(), + dst: WritableXmm::from_writable_reg(dst).unwrap(), + } } pub(crate) fn xmm_unary_rm_r_evex(op: Avx512Opcode, src: RegMem, dst: Writable) -> Inst { src.assert_regclass_is(RegClass::V128); debug_assert!(dst.to_reg().get_class() == RegClass::V128); - Inst::XmmUnaryRmREvex { op, src, dst } + Inst::XmmUnaryRmREvex { + op, + src: XmmMem::new(src).unwrap(), + dst: WritableXmm::from_writable_reg(dst).unwrap(), + } } pub(crate) fn xmm_rm_r(op: SseOpcode, src: RegMem, dst: Writable) -> Self { @@ -780,9 +293,9 @@ impl Inst { debug_assert!(dst.to_reg().get_class() == RegClass::V128); Inst::XmmRmR { op, - src1: dst.to_reg(), - src2: src, - dst, + src1: Xmm::new(dst.to_reg()).unwrap(), + src2: XmmMem::new(src).unwrap(), + dst: WritableXmm::from_writable_reg(dst).unwrap(), } } @@ -797,15 +310,17 @@ impl Inst { debug_assert!(dst.to_reg().get_class() == RegClass::V128); Inst::XmmRmREvex { op, - src1, - src2, - dst, + src1: XmmMem::new(src1).unwrap(), + src2: Xmm::new(src2).unwrap(), + dst: WritableXmm::from_writable_reg(dst).unwrap(), } } pub(crate) fn xmm_uninit_value(dst: Writable) -> Self { debug_assert!(dst.to_reg().get_class() == RegClass::V128); - Inst::XmmUninitializedValue { dst } + Inst::XmmUninitializedValue { + dst: WritableXmm::from_writable_reg(dst).unwrap(), + } } pub(crate) fn xmm_mov_r_m(op: SseOpcode, src: Reg, dst: impl Into) -> Inst { @@ -828,8 +343,8 @@ impl Inst { debug_assert!(dst_size.is_one_of(&[OperandSize::Size32, OperandSize::Size64])); Inst::XmmToGpr { op, - src, - dst, + src: Xmm::new(src).unwrap(), + dst: WritableGpr::from_writable_reg(dst).unwrap(), dst_size, } } @@ -846,7 +361,7 @@ impl Inst { Inst::GprToXmm { op, src, - dst, + dst: WritableXmm::from_writable_reg(dst).unwrap(), src_size, } } @@ -870,10 +385,10 @@ impl Inst { debug_assert!(tmp_gpr2.to_reg().get_class() == RegClass::I64); debug_assert!(dst.to_reg().get_class() == RegClass::V128); Inst::CvtUint64ToFloatSeq { - src, - dst, - tmp_gpr1, - tmp_gpr2, + src: WritableGpr::from_writable_reg(src).unwrap(), + dst: WritableXmm::from_writable_reg(dst).unwrap(), + tmp_gpr1: WritableGpr::from_writable_reg(tmp_gpr1).unwrap(), + tmp_gpr2: WritableGpr::from_writable_reg(tmp_gpr2).unwrap(), dst_size, } } @@ -897,10 +412,10 @@ impl Inst { src_size, dst_size, is_saturating, - src, - dst, - tmp_gpr, - tmp_xmm, + src: WritableXmm::from_writable_reg(src).unwrap(), + dst: WritableGpr::from_writable_reg(dst).unwrap(), + tmp_gpr: WritableGpr::from_writable_reg(tmp_gpr).unwrap(), + tmp_xmm: WritableXmm::from_writable_reg(tmp_xmm).unwrap(), } } @@ -923,10 +438,10 @@ impl Inst { src_size, dst_size, is_saturating, - src, - dst, - tmp_gpr, - tmp_xmm, + src: WritableXmm::from_writable_reg(src).unwrap(), + dst: WritableGpr::from_writable_reg(dst).unwrap(), + tmp_gpr: WritableGpr::from_writable_reg(tmp_gpr).unwrap(), + tmp_xmm: WritableXmm::from_writable_reg(tmp_xmm).unwrap(), } } @@ -976,9 +491,9 @@ impl Inst { debug_assert!(dst.to_reg().get_class() == RegClass::V128); Inst::XmmRmiReg { opcode, - src1: dst.to_reg(), - src2: src, - dst, + src1: Xmm::new(dst.to_reg()).unwrap(), + src2: XmmMemImm::new(src).unwrap(), + dst: WritableXmm::from_writable_reg(dst).unwrap(), } } @@ -1018,7 +533,7 @@ impl Inst { debug_assert!(dst.to_reg().get_class() == RegClass::I64); Inst::LoadEffectiveAddress { addr: addr.into(), - dst, + dst: WritableGpr::from_writable_reg(dst).unwrap(), } } @@ -1270,7 +785,7 @@ impl Inst { } Self::XmmRmR { op, src2, dst, .. } => { - src2.to_reg() == Some(dst.to_reg()) + src2.clone().to_reg_mem().to_reg() == Some(dst.to_reg().to_reg()) && (*op == SseOpcode::Xorps || *op == SseOpcode::Xorpd || *op == SseOpcode::Pxor @@ -1331,7 +846,11 @@ impl Inst { Inst::XmmRmiReg { src1, dst, .. } => { if *src1 != dst.to_reg() { debug_assert!(src1.is_virtual()); - insts.push(Self::gen_move(*dst, *src1, types::I8X16)); + insts.push(Self::gen_move( + dst.to_writable_reg(), + src1.to_reg(), + types::I8X16, + )); *src1 = dst.to_reg(); } insts.push(self); @@ -1339,7 +858,11 @@ impl Inst { Inst::XmmRmR { src1, dst, .. } => { if *src1 != dst.to_reg() { debug_assert!(src1.is_virtual()); - insts.push(Self::gen_move(*dst, *src1, types::I8X16)); + insts.push(Self::gen_move( + dst.to_writable_reg(), + src1.to_reg(), + types::I8X16, + )); *src1 = dst.to_reg(); } insts.push(self); @@ -1368,7 +891,11 @@ impl Inst { Inst::Not { src, dst, .. } | Inst::Neg { src, dst, .. } => { if *src != dst.to_reg() { debug_assert!(src.is_virtual()); - insts.push(Self::gen_move(*dst, *src, types::I64)); + insts.push(Self::gen_move( + dst.to_writable_reg(), + src.to_reg(), + types::I64, + )); *src = dst.to_reg(); } insts.push(self); @@ -1626,19 +1153,19 @@ impl PrettyPrint for Inst { "{} {}, {}", ljustify2(op.to_string(), suffix_bwlq(*size)), src.show_rru_sized(mb_rru, size.to_bytes()), - show_ireg_sized(dst.to_reg(), mb_rru, size.to_bytes()), + show_ireg_sized(dst.to_reg().to_reg(), mb_rru, size.to_bytes()), ), Inst::Not { size, src: _, dst } => format!( "{} {}", ljustify2("not".to_string(), suffix_bwlq(*size)), - show_ireg_sized(dst.to_reg(), mb_rru, size.to_bytes()) + show_ireg_sized(dst.to_reg().to_reg(), mb_rru, size.to_bytes()) ), Inst::Neg { size, src: _, dst } => format!( "{} {}", ljustify2("neg".to_string(), suffix_bwlq(*size)), - show_ireg_sized(dst.to_reg(), mb_rru, size.to_bytes()) + show_ireg_sized(dst.to_reg().to_reg(), mb_rru, size.to_bytes()) ), Inst::Div { @@ -1696,14 +1223,14 @@ impl PrettyPrint for Inst { "{} {}, {}", ljustify(op.to_string()), src.show_rru_sized(mb_rru, op.src_size()), - show_ireg_sized(dst.to_reg(), mb_rru, 8), + show_ireg_sized(dst.to_reg().to_reg(), mb_rru, 8), ), Inst::XmmUnaryRmREvex { op, src, dst, .. } => format!( "{} {}, {}", ljustify(op.to_string()), src.show_rru_sized(mb_rru, 8), - show_ireg_sized(dst.to_reg(), mb_rru, 8), + show_ireg_sized(dst.to_reg().to_reg(), mb_rru, 8), ), Inst::XmmMovRM { op, src, dst, .. } => format!( @@ -1717,7 +1244,7 @@ impl PrettyPrint for Inst { "{} {}, {}", ljustify(op.to_string()), src2.show_rru_sized(mb_rru, 8), - show_ireg_sized(dst.to_reg(), mb_rru, 8), + show_ireg_sized(dst.to_reg().to_reg(), mb_rru, 8), ), Inst::XmmRmREvex { @@ -1730,8 +1257,8 @@ impl PrettyPrint for Inst { "{} {}, {}, {}", ljustify(op.to_string()), src1.show_rru_sized(mb_rru, 8), - show_ireg_sized(*src2, mb_rru, 8), - show_ireg_sized(dst.to_reg(), mb_rru, 8), + show_ireg_sized(src2.to_reg(), mb_rru, 8), + show_ireg_sized(dst.to_reg().to_reg(), mb_rru, 8), ), Inst::XmmMinMaxSeq { @@ -1795,7 +1322,7 @@ impl PrettyPrint for Inst { "{} {}, {}", ljustify(op.to_string()), src.show_rru(mb_rru), - show_ireg_sized(dst.to_reg(), mb_rru, dst_size), + show_ireg_sized(dst.to_reg().to_reg(), mb_rru, dst_size), ) } @@ -1830,7 +1357,7 @@ impl PrettyPrint for Inst { "f32" } )), - show_ireg_sized(src.to_reg(), mb_rru, 8), + show_ireg_sized(src.to_reg().to_reg(), mb_rru, 8), dst.show_rru(mb_rru), ), @@ -1847,8 +1374,8 @@ impl PrettyPrint for Inst { src_size.to_bits(), dst_size.to_bits() )), - show_ireg_sized(src.to_reg(), mb_rru, 8), - show_ireg_sized(dst.to_reg(), mb_rru, dst_size.to_bytes()), + show_ireg_sized(src.to_reg().to_reg(), mb_rru, 8), + show_ireg_sized(dst.to_reg().to_reg(), mb_rru, dst_size.to_bytes()), ), Inst::CvtFloatToUintSeq { @@ -1864,8 +1391,8 @@ impl PrettyPrint for Inst { src_size.to_bits(), dst_size.to_bits() )), - show_ireg_sized(src.to_reg(), mb_rru, 8), - show_ireg_sized(dst.to_reg(), mb_rru, dst_size.to_bytes()), + show_ireg_sized(src.to_reg().to_reg(), mb_rru, 8), + show_ireg_sized(dst.to_reg().to_reg(), mb_rru, dst_size.to_bytes()), ), Inst::Imm { @@ -2174,11 +1701,11 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) { } Inst::Not { src, dst, .. } => { debug_assert_eq!(*src, dst.to_reg()); - collector.add_mod(*dst); + collector.add_mod(dst.to_writable_reg()); } Inst::Neg { src, dst, .. } => { debug_assert_eq!(*src, dst.to_reg()); - collector.add_mod(*dst); + collector.add_mod(dst.to_writable_reg()); } Inst::Div { size, @@ -2249,11 +1776,13 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) { } } } - Inst::UnaryRmR { src, dst, .. } - | Inst::XmmUnaryRmR { src, dst, .. } - | Inst::XmmUnaryRmREvex { src, dst, .. } => { - src.get_regs_as_uses(collector); - collector.add_def(*dst); + Inst::UnaryRmR { src, dst, .. } => { + src.clone().to_reg_mem().get_regs_as_uses(collector); + collector.add_def(dst.to_writable_reg()); + } + Inst::XmmUnaryRmR { src, dst, .. } | Inst::XmmUnaryRmREvex { src, dst, .. } => { + src.clone().to_reg_mem().get_regs_as_uses(collector); + collector.add_def(dst.to_writable_reg()); } Inst::XmmRmR { src1, @@ -2265,10 +1794,10 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) { debug_assert_eq!(*src1, dst.to_reg()); if inst.produces_const() { // No need to account for src, since src == dst. - collector.add_def(*dst); + collector.add_def(dst.to_writable_reg()); } else { - src2.get_regs_as_uses(collector); - collector.add_mod(*dst); + src2.clone().to_reg_mem().get_regs_as_uses(collector); + collector.add_mod(dst.to_writable_reg()); // Some instructions have an implicit use of XMM0. if *op == SseOpcode::Blendvpd || *op == SseOpcode::Blendvps @@ -2285,11 +1814,11 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) { dst, .. } => { - src1.get_regs_as_uses(collector); - collector.add_use(*src2); + src1.clone().to_reg_mem().get_regs_as_uses(collector); + collector.add_use(src2.to_reg()); match *op { - Avx512Opcode::Vpermi2b => collector.add_mod(*dst), - _ => collector.add_def(*dst), + Avx512Opcode::Vpermi2b => collector.add_mod(dst.to_writable_reg()), + _ => collector.add_def(dst.to_writable_reg()), } } Inst::XmmRmRImm { @@ -2320,7 +1849,7 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) { collector.add_mod(*dst); } } - Inst::XmmUninitializedValue { dst } => collector.add_def(*dst), + Inst::XmmUninitializedValue { dst } => collector.add_def(dst.to_writable_reg()), Inst::XmmLoadConst { dst, .. } => collector.add_def(*dst), Inst::XmmMinMaxSeq { lhs, rhs_dst, .. } => { collector.add_use(*lhs); @@ -2330,8 +1859,8 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) { src1, src2, dst, .. } => { debug_assert_eq!(*src1, dst.to_reg()); - src2.get_regs_as_uses(collector); - collector.add_mod(*dst); + src2.clone().to_reg_mem_imm().get_regs_as_uses(collector); + collector.add_mod(dst.to_writable_reg()); } Inst::XmmMovRM { src, dst, .. } => { collector.add_use(*src); @@ -2344,13 +1873,17 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) { Inst::Imm { dst, .. } => { collector.add_def(*dst); } - Inst::MovRR { src, dst, .. } | Inst::XmmToGpr { src, dst, .. } => { + Inst::MovRR { src, dst, .. } => { collector.add_use(*src); collector.add_def(*dst); } + Inst::XmmToGpr { src, dst, .. } => { + collector.add_use(src.to_reg()); + collector.add_def(dst.to_writable_reg()); + } Inst::GprToXmm { src, dst, .. } => { src.get_regs_as_uses(collector); - collector.add_def(*dst); + collector.add_def(dst.to_writable_reg()); } Inst::CvtUint64ToFloatSeq { src, @@ -2359,10 +1892,10 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) { tmp_gpr2, .. } => { - collector.add_mod(*src); - collector.add_def(*dst); - collector.add_def(*tmp_gpr1); - collector.add_def(*tmp_gpr2); + collector.add_mod(src.to_writable_reg()); + collector.add_def(dst.to_writable_reg()); + collector.add_def(tmp_gpr1.to_writable_reg()); + collector.add_def(tmp_gpr2.to_writable_reg()); } Inst::CvtFloatToSintSeq { src, @@ -2378,19 +1911,23 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) { tmp_xmm, .. } => { - collector.add_mod(*src); - collector.add_def(*dst); - collector.add_def(*tmp_gpr); - collector.add_def(*tmp_xmm); + collector.add_mod(src.to_writable_reg()); + collector.add_def(dst.to_writable_reg()); + collector.add_def(tmp_gpr.to_writable_reg()); + collector.add_def(tmp_xmm.to_writable_reg()); } Inst::MovzxRmR { src, dst, .. } => { src.get_regs_as_uses(collector); collector.add_def(*dst); } - Inst::Mov64MR { src, dst, .. } | Inst::LoadEffectiveAddress { addr: src, dst } => { + Inst::Mov64MR { src, dst, .. } => { src.get_regs_as_uses(collector); collector.add_def(*dst) } + Inst::LoadEffectiveAddress { addr: src, dst } => { + src.get_regs_as_uses(collector); + collector.add_def(dst.to_writable_reg()) + } Inst::MovsxRmR { src, dst, .. } => { src.get_regs_as_uses(collector); collector.add_def(*dst); @@ -2535,8 +2072,8 @@ impl Amode { ref mut index, .. } => { - map.map_use(base); - map.map_use(index); + base.map_use(map); + index.map_use(map); } Amode::RipRelative { .. } => { // RIP isn't involved in regalloc. @@ -2622,7 +2159,7 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { } Inst::Not { src, dst, .. } | Inst::Neg { src, dst, .. } => { debug_assert_eq!(*src, dst.to_reg()); - mapper.map_mod(dst); + dst.map_mod(mapper); *src = dst.to_reg(); } Inst::Div { divisor, .. } => divisor.map_uses(mapper), @@ -2643,14 +2180,17 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { ref mut src, ref mut dst, .. + } => { + src.map_uses(mapper); + dst.map_def(mapper); } - | Inst::UnaryRmR { + Inst::UnaryRmR { ref mut src, ref mut dst, .. } => { src.map_uses(mapper); - mapper.map_def(dst); + dst.map_def(mapper); } Inst::XmmRmRImm { ref op, @@ -2691,11 +2231,11 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { debug_assert_eq!(*src1, dst.to_reg()); if produces_const { src2.map_as_def(mapper); - mapper.map_def(dst); + dst.map_def(mapper); *src1 = dst.to_reg(); } else { src2.map_uses(mapper); - mapper.map_mod(dst); + dst.map_mod(mapper); *src1 = dst.to_reg(); } } @@ -2707,10 +2247,10 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { .. } => { src1.map_uses(mapper); - mapper.map_use(src2); + src2.map_use(mapper); match *op { - Avx512Opcode::Vpermi2b => mapper.map_mod(dst), - _ => mapper.map_def(dst), + Avx512Opcode::Vpermi2b => dst.map_mod(mapper), + _ => dst.map_def(mapper), } } Inst::XmmRmiReg { @@ -2721,11 +2261,11 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { } => { debug_assert_eq!(*src1, dst.to_reg()); src2.map_uses(mapper); - mapper.map_mod(dst); + dst.map_mod(mapper); *src1 = dst.to_reg(); } Inst::XmmUninitializedValue { ref mut dst, .. } => { - mapper.map_def(dst); + dst.map_def(mapper); } Inst::XmmLoadConst { ref mut dst, .. } => { mapper.map_def(dst); @@ -2759,14 +2299,17 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { ref mut src, ref mut dst, .. + } => { + mapper.map_use(src); + mapper.map_def(dst); } - | Inst::XmmToGpr { + Inst::XmmToGpr { ref mut src, ref mut dst, .. } => { - mapper.map_use(src); - mapper.map_def(dst); + src.map_use(mapper); + dst.map_def(mapper); } Inst::GprToXmm { ref mut src, @@ -2774,7 +2317,7 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { .. } => { src.map_uses(mapper); - mapper.map_def(dst); + dst.map_def(mapper); } Inst::CvtUint64ToFloatSeq { ref mut src, @@ -2783,10 +2326,10 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { ref mut tmp_gpr2, .. } => { - mapper.map_mod(src); - mapper.map_def(dst); - mapper.map_def(tmp_gpr1); - mapper.map_def(tmp_gpr2); + src.map_mod(mapper); + dst.map_def(mapper); + tmp_gpr1.map_def(mapper); + tmp_gpr2.map_def(mapper); } Inst::CvtFloatToSintSeq { ref mut src, @@ -2802,10 +2345,10 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { ref mut tmp_xmm, .. } => { - mapper.map_mod(src); - mapper.map_def(dst); - mapper.map_def(tmp_gpr); - mapper.map_def(tmp_xmm); + src.map_mod(mapper); + dst.map_def(mapper); + tmp_gpr.map_def(mapper); + tmp_xmm.map_def(mapper); } Inst::MovzxRmR { ref mut src, @@ -2815,10 +2358,14 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { src.map_uses(mapper); mapper.map_def(dst); } - Inst::Mov64MR { src, dst, .. } | Inst::LoadEffectiveAddress { addr: src, dst } => { + Inst::Mov64MR { src, dst, .. } => { src.map_uses(mapper); mapper.map_def(dst); } + Inst::LoadEffectiveAddress { addr: src, dst } => { + src.map_uses(mapper); + dst.map_def(mapper); + } Inst::MovsxRmR { ref mut src, ref mut dst, @@ -2989,8 +2536,8 @@ impl MachInst for Inst { || *op == SseOpcode::Movdqa || *op == SseOpcode::Movdqu => { - if let RegMem::Reg { reg } = src { - Some((*dst, *reg)) + if let RegMem::Reg { reg } = src.clone().to_reg_mem() { + Some((dst.to_writable_reg(), reg)) } else { None } diff --git a/cranelift/codegen/src/isa/x64/lower.isle b/cranelift/codegen/src/isa/x64/lower.isle index 9b881b5f691c..8de9110bbdfe 100644 --- a/cranelift/codegen/src/isa/x64/lower.isle +++ b/cranelift/codegen/src/isa/x64/lower.isle @@ -95,23 +95,23 @@ (rule (lower (has_type (multi_lane 8 16) (iadd x y))) - (value_reg (paddb (put_in_reg x) - (put_in_reg_mem y)))) + (value_xmm (paddb (put_in_xmm x) + (put_in_xmm_mem y)))) (rule (lower (has_type (multi_lane 16 8) (iadd x y))) - (value_reg (paddw (put_in_reg x) - (put_in_reg_mem y)))) + (value_xmm (paddw (put_in_xmm x) + (put_in_xmm_mem y)))) (rule (lower (has_type (multi_lane 32 4) (iadd x y))) - (value_reg (paddd (put_in_reg x) - (put_in_reg_mem y)))) + (value_xmm (paddd (put_in_xmm x) + (put_in_xmm_mem y)))) (rule (lower (has_type (multi_lane 64 2) (iadd x y))) - (value_reg (paddq (put_in_reg x) - (put_in_reg_mem y)))) + (value_xmm (paddq (put_in_xmm x) + (put_in_xmm_mem y)))) ;; `i128` (rule (lower (has_type $I128 (iadd x y))) @@ -131,25 +131,25 @@ (rule (lower (has_type (multi_lane 8 16) (sadd_sat x y))) - (value_reg (paddsb (put_in_reg x) - (put_in_reg_mem y)))) + (value_xmm (paddsb (put_in_xmm x) + (put_in_xmm_mem y)))) (rule (lower (has_type (multi_lane 16 8) (sadd_sat x y))) - (value_reg (paddsw (put_in_reg x) - (put_in_reg_mem y)))) + (value_xmm (paddsw (put_in_xmm x) + (put_in_xmm_mem y)))) ;;;; Rules for `uadd_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (rule (lower (has_type (multi_lane 8 16) (uadd_sat x y))) - (value_reg (paddusb (put_in_reg x) - (put_in_reg_mem y)))) + (value_xmm (paddusb (put_in_xmm x) + (put_in_xmm_mem y)))) (rule (lower (has_type (multi_lane 16 8) (uadd_sat x y))) - (value_reg (paddusw (put_in_reg x) - (put_in_reg_mem y)))) + (value_xmm (paddusw (put_in_xmm x) + (put_in_xmm_mem y)))) ;;;; Rules for `iadd_ifcout` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -264,23 +264,23 @@ (rule (lower (has_type (multi_lane 8 16) (isub x y))) - (value_reg (psubb (put_in_reg x) - (put_in_reg_mem y)))) + (value_xmm (psubb (put_in_xmm x) + (put_in_xmm_mem y)))) (rule (lower (has_type (multi_lane 16 8) (isub x y))) - (value_reg (psubw (put_in_reg x) - (put_in_reg_mem y)))) + (value_xmm (psubw (put_in_xmm x) + (put_in_xmm_mem y)))) (rule (lower (has_type (multi_lane 32 4) (isub x y))) - (value_reg (psubd (put_in_reg x) - (put_in_reg_mem y)))) + (value_xmm (psubd (put_in_xmm x) + (put_in_xmm_mem y)))) (rule (lower (has_type (multi_lane 64 2) (isub x y))) - (value_reg (psubq (put_in_reg x) - (put_in_reg_mem y)))) + (value_xmm (psubq (put_in_xmm x) + (put_in_xmm_mem y)))) ;; `i128` (rule (lower (has_type $I128 (isub x y))) @@ -300,25 +300,25 @@ (rule (lower (has_type (multi_lane 8 16) (ssub_sat x y))) - (value_reg (psubsb (put_in_reg x) - (put_in_reg_mem y)))) + (value_xmm (psubsb (put_in_xmm x) + (put_in_xmm_mem y)))) (rule (lower (has_type (multi_lane 16 8) (ssub_sat x y))) - (value_reg (psubsw (put_in_reg x) - (put_in_reg_mem y)))) + (value_xmm (psubsw (put_in_xmm x) + (put_in_xmm_mem y)))) ;;;; Rules for `usub_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (rule (lower (has_type (multi_lane 8 16) (usub_sat x y))) - (value_reg (psubusb (put_in_reg x) - (put_in_reg_mem y)))) + (value_xmm (psubusb (put_in_xmm x) + (put_in_xmm_mem y)))) (rule (lower (has_type (multi_lane 16 8) (usub_sat x y))) - (value_reg (psubusw (put_in_reg x) - (put_in_reg_mem y)))) + (value_xmm (psubusw (put_in_xmm x) + (put_in_xmm_mem y)))) ;;;; Rules for `band` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -360,14 +360,16 @@ ;; SSE. -(decl sse_and (Type Reg RegMem) Reg) +(decl sse_and (Type Xmm XmmMem) Xmm) (rule (sse_and $F32X4 x y) (andps x y)) (rule (sse_and $F64X2 x y) (andpd x y)) (rule (sse_and (multi_lane _bits _lanes) x y) (pand x y)) (rule (lower (has_type ty @ (multi_lane _bits _lanes) (band x y))) - (value_reg (sse_and ty (put_in_reg x) (put_in_reg_mem y)))) + (value_xmm (sse_and ty + (put_in_xmm x) + (put_in_xmm_mem y)))) ;; `{i,b}128`. @@ -432,14 +434,16 @@ ;; SSE. -(decl sse_or (Type Reg RegMem) Reg) +(decl sse_or (Type Xmm XmmMem) Xmm) (rule (sse_or $F32X4 x y) (orps x y)) (rule (sse_or $F64X2 x y) (orpd x y)) (rule (sse_or (multi_lane _bits _lanes) x y) (por x y)) (rule (lower (has_type ty @ (multi_lane _bits _lanes) (bor x y))) - (value_reg (sse_or ty (put_in_reg x) (put_in_reg_mem y)))) + (value_xmm (sse_or ty + (put_in_xmm x) + (put_in_xmm_mem y)))) ;; `{i,b}128`. @@ -507,7 +511,7 @@ ;; SSE. (rule (lower (has_type ty @ (multi_lane _bits _lanes) (bxor x y))) - (value_reg (sse_xor ty (put_in_reg x) (put_in_reg_mem y)))) + (value_xmm (sse_xor ty (put_in_xmm x) (put_in_xmm_mem y)))) ;; `{i,b}128`. @@ -578,16 +582,16 @@ ;; instructions. The basic idea, whether the amount to shift by is an immediate ;; or not, is to use a 16x8 shift and then mask off the incorrect bits to 0s. (rule (lower (has_type $I8X16 (ishl src amt))) - (let ((src_ Reg (put_in_reg src)) + (let ((src_ Xmm (put_in_xmm src)) (amt_gpr RegMemImm (put_in_reg_mem_imm amt)) - (amt_xmm RegMemImm (reg_mem_imm_to_xmm amt_gpr)) + (amt_xmm XmmMemImm (mov_rmi_to_xmm amt_gpr)) ;; Shift `src` using 16x8. Unfortunately, a 16x8 shift will only be ;; correct for half of the lanes; the others must be fixed up with ;; the mask below. - (unmasked Reg (psllw src_ amt_xmm)) + (unmasked Xmm (psllw src_ amt_xmm)) (mask_addr SyntheticAmode (ishl_i8x16_mask amt_gpr)) (mask Reg (x64_load $I8X16 mask_addr (ExtKind.None)))) - (value_reg (sse_and $I8X16 unmasked (RegMem.Reg mask))))) + (value_xmm (sse_and $I8X16 unmasked (xmm_mem_new (RegMem.Reg mask)))))) ;; Get the address of the mask to use when fixing up the lanes that weren't ;; correctly generated by the 16x8 shift. @@ -608,25 +612,28 @@ (extern constructor ishl_i8x16_mask_table ishl_i8x16_mask_table) (rule (ishl_i8x16_mask (RegMemImm.Reg amt)) (let ((mask_table SyntheticAmode (ishl_i8x16_mask_table)) - (base_mask_addr Reg (lea mask_table)) + (base_mask_addr Gpr (lea mask_table)) (mask_offset Reg (shl $I64 amt (Imm8Reg.Imm8 4)))) (amode_to_synthetic_amode (amode_imm_reg_reg_shift 0 base_mask_addr - mask_offset + (gpr_new mask_offset) 0)))) (rule (ishl_i8x16_mask (RegMemImm.Mem amt)) (ishl_i8x16_mask (RegMemImm.Reg (x64_load $I64 amt (ExtKind.None))))) ;; 16x8, 32x4, and 64x2 shifts can each use a single instruction. + (rule (lower (has_type $I16X8 (ishl src amt))) - (value_reg (psllw (put_in_reg src) - (reg_mem_imm_to_xmm (put_in_reg_mem_imm amt))))) + (value_xmm (psllw (put_in_xmm src) + (mov_rmi_to_xmm (put_in_reg_mem_imm amt))))) + (rule (lower (has_type $I32X4 (ishl src amt))) - (value_reg (pslld (put_in_reg src) - (reg_mem_imm_to_xmm (put_in_reg_mem_imm amt))))) + (value_xmm (pslld (put_in_xmm src) + (mov_rmi_to_xmm (put_in_reg_mem_imm amt))))) + (rule (lower (has_type $I64X2 (ishl src amt))) - (value_reg (psllq (put_in_reg src) - (reg_mem_imm_to_xmm (put_in_reg_mem_imm amt))))) + (value_xmm (psllq (put_in_xmm src) + (mov_rmi_to_xmm (put_in_reg_mem_imm amt))))) ;;;; Rules for `ushr` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -672,16 +679,18 @@ ;; There are no 8x16 shifts in x64. Do the same 16x8-shift-and-mask thing we do ;; with 8x16 `ishl`. (rule (lower (has_type $I8X16 (ushr src amt))) - (let ((src_ Reg (put_in_reg src)) + (let ((src_ Xmm (put_in_xmm src)) (amt_gpr RegMemImm (put_in_reg_mem_imm amt)) - (amt_xmm RegMemImm (reg_mem_imm_to_xmm amt_gpr)) + (amt_xmm XmmMemImm (mov_rmi_to_xmm amt_gpr)) ;; Shift `src` using 16x8. Unfortunately, a 16x8 shift will only be ;; correct for half of the lanes; the others must be fixed up with ;; the mask below. - (unmasked Reg (psrlw src_ amt_xmm)) + (unmasked Xmm (psrlw src_ amt_xmm)) (mask_addr SyntheticAmode (ushr_i8x16_mask amt_gpr)) (mask Reg (x64_load $I8X16 mask_addr (ExtKind.None)))) - (value_reg (sse_and $I8X16 unmasked (RegMem.Reg mask))))) + (value_xmm (sse_and $I8X16 + unmasked + (xmm_mem_new (RegMem.Reg mask)))))) ;; Get the address of the mask to use when fixing up the lanes that weren't ;; correctly generated by the 16x8 shift. @@ -702,25 +711,28 @@ (extern constructor ushr_i8x16_mask_table ushr_i8x16_mask_table) (rule (ushr_i8x16_mask (RegMemImm.Reg amt)) (let ((mask_table SyntheticAmode (ushr_i8x16_mask_table)) - (base_mask_addr Reg (lea mask_table)) + (base_mask_addr Gpr (lea mask_table)) (mask_offset Reg (shl $I64 amt (Imm8Reg.Imm8 4)))) (amode_to_synthetic_amode (amode_imm_reg_reg_shift 0 base_mask_addr - mask_offset + (gpr_new mask_offset) 0)))) (rule (ushr_i8x16_mask (RegMemImm.Mem amt)) (ushr_i8x16_mask (RegMemImm.Reg (x64_load $I64 amt (ExtKind.None))))) ;; 16x8, 32x4, and 64x2 shifts can each use a single instruction. + (rule (lower (has_type $I16X8 (ushr src amt))) - (value_reg (psrlw (put_in_reg src) - (reg_mem_imm_to_xmm (put_in_reg_mem_imm amt))))) + (value_xmm (psrlw (put_in_xmm src) + (mov_rmi_to_xmm (put_in_reg_mem_imm amt))))) + (rule (lower (has_type $I32X4 (ushr src amt))) - (value_reg (psrld (put_in_reg src) - (reg_mem_imm_to_xmm (put_in_reg_mem_imm amt))))) + (value_xmm (psrld (put_in_xmm src) + (mov_rmi_to_xmm (put_in_reg_mem_imm amt))))) + (rule (lower (has_type $I64X2 (ushr src amt))) - (value_reg (psrlq (put_in_reg src) - (reg_mem_imm_to_xmm (put_in_reg_mem_imm amt))))) + (value_xmm (psrlq (put_in_xmm src) + (mov_rmi_to_xmm (put_in_reg_mem_imm amt))))) ;;;; Rules for `sshr` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -780,33 +792,35 @@ ;; shifted_hi.i16x8 = shift each lane of `high` ;; result = [s0'', s1'', ..., s15''] (rule (lower (has_type $I8X16 (sshr src amt @ (value_type amt_ty)))) - (let ((src_ Reg (put_in_reg src)) + (let ((src_ Xmm (put_in_xmm src)) ;; In order for `packsswb` later to only use the high byte of each ;; 16x8 lane, we shift right an extra 8 bits, relying on `psraw` to ;; fill in the upper bits appropriately. - (lo Reg (punpcklbw src_ (RegMem.Reg src_))) - (hi Reg (punpckhbw src_ (RegMem.Reg src_))) - (amt_ RegMemImm (sshr_i8x16_bigger_shift amt_ty (put_in_reg_mem_imm amt))) - (shifted_lo Reg (psraw lo amt_)) - (shifted_hi Reg (psraw hi amt_))) - (value_reg (packsswb shifted_lo (RegMem.Reg shifted_hi))))) - -(decl sshr_i8x16_bigger_shift (Type RegMemImm) RegMemImm) + (lo Xmm (punpcklbw src_ (xmm_to_xmm_mem src_))) + (hi Xmm (punpckhbw src_ (xmm_to_xmm_mem src_))) + (amt_ XmmMemImm (sshr_i8x16_bigger_shift amt_ty (put_in_reg_mem_imm amt))) + (shifted_lo Xmm (psraw lo amt_)) + (shifted_hi Xmm (psraw hi amt_))) + (value_xmm (packsswb shifted_lo (xmm_to_xmm_mem shifted_hi))))) + +(decl sshr_i8x16_bigger_shift (Type RegMemImm) XmmMemImm) (rule (sshr_i8x16_bigger_shift _ty (RegMemImm.Imm i)) - (RegMemImm.Imm (u32_add i 8))) + (xmm_mem_imm_new (RegMemImm.Imm (u32_add i 8)))) (rule (sshr_i8x16_bigger_shift ty (RegMemImm.Reg r)) - (reg_mem_imm_to_xmm (RegMemImm.Reg (add ty r (RegMemImm.Imm 8))))) + (mov_rmi_to_xmm (RegMemImm.Reg (add ty r (RegMemImm.Imm 8))))) (rule (sshr_i8x16_bigger_shift ty rmi @ (RegMemImm.Mem _m)) - (reg_mem_imm_to_xmm (RegMemImm.Reg (add ty (imm ty 8) rmi)))) + (mov_rmi_to_xmm (RegMemImm.Reg (add ty (imm ty 8) rmi)))) ;; `sshr.{i16x8,i32x4}` can be a simple `psra{w,d}`, we just have to make sure ;; that if the shift amount is in a register, it is in an XMM register. + (rule (lower (has_type $I16X8 (sshr src amt))) - (value_reg (psraw (put_in_reg src) - (reg_mem_imm_to_xmm (put_in_reg_mem_imm amt))))) + (value_xmm (psraw (put_in_xmm src) + (mov_rmi_to_xmm (put_in_reg_mem_imm amt))))) + (rule (lower (has_type $I32X4 (sshr src amt))) - (value_reg (psrad (put_in_reg src) - (reg_mem_imm_to_xmm (put_in_reg_mem_imm amt))))) + (value_xmm (psrad (put_in_xmm src) + (mov_rmi_to_xmm (put_in_reg_mem_imm amt))))) ;; The `sshr.i64x2` CLIF instruction has no single x86 instruction in the older ;; feature sets. Newer ones like AVX512VL + AVX512F include `vpsraq`, a 128-bit @@ -817,14 +831,15 @@ ;; ;; (TODO: when EVEX support is available, add an alternate lowering here). (rule (lower (has_type $I64X2 (sshr src amt))) - (let ((src_ Reg (put_in_reg src)) - (lo Reg (pextrd $I64 src_ 0)) - (hi Reg (pextrd $I64 src_ 1)) + (let ((src_ Xmm (put_in_xmm src)) + (lo Gpr (pextrd $I64 src_ 0)) + (hi Gpr (pextrd $I64 src_ 1)) (amt_ Imm8Reg (put_masked_in_imm8_reg amt $I64)) - (shifted_lo Reg (sar $I64 lo amt_)) - (shifted_hi Reg (sar $I64 hi amt_))) - (value_reg (make_i64x2_from_lanes (RegMem.Reg shifted_lo) - (RegMem.Reg shifted_hi))))) + (shifted_lo Reg (sar $I64 (gpr_to_reg lo) amt_)) + (shifted_hi Reg (sar $I64 (gpr_to_reg hi) amt_))) + (value_xmm (make_i64x2_from_lanes (gpr_mem_new (RegMem.Reg shifted_lo)) + (gpr_mem_new (RegMem.Reg shifted_hi)))))) + ;;;; Rules for `rotl` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; `i16` and `i8`: we need to extend the shift amount, or mask the @@ -910,35 +925,35 @@ ;; `i64` and smaller. (rule (lower (has_type (fits_in_64 ty) (ineg x))) - (value_reg (neg ty (put_in_reg x)))) + (value_gpr (neg ty (put_in_gpr x)))) ;; SSE. (rule (lower (has_type $I8X16 (ineg x))) - (value_reg (psubb (imm $I8X16 0) - (put_in_reg_mem x)))) + (value_xmm (psubb (xmm_new (imm $I8X16 0)) + (put_in_xmm_mem x)))) (rule (lower (has_type $I16X8 (ineg x))) - (value_reg (psubw (imm $I16X8 0) - (put_in_reg_mem x)))) + (value_xmm (psubw (xmm_new (imm $I16X8 0)) + (put_in_xmm_mem x)))) (rule (lower (has_type $I32X4 (ineg x))) - (value_reg (psubd (imm $I32X4 0) - (put_in_reg_mem x)))) + (value_xmm (psubd (xmm_new (imm $I32X4 0)) + (put_in_xmm_mem x)))) (rule (lower (has_type $I64X2 (ineg x))) - (value_reg (psubq (imm $I64X2 0) - (put_in_reg_mem x)))) + (value_xmm (psubq (xmm_new (imm $I64X2 0)) + (put_in_xmm_mem x)))) ;;;; Rules for `avg_round` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (rule (lower (has_type (multi_lane 8 16) (avg_round x y))) - (value_reg (pavgb (put_in_reg x) (put_in_reg_mem y)))) + (value_xmm (pavgb (put_in_xmm x) (put_in_xmm_mem y)))) (rule (lower (has_type (multi_lane 16 8) (avg_round x y))) - (value_reg (pavgw (put_in_reg x) (put_in_reg_mem y)))) + (value_xmm (pavgw (put_in_xmm x) (put_in_xmm_mem y)))) ;;;; Rules for `imul` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -1017,10 +1032,10 @@ ;; (No i8x16 multiply.) (rule (lower (has_type (multi_lane 16 8) (imul x y))) - (value_reg (pmullw (put_in_reg x) (put_in_reg_mem y)))) + (value_xmm (pmullw (put_in_xmm x) (put_in_xmm_mem y)))) (rule (lower (has_type (multi_lane 32 4) (imul x y))) - (value_reg (pmulld (put_in_reg x) (put_in_reg_mem y)))) + (value_xmm (pmulld (put_in_xmm x) (put_in_xmm_mem y)))) ;; With AVX-512 we can implement `i64x2` multiplication with a single ;; instruction. @@ -1028,7 +1043,7 @@ (avx512dq_enabled) (multi_lane 64 2)) (imul x y))) - (value_reg (vpmullq (put_in_reg_mem x) (put_in_reg y)))) + (value_xmm (vpmullq (put_in_xmm_mem x) (put_in_xmm y)))) ;; Otherwise, for i64x2 multiplication we describe a lane A as being composed of ;; a 32-bit upper half "Ah" and a 32-bit lower half "Al". The 32-bit long hand @@ -1052,24 +1067,24 @@ ;; 32-bits when doing calculations, i.e., `Ah == A >> 32`. (rule (lower (has_type (multi_lane 64 2) (imul a b))) - (let ((a0 Reg (put_in_reg a)) - (b0 Reg (put_in_reg b)) + (let ((a0 Xmm (put_in_xmm a)) + (b0 Xmm (put_in_xmm b)) ;; a_hi = A >> 32 - (a_hi Reg (psrlq a0 (RegMemImm.Imm 32))) + (a_hi Xmm (psrlq a0 (xmm_mem_imm_new (RegMemImm.Imm 32)))) ;; ah_bl = Ah * Bl - (ah_bl Reg (pmuludq a_hi (RegMem.Reg b0))) + (ah_bl Xmm (pmuludq a_hi (xmm_to_xmm_mem b0))) ;; b_hi = B >> 32 - (b_hi Reg (psrlq b0 (RegMemImm.Imm 32))) + (b_hi Xmm (psrlq b0 (xmm_mem_imm_new (RegMemImm.Imm 32)))) ;; al_bh = Al * Bh - (al_bh Reg (pmuludq a0 (RegMem.Reg b_hi))) + (al_bh Xmm (pmuludq a0 (xmm_to_xmm_mem b_hi))) ;; aa_bb = ah_bl + al_bh - (aa_bb Reg (paddq ah_bl (RegMem.Reg al_bh))) + (aa_bb Xmm (paddq ah_bl (xmm_to_xmm_mem al_bh))) ;; aa_bb_shifted = aa_bb << 32 - (aa_bb_shifted Reg (psllq aa_bb (RegMemImm.Imm 32))) + (aa_bb_shifted Xmm (psllq aa_bb (xmm_mem_imm_new (RegMemImm.Imm 32)))) ;; al_bl = Al * Bl - (al_bl Reg (pmuludq a0 (RegMem.Reg b0)))) + (al_bl Xmm (pmuludq a0 (xmm_to_xmm_mem b0)))) ;; al_bl + aa_bb_shifted - (value_reg (paddq al_bl (RegMem.Reg aa_bb_shifted))))) + (value_xmm (paddq al_bl (xmm_to_xmm_mem aa_bb_shifted))))) ;; Special case for `i16x8.extmul_high_i8x16_s`. (rule (lower (has_type (multi_lane 16 8) @@ -1077,13 +1092,13 @@ x))) (def_inst (swiden_high (and (value_type (multi_lane 8 16)) y)))))) - (let ((x1 Reg (put_in_reg x)) - (x2 Reg (palignr x1 (RegMem.Reg x1) 8 (OperandSize.Size32))) - (x3 Reg (pmovsxbw (RegMem.Reg x2))) - (y1 Reg (put_in_reg y)) - (y2 Reg (palignr y1 (RegMem.Reg y1) 8 (OperandSize.Size32))) - (y3 Reg (pmovsxbw (RegMem.Reg y2)))) - (value_reg (pmullw x3 (RegMem.Reg y3))))) + (let ((x1 Xmm (put_in_xmm x)) + (x2 Xmm (palignr x1 (xmm_to_xmm_mem x1) 8 (OperandSize.Size32))) + (x3 Xmm (pmovsxbw (xmm_to_xmm_mem x2))) + (y1 Xmm (put_in_xmm y)) + (y2 Xmm (palignr y1 (xmm_to_xmm_mem y1) 8 (OperandSize.Size32))) + (y3 Xmm (pmovsxbw (xmm_to_xmm_mem y2)))) + (value_xmm (pmullw x3 (xmm_to_xmm_mem y3))))) ;; Special case for `i32x4.extmul_high_i16x8_s`. (rule (lower (has_type (multi_lane 32 4) @@ -1091,11 +1106,11 @@ x))) (def_inst (swiden_high (and (value_type (multi_lane 16 8)) y)))))) - (let ((x2 Reg (put_in_reg x)) - (y2 Reg (put_in_reg y)) - (lo Reg (pmullw x2 (RegMem.Reg y2))) - (hi Reg (pmulhw x2 (RegMem.Reg y2)))) - (value_reg (punpckhwd lo (RegMem.Reg hi))))) + (let ((x2 Xmm (put_in_xmm x)) + (y2 Xmm (put_in_xmm y)) + (lo Xmm (pmullw x2 (xmm_to_xmm_mem y2))) + (hi Xmm (pmulhw x2 (xmm_to_xmm_mem y2)))) + (value_xmm (punpckhwd lo (xmm_to_xmm_mem hi))))) ;; Special case for `i64x2.extmul_high_i32x4_s`. (rule (lower (has_type (multi_lane 64 2) @@ -1103,13 +1118,13 @@ x))) (def_inst (swiden_high (and (value_type (multi_lane 32 4)) y)))))) - (let ((x2 Reg (pshufd (put_in_reg_mem x) + (let ((x2 Xmm (pshufd (put_in_xmm_mem x) 0xFA (OperandSize.Size32))) - (y2 Reg (pshufd (put_in_reg_mem y) + (y2 Xmm (pshufd (put_in_xmm_mem y) 0xFA (OperandSize.Size32)))) - (value_reg (pmuldq x2 (RegMem.Reg y2))))) + (value_xmm (pmuldq x2 (xmm_to_xmm_mem y2))))) ;; Special case for `i16x8.extmul_low_i8x16_s`. (rule (lower (has_type (multi_lane 16 8) @@ -1117,9 +1132,9 @@ x))) (def_inst (swiden_low (and (value_type (multi_lane 8 16)) y)))))) - (let ((x2 Reg (pmovsxbw (put_in_reg_mem x))) - (y2 Reg (pmovsxbw (put_in_reg_mem y)))) - (value_reg (pmullw x2 (RegMem.Reg y2))))) + (let ((x2 Xmm (pmovsxbw (put_in_xmm_mem x))) + (y2 Xmm (pmovsxbw (put_in_xmm_mem y)))) + (value_xmm (pmullw x2 (xmm_to_xmm_mem y2))))) ;; Special case for `i32x4.extmul_low_i16x8_s`. (rule (lower (has_type (multi_lane 32 4) @@ -1127,11 +1142,11 @@ x))) (def_inst (swiden_low (and (value_type (multi_lane 16 8)) y)))))) - (let ((x2 Reg (put_in_reg x)) - (y2 Reg (put_in_reg y)) - (lo Reg (pmullw x2 (RegMem.Reg y2))) - (hi Reg (pmulhw x2 (RegMem.Reg y2)))) - (value_reg (punpcklwd lo (RegMem.Reg hi))))) + (let ((x2 Xmm (put_in_xmm x)) + (y2 Xmm (put_in_xmm y)) + (lo Xmm (pmullw x2 (xmm_to_xmm_mem y2))) + (hi Xmm (pmulhw x2 (xmm_to_xmm_mem y2)))) + (value_xmm (punpcklwd lo (xmm_to_xmm_mem hi))))) ;; Special case for `i64x2.extmul_low_i32x4_s`. (rule (lower (has_type (multi_lane 64 2) @@ -1139,13 +1154,13 @@ x))) (def_inst (swiden_low (and (value_type (multi_lane 32 4)) y)))))) - (let ((x2 Reg (pshufd (put_in_reg_mem x) + (let ((x2 Xmm (pshufd (put_in_xmm_mem x) 0x50 (OperandSize.Size32))) - (y2 Reg (pshufd (put_in_reg_mem y) + (y2 Xmm (pshufd (put_in_xmm_mem y) 0x50 (OperandSize.Size32)))) - (value_reg (pmuldq x2 (RegMem.Reg y2))))) + (value_xmm (pmuldq x2 (xmm_to_xmm_mem y2))))) ;; Special case for `i16x8.extmul_high_i8x16_u`. (rule (lower (has_type (multi_lane 16 8) @@ -1153,13 +1168,13 @@ x))) (def_inst (uwiden_high (and (value_type (multi_lane 8 16)) y)))))) - (let ((x1 Reg (put_in_reg x)) - (x2 Reg (palignr x1 (RegMem.Reg x1) 8 (OperandSize.Size32))) - (x3 Reg (pmovzxbw (RegMem.Reg x2))) - (y1 Reg (put_in_reg y)) - (y2 Reg (palignr y1 (RegMem.Reg y1) 8 (OperandSize.Size32))) - (y3 Reg (pmovzxbw (RegMem.Reg y2)))) - (value_reg (pmullw x3 (RegMem.Reg y3))))) + (let ((x1 Xmm (put_in_xmm x)) + (x2 Xmm (palignr x1 (xmm_to_xmm_mem x1) 8 (OperandSize.Size32))) + (x3 Xmm (pmovzxbw (xmm_to_xmm_mem x2))) + (y1 Xmm (put_in_xmm y)) + (y2 Xmm (palignr y1 (xmm_to_xmm_mem y1) 8 (OperandSize.Size32))) + (y3 Xmm (pmovzxbw (xmm_to_xmm_mem y2)))) + (value_xmm (pmullw x3 (xmm_to_xmm_mem y3))))) ;; Special case for `i32x4.extmul_high_i16x8_u`. (rule (lower (has_type (multi_lane 32 4) @@ -1167,11 +1182,11 @@ x))) (def_inst (uwiden_high (and (value_type (multi_lane 16 8)) y)))))) - (let ((x2 Reg (put_in_reg x)) - (y2 Reg (put_in_reg y)) - (lo Reg (pmullw x2 (RegMem.Reg y2))) - (hi Reg (pmulhuw x2 (RegMem.Reg y2)))) - (value_reg (punpckhwd lo (RegMem.Reg hi))))) + (let ((x2 Xmm (put_in_xmm x)) + (y2 Xmm (put_in_xmm y)) + (lo Xmm (pmullw x2 (xmm_to_xmm_mem y2))) + (hi Xmm (pmulhuw x2 (xmm_to_xmm_mem y2)))) + (value_xmm (punpckhwd lo (xmm_to_xmm_mem hi))))) ;; Special case for `i64x2.extmul_high_i32x4_u`. (rule (lower (has_type (multi_lane 64 2) @@ -1179,13 +1194,13 @@ x))) (def_inst (uwiden_high (and (value_type (multi_lane 32 4)) y)))))) - (let ((x2 Reg (pshufd (put_in_reg_mem x) + (let ((x2 Xmm (pshufd (put_in_xmm_mem x) 0xFA (OperandSize.Size32))) - (y2 Reg (pshufd (put_in_reg_mem y) + (y2 Xmm (pshufd (put_in_xmm_mem y) 0xFA (OperandSize.Size32)))) - (value_reg (pmuludq x2 (RegMem.Reg y2))))) + (value_xmm (pmuludq x2 (xmm_to_xmm_mem y2))))) ;; Special case for `i16x8.extmul_low_i8x16_u`. (rule (lower (has_type (multi_lane 16 8) @@ -1193,9 +1208,9 @@ x))) (def_inst (uwiden_low (and (value_type (multi_lane 8 16)) y)))))) - (let ((x2 Reg (pmovzxbw (put_in_reg_mem x))) - (y2 Reg (pmovzxbw (put_in_reg_mem y)))) - (value_reg (pmullw x2 (RegMem.Reg y2))))) + (let ((x2 Xmm (pmovzxbw (put_in_xmm_mem x))) + (y2 Xmm (pmovzxbw (put_in_xmm_mem y)))) + (value_xmm (pmullw x2 (xmm_to_xmm_mem y2))))) ;; Special case for `i32x4.extmul_low_i16x8_u`. (rule (lower (has_type (multi_lane 32 4) @@ -1203,11 +1218,11 @@ x))) (def_inst (uwiden_low (and (value_type (multi_lane 16 8)) y)))))) - (let ((x2 Reg (put_in_reg x)) - (y2 Reg (put_in_reg y)) - (lo Reg (pmullw x2 (RegMem.Reg y2))) - (hi Reg (pmulhuw x2 (RegMem.Reg y2)))) - (value_reg (punpcklwd lo (RegMem.Reg hi))))) + (let ((x2 Xmm (put_in_xmm x)) + (y2 Xmm (put_in_xmm y)) + (lo Xmm (pmullw x2 (xmm_to_xmm_mem y2))) + (hi Xmm (pmulhuw x2 (xmm_to_xmm_mem y2)))) + (value_xmm (punpcklwd lo (xmm_to_xmm_mem hi))))) ;; Special case for `i64x2.extmul_low_i32x4_u`. (rule (lower (has_type (multi_lane 64 2) @@ -1215,17 +1230,17 @@ x))) (def_inst (uwiden_low (and (value_type (multi_lane 32 4)) y)))))) - (let ((x2 Reg (pshufd (put_in_reg_mem x) + (let ((x2 Xmm (pshufd (put_in_xmm_mem x) 0x50 (OperandSize.Size32))) - (y2 Reg (pshufd (put_in_reg_mem y) + (y2 Xmm (pshufd (put_in_xmm_mem y) 0x50 (OperandSize.Size32)))) - (value_reg (pmuludq x2 (RegMem.Reg y2))))) + (value_xmm (pmuludq x2 (xmm_to_xmm_mem y2))))) ;;;; Rules for `band_not` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -(decl sse_and_not (Type Reg RegMem) Reg) +(decl sse_and_not (Type Xmm XmmMem) Xmm) (rule (sse_and_not $F32X4 x y) (andnps x y)) (rule (sse_and_not $F64X2 x y) (andnpd x y)) (rule (sse_and_not (multi_lane _bits _lanes) x y) (pandn x y)) @@ -1238,64 +1253,66 @@ ;; ;; pandn(x, y) = and(not(x), y) (rule (lower (has_type ty (band_not x y))) - (value_reg (sse_and_not ty - (put_in_reg y) - (put_in_reg_mem x)))) + (value_xmm (sse_and_not ty + (put_in_xmm y) + (put_in_xmm_mem x)))) ;;;; Rules for `iabs` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (rule (lower (has_type $I8X16 (iabs x))) - (value_reg (pabsb (put_in_reg_mem x)))) + (value_xmm (pabsb (put_in_xmm_mem x)))) (rule (lower (has_type $I16X8 (iabs x))) - (value_reg (pabsw (put_in_reg_mem x)))) + (value_xmm (pabsw (put_in_xmm_mem x)))) (rule (lower (has_type $I32X4 (iabs x))) - (value_reg (pabsd (put_in_reg_mem x)))) + (value_xmm (pabsd (put_in_xmm_mem x)))) ;; When AVX512 is available, we can use a single `vpabsq` instruction. (rule (lower (has_type (and (avx512vl_enabled) (avx512f_enabled) $I64X2) (iabs x))) - (value_reg (vpabsq (put_in_reg_mem x)))) + (value_xmm (vpabsq (put_in_xmm_mem x)))) -;; Otherwise, we use a separate register, `neg`, to contain the results of `0 - +;; Otherwise, we use a separate xmmister, `neg`, to contain the results of `0 - ;; x` and then blend in those results with `blendvpd` if the MSB of `neg` was ;; set to 1 (i.e. if `neg` was negative or, conversely, if `x` was originally ;; positive). (rule (lower (has_type $I64X2 (iabs x))) - (let ((rx Reg (put_in_reg x)) - (neg Reg (psubq (imm $I64X2 0) (RegMem.Reg rx)))) - (value_reg (blendvpd neg (RegMem.Reg rx) neg)))) + (let ((rx Xmm (put_in_xmm x)) + (neg Xmm (psubq (xmm_new (imm $I64X2 0)) (xmm_to_xmm_mem rx)))) + (value_xmm (blendvpd neg (xmm_to_xmm_mem rx) neg)))) ;;;; Rules for `fabs` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; Special case for `f32x4.abs`. (rule (lower (has_type $F32X4 (fabs x))) - (value_reg (andps (put_in_reg x) - (RegMem.Reg (psrld (vector_all_ones $F32X4) (RegMemImm.Imm 1)))))) + (value_xmm (andps (put_in_xmm x) + (xmm_to_xmm_mem (psrld (vector_all_ones $F32X4) + (xmm_mem_imm_new (RegMemImm.Imm 1))))))) ;; Special case for `f64x2.abs`. (rule (lower (has_type $F64X2 (fabs x))) - (value_reg (andpd (put_in_reg x) - (RegMem.Reg (psrlq (vector_all_ones $F64X2) (RegMemImm.Imm 1)))))) + (value_xmm (andpd (put_in_xmm x) + (xmm_to_xmm_mem (psrlq (vector_all_ones $F64X2) + (xmm_mem_imm_new (RegMemImm.Imm 1))))))) ;;;; Rules for `bnot` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; `i64` and smaller. (rule (lower (has_type (fits_in_64 ty) (bnot x))) - (value_reg (not ty (put_in_reg x)))) + (value_gpr (not ty (put_in_gpr x)))) ;; `i128`. (decl i128_not (Value) ValueRegs) (rule (i128_not x) (let ((x_regs ValueRegs (put_in_regs x)) - (x_lo Reg (value_regs_get x_regs 0)) - (x_hi Reg (value_regs_get x_regs 1))) - (value_regs (not $I64 x_lo) + (x_lo Gpr (gpr_new (value_regs_get x_regs 0))) + (x_hi Gpr (gpr_new (value_regs_get x_regs 1)))) + (value_gprs (not $I64 x_lo) (not $I64 x_hi)))) (rule (lower (has_type $I128 (bnot x))) @@ -1307,7 +1324,7 @@ ;; Special case for vector-types where bit-negation is an xor against an ;; all-one value (rule (lower (has_type ty @ (multi_lane _bits _lanes) (bnot x))) - (value_reg (sse_xor ty (put_in_reg x) (RegMem.Reg (vector_all_ones ty))))) + (value_xmm (sse_xor ty (put_in_xmm x) (xmm_to_xmm_mem (vector_all_ones ty))))) ;;;; Rules for `bitselect` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -1318,48 +1335,53 @@ ;; a = and if_true, condition ;; b = and_not condition, if_false ;; or b, a - (let ((cond_reg Reg (put_in_reg condition)) - (a Reg (sse_and ty (put_in_reg if_true) (RegMem.Reg cond_reg))) - (b Reg (sse_and_not ty cond_reg (put_in_reg_mem if_false)))) - (value_reg (sse_or ty b (RegMem.Reg a))))) + (let ((cond_xmm Xmm (put_in_xmm condition)) + (a Xmm (sse_and ty (put_in_xmm if_true) (xmm_to_xmm_mem cond_xmm))) + (b Xmm (sse_and_not ty cond_xmm (put_in_xmm_mem if_false)))) + (value_xmm (sse_or ty b (xmm_to_xmm_mem a))))) ;;;; Rules for `vselect` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (rule (lower (has_type ty @ (multi_lane _bits _lanes) (vselect condition if_true if_false))) - (value_reg (sse_blend ty - (put_in_reg_mem condition) - (put_in_reg_mem if_true) - (put_in_reg if_false)))) + (value_xmm (sse_blend ty + (put_in_xmm_mem condition) + (put_in_xmm_mem if_true) + (put_in_xmm if_false)))) ;;;; Rules for `insertlane` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (rule (lower (insertlane vec @ (value_type ty) val (u8_from_uimm8 idx))) - (value_reg (vec_insert_lane ty (put_in_reg vec) (put_in_reg_mem val) idx))) + (value_xmm (vec_insert_lane ty (put_in_xmm vec) (put_in_reg_mem val) idx))) ;; Helper function used below for `insertlane` but also here for other ;; lowerings. ;; ;; Note that the `Type` used here is the type of vector the insertion is ;; happening into, or the type of the first `Reg` argument. -(decl vec_insert_lane (Type Reg RegMem u8) Reg) +(decl vec_insert_lane (Type Xmm RegMem u8) Xmm) ;; i8x16.replace_lane -(rule (vec_insert_lane $I8X16 vec val idx) (pinsrb vec val idx)) +(rule (vec_insert_lane $I8X16 vec val idx) + (pinsrb vec (gpr_mem_new val) idx)) ;; i16x8.replace_lane -(rule (vec_insert_lane $I16X8 vec val idx) (pinsrw vec val idx)) +(rule (vec_insert_lane $I16X8 vec val idx) + (pinsrw vec (gpr_mem_new val) idx)) ;; i32x4.replace_lane -(rule (vec_insert_lane $I32X4 vec val idx) (pinsrd vec val idx (OperandSize.Size32))) +(rule (vec_insert_lane $I32X4 vec val idx) + (pinsrd vec (gpr_mem_new val) idx (OperandSize.Size32))) ;; i64x2.replace_lane -(rule (vec_insert_lane $I64X2 vec val idx) (pinsrd vec val idx (OperandSize.Size64))) +(rule (vec_insert_lane $I64X2 vec val idx) + (pinsrd vec (gpr_mem_new val) idx (OperandSize.Size64))) ;; f32x4.replace_lane -(rule (vec_insert_lane $F32X4 vec val idx) (insertps vec val (sse_insertps_lane_imm idx))) +(rule (vec_insert_lane $F32X4 vec val idx) + (insertps vec (xmm_mem_new val) (sse_insertps_lane_imm idx))) -;; external rust code used to calculate the immediate value to `insertps` +;; External rust code used to calculate the immediate value to `insertps`. (decl sse_insertps_lane_imm (u8) u8) (extern constructor sse_insertps_lane_imm sse_insertps_lane_imm) @@ -1378,60 +1400,63 @@ ;; load from memory into a temp register and then the second `movsd` (modeled ;; internally as `xmm_rm_r` will merge the temp register into our `vec` ;; register. -(rule (vec_insert_lane $F64X2 vec (RegMem.Reg val) 0) (movsd vec (RegMem.Reg val))) +(rule (vec_insert_lane $F64X2 vec (RegMem.Reg val) 0) + (movsd vec (xmm_mem_new (RegMem.Reg val)))) (rule (vec_insert_lane $F64X2 vec mem 0) - (movsd vec (RegMem.Reg (xmm_unary_rm_r (SseOpcode.Movsd) mem)))) + (movsd vec (xmm_to_xmm_mem (xmm_unary_rm_r (SseOpcode.Movsd) + (xmm_mem_new mem))))) ;; f64x2.replace_lane 1 ;; ;; Here the `movlhps` instruction is used specifically to specialize moving ;; into the second lane where unlike above cases we're not using the lane ;; immediate as an immediate to the instruction itself. -(rule (vec_insert_lane $F64X2 vec val 1) (movlhps vec val)) +(rule (vec_insert_lane $F64X2 vec val 1) + (movlhps vec (xmm_mem_new val))) ;;;; Rules for `imax` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (rule (lower (has_type $I8X16 (imax x y))) - (value_reg (pmaxsb (put_in_reg x) (put_in_reg_mem y)))) + (value_xmm (pmaxsb (put_in_xmm x) (put_in_xmm_mem y)))) (rule (lower (has_type $I16X8 (imax x y))) - (value_reg (pmaxsw (put_in_reg x) (put_in_reg_mem y)))) + (value_xmm (pmaxsw (put_in_xmm x) (put_in_xmm_mem y)))) (rule (lower (has_type $I32X4 (imax x y))) - (value_reg (pmaxsd (put_in_reg x) (put_in_reg_mem y)))) + (value_xmm (pmaxsd (put_in_xmm x) (put_in_xmm_mem y)))) ;;;; Rules for `imin` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (rule (lower (has_type $I8X16 (imin x y))) - (value_reg (pminsb (put_in_reg x) (put_in_reg_mem y)))) + (value_xmm (pminsb (put_in_xmm x) (put_in_xmm_mem y)))) (rule (lower (has_type $I16X8 (imin x y))) - (value_reg (pminsw (put_in_reg x) (put_in_reg_mem y)))) + (value_xmm (pminsw (put_in_xmm x) (put_in_xmm_mem y)))) (rule (lower (has_type $I32X4 (imin x y))) - (value_reg (pminsd (put_in_reg x) (put_in_reg_mem y)))) + (value_xmm (pminsd (put_in_xmm x) (put_in_xmm_mem y)))) ;;;; Rules for `umax` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (rule (lower (has_type $I8X16 (umax x y))) - (value_reg (pmaxub (put_in_reg x) (put_in_reg_mem y)))) + (value_xmm (pmaxub (put_in_xmm x) (put_in_xmm_mem y)))) (rule (lower (has_type $I16X8 (umax x y))) - (value_reg (pmaxuw (put_in_reg x) (put_in_reg_mem y)))) + (value_xmm (pmaxuw (put_in_xmm x) (put_in_xmm_mem y)))) (rule (lower (has_type $I32X4 (umax x y))) - (value_reg (pmaxud (put_in_reg x) (put_in_reg_mem y)))) + (value_xmm (pmaxud (put_in_xmm x) (put_in_xmm_mem y)))) ;;;; Rules for `umin` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (rule (lower (has_type $I8X16 (umin x y))) - (value_reg (pminub (put_in_reg x) (put_in_reg_mem y)))) + (value_xmm (pminub (put_in_xmm x) (put_in_xmm_mem y)))) (rule (lower (has_type $I16X8 (umin x y))) - (value_reg (pminuw (put_in_reg x) (put_in_reg_mem y)))) + (value_xmm (pminuw (put_in_xmm x) (put_in_xmm_mem y)))) (rule (lower (has_type $I32X4 (umin x y))) - (value_reg (pminud (put_in_reg x) (put_in_reg_mem y)))) + (value_xmm (pminud (put_in_xmm x) (put_in_xmm_mem y)))) ;;;; Rules for `trap` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; diff --git a/cranelift/codegen/src/isa/x64/lower.rs b/cranelift/codegen/src/isa/x64/lower.rs index d5503416de9f..3e74dddda93d 100644 --- a/cranelift/codegen/src/isa/x64/lower.rs +++ b/cranelift/codegen/src/isa/x64/lower.rs @@ -1,7 +1,7 @@ //! Lowering rules for X64. // ISLE integration glue. -mod isle; +pub(super) mod isle; use crate::data_value::DataValue; use crate::ir::{ @@ -1057,7 +1057,13 @@ fn lower_to_amode>(ctx: &mut C, spec: InsnInput, offset: i ) }; - return Amode::imm_reg_reg_shift(offset as u32, base, index, shift).with_flags(flags); + return Amode::imm_reg_reg_shift( + offset as u32, + Gpr::new(base).unwrap(), + Gpr::new(index).unwrap(), + shift, + ) + .with_flags(flags); } let input = put_input_in_reg(ctx, spec); @@ -3950,7 +3956,13 @@ fn lower_insn_to_regs>( let index = put_input_in_reg(ctx, inputs[1]); let shift = 0; let flags = ctx.memflags(insn).expect("load should have memflags"); - Amode::imm_reg_reg_shift(offset as u32, base, index, shift).with_flags(flags) + Amode::imm_reg_reg_shift( + offset as u32, + Gpr::new(base).unwrap(), + Gpr::new(index).unwrap(), + shift, + ) + .with_flags(flags) } _ => unreachable!(), }; @@ -4054,7 +4066,13 @@ fn lower_insn_to_regs>( let index = put_input_in_reg(ctx, inputs[2]); let shift = 0; let flags = ctx.memflags(insn).expect("store should have memflags"); - Amode::imm_reg_reg_shift(offset as u32, base, index, shift).with_flags(flags) + Amode::imm_reg_reg_shift( + offset as u32, + Gpr::new(base).unwrap(), + Gpr::new(index).unwrap(), + shift, + ) + .with_flags(flags) } _ => unreachable!(), diff --git a/cranelift/codegen/src/isa/x64/lower/isle.rs b/cranelift/codegen/src/isa/x64/lower/isle.rs index c6bd7651b413..3aff4de79037 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle.rs +++ b/cranelift/codegen/src/isa/x64/lower/isle.rs @@ -1,26 +1,28 @@ //! ISLE integration glue code for x64 lowering. // Pull in the ISLE generated code. -mod generated_code; +pub(crate) mod generated_code; +use generated_code::MInst; +use regalloc::Writable; // Types that the generated ISLE code uses via `use super::*`. -use super::{ - is_mergeable_load, lower_to_amode, AluRmiROpcode, Inst as MInst, OperandSize, Reg, RegMemImm, -}; +use super::{is_mergeable_load, lower_to_amode, Reg}; use crate::{ - ir::{immediates::*, types::*, Inst, InstructionData, Opcode, TrapCode, Value, ValueList}, - isa::x64::{ - inst::{ - args::{ - Amode, Avx512Opcode, CmpOpcode, ExtKind, ExtMode, FcmpImm, Imm8Reg, RegMem, - ShiftKind, SseOpcode, SyntheticAmode, CC, - }, - regs, x64_map_regs, + ir::{ + immediates::*, types::*, Inst, InstructionData, Opcode, TrapCode, Value, ValueLabel, + ValueList, + }, + isa::{ + settings::Flags, + unwind::UnwindInst, + x64::{ + inst::{args::*, regs, x64_map_regs}, + settings::Flags as IsaFlags, }, - settings::Flags as IsaFlags, }, - machinst::{isle::*, InsnInput, InsnOutput, LowerCtx, VCodeConstantData}, - settings::Flags, + machinst::{ + isle::*, AtomicRmwOp, InsnInput, InsnOutput, LowerCtx, VCodeConstant, VCodeConstantData, + }, }; use std::convert::TryFrom; @@ -248,8 +250,8 @@ where } #[inline] - fn xmm0(&mut self) -> WritableReg { - WritableReg::from_reg(regs::xmm0()) + fn xmm0(&mut self) -> WritableXmm { + WritableXmm::from_reg(Xmm::new(regs::xmm0()).unwrap()) } #[inline] @@ -258,7 +260,7 @@ where } #[inline] - fn amode_imm_reg_reg_shift(&mut self, simm32: u32, base: Reg, index: Reg, shift: u8) -> Amode { + fn amode_imm_reg_reg_shift(&mut self, simm32: u32, base: Gpr, index: Gpr, shift: u8) -> Amode { Amode::imm_reg_reg_shift(simm32, base, index, shift) } @@ -267,6 +269,16 @@ where amode.clone().into() } + #[inline] + fn writable_gpr_to_reg(&mut self, r: WritableGpr) -> WritableReg { + r.to_writable_reg() + } + + #[inline] + fn writable_xmm_to_reg(&mut self, r: WritableXmm) -> WritableReg { + r.to_writable_reg() + } + fn ishl_i8x16_mask_for_const(&mut self, amt: u32) -> SyntheticAmode { // When the shift amount is known, we can statically (i.e. at compile // time) determine the mask to use and only emit that. @@ -302,6 +314,96 @@ where .use_constant(VCodeConstantData::WellKnown(&I8X16_USHR_MASKS)); SyntheticAmode::ConstantOffset(mask_table) } + + #[inline] + fn writable_reg_to_xmm(&mut self, r: WritableReg) -> WritableXmm { + Writable::from_reg(Xmm::new(r.to_reg()).unwrap()) + } + + #[inline] + fn writable_xmm_to_xmm(&mut self, r: WritableXmm) -> Xmm { + r.to_reg() + } + + #[inline] + fn writable_gpr_to_gpr(&mut self, r: WritableGpr) -> Gpr { + r.to_reg() + } + + #[inline] + fn gpr_to_reg(&mut self, r: Gpr) -> Reg { + r.into() + } + + #[inline] + fn xmm_to_reg(&mut self, r: Xmm) -> Reg { + r.into() + } + + #[inline] + fn xmm_to_xmm_mem_imm(&mut self, r: Xmm) -> XmmMemImm { + r.into() + } + + #[inline] + fn temp_writable_gpr(&mut self) -> WritableGpr { + Writable::from_reg(Gpr::new(self.temp_writable_reg(I64).to_reg()).unwrap()) + } + + #[inline] + fn temp_writable_xmm(&mut self) -> WritableXmm { + Writable::from_reg(Xmm::new(self.temp_writable_reg(I8X16).to_reg()).unwrap()) + } + + #[inline] + fn xmm_mem_new(&mut self, rm: &RegMem) -> XmmMem { + XmmMem::new(rm.clone()).unwrap() + } + + #[inline] + fn gpr_mem_imm_new(&mut self, rmi: &RegMemImm) -> GprMemImm { + GprMemImm::new(rmi.clone()).unwrap() + } + + #[inline] + fn xmm_mem_imm_new(&mut self, rmi: &RegMemImm) -> XmmMemImm { + XmmMemImm::new(rmi.clone()).unwrap() + } + + #[inline] + fn xmm_to_xmm_mem(&mut self, r: Xmm) -> XmmMem { + r.into() + } + + #[inline] + fn xmm_mem_to_reg_mem(&mut self, xm: &XmmMem) -> RegMem { + xm.clone().into() + } + + #[inline] + fn gpr_mem_to_reg_mem(&mut self, gm: &GprMem) -> RegMem { + gm.clone().into() + } + + #[inline] + fn xmm_new(&mut self, r: Reg) -> Xmm { + Xmm::new(r).unwrap() + } + + #[inline] + fn gpr_new(&mut self, r: Reg) -> Gpr { + Gpr::new(r).unwrap() + } + + #[inline] + fn gpr_mem_new(&mut self, rm: &RegMem) -> GprMem { + GprMem::new(rm.clone()).unwrap() + } + + #[inline] + fn reg_to_gpr_mem(&mut self, r: Reg) -> GprMem { + GprMem::new(RegMem::reg(r)).unwrap() + } } // Since x64 doesn't have 8x16 shifts and we must use a 16x8 shift instead, we diff --git a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest index 6d2b8256ac8d..a1e5d89a632c 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest +++ b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest @@ -1,4 +1,4 @@ src/clif.isle 9ea75a6f790b5c03 -src/prelude.isle 51d2aef2566c1c96 -src/isa/x64/inst.isle 61004acbb1289816 -src/isa/x64/lower.isle 82db7f7d47ac7809 +src/prelude.isle 3d00cfa50c4d52d9 +src/isa/x64/inst.isle 1e5ee6f51e890a17 +src/isa/x64/lower.isle 144c33c4e64a17a7 diff --git a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs index 9c2ad6974087..64f7149ef56f 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs +++ b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs @@ -68,11 +68,31 @@ pub trait Context { fn put_in_reg_mem_imm(&mut self, arg0: Value) -> RegMemImm; fn put_in_reg_mem(&mut self, arg0: Value) -> RegMem; fn synthetic_amode_to_reg_mem(&mut self, arg0: &SyntheticAmode) -> RegMem; - fn amode_imm_reg_reg_shift(&mut self, arg0: u32, arg1: Reg, arg2: Reg, arg3: u8) -> Amode; + fn amode_imm_reg_reg_shift(&mut self, arg0: u32, arg1: Gpr, arg2: Gpr, arg3: u8) -> Amode; fn amode_to_synthetic_amode(&mut self, arg0: &Amode) -> SyntheticAmode; fn put_masked_in_imm8_reg(&mut self, arg0: Value, arg1: Type) -> Imm8Reg; fn encode_fcmp_imm(&mut self, arg0: &FcmpImm) -> u8; - fn xmm0(&mut self) -> WritableReg; + fn writable_gpr_to_reg(&mut self, arg0: WritableGpr) -> WritableReg; + fn writable_xmm_to_reg(&mut self, arg0: WritableXmm) -> WritableReg; + fn writable_reg_to_xmm(&mut self, arg0: WritableReg) -> WritableXmm; + fn writable_xmm_to_xmm(&mut self, arg0: WritableXmm) -> Xmm; + fn writable_gpr_to_gpr(&mut self, arg0: WritableGpr) -> Gpr; + fn gpr_to_reg(&mut self, arg0: Gpr) -> Reg; + fn xmm_to_reg(&mut self, arg0: Xmm) -> Reg; + fn xmm_to_xmm_mem_imm(&mut self, arg0: Xmm) -> XmmMemImm; + fn temp_writable_gpr(&mut self) -> WritableGpr; + fn temp_writable_xmm(&mut self) -> WritableXmm; + fn xmm_mem_new(&mut self, arg0: &RegMem) -> XmmMem; + fn gpr_mem_imm_new(&mut self, arg0: &RegMemImm) -> GprMemImm; + fn xmm_mem_imm_new(&mut self, arg0: &RegMemImm) -> XmmMemImm; + fn xmm_to_xmm_mem(&mut self, arg0: Xmm) -> XmmMem; + fn xmm_mem_to_reg_mem(&mut self, arg0: &XmmMem) -> RegMem; + fn gpr_mem_to_reg_mem(&mut self, arg0: &GprMem) -> RegMem; + fn xmm_new(&mut self, arg0: Reg) -> Xmm; + fn gpr_new(&mut self, arg0: Reg) -> Gpr; + fn gpr_mem_new(&mut self, arg0: &RegMem) -> GprMem; + fn reg_to_gpr_mem(&mut self, arg0: Reg) -> GprMem; + fn xmm0(&mut self) -> WritableXmm; fn avx512vl_enabled(&mut self, arg0: Type) -> Option<()>; fn avx512dq_enabled(&mut self, arg0: Type) -> Option<()>; fn avx512f_enabled(&mut self, arg0: Type) -> Option<()>; @@ -92,26 +112,337 @@ pub trait Context { fn sse_insertps_lane_imm(&mut self, arg0: u8) -> u8; } -/// Internal type SideEffectNoResult: defined at src/prelude.isle line 282. +/// Internal type SideEffectNoResult: defined at src/prelude.isle line 294. #[derive(Clone, Debug)] pub enum SideEffectNoResult { Inst { inst: MInst }, } -/// Internal type ProducesFlags: defined at src/prelude.isle line 295. +/// Internal type ProducesFlags: defined at src/prelude.isle line 307. #[derive(Clone, Debug)] pub enum ProducesFlags { ProducesFlags { inst: MInst, result: Reg }, } -/// Internal type ConsumesFlags: defined at src/prelude.isle line 298. +/// Internal type ConsumesFlags: defined at src/prelude.isle line 310. #[derive(Clone, Debug)] pub enum ConsumesFlags { ConsumesFlags { inst: MInst, result: Reg }, } -/// Internal type ExtendKind: defined at src/isa/x64/inst.isle line 464. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type MInst: defined at src/isa/x64/inst.isle line 8. +#[derive(Clone)] +pub enum MInst { + Nop { + len: u8, + }, + AluRmiR { + size: OperandSize, + op: AluRmiROpcode, + src1: Reg, + src2: RegMemImm, + dst: WritableReg, + }, + UnaryRmR { + size: OperandSize, + op: UnaryRmROpcode, + src: GprMem, + dst: WritableGpr, + }, + Not { + size: OperandSize, + src: Gpr, + dst: WritableGpr, + }, + Neg { + size: OperandSize, + src: Gpr, + dst: WritableGpr, + }, + Div { + size: OperandSize, + signed: bool, + divisor: RegMem, + dividend: Reg, + dst_quotient: WritableReg, + dst_remainder: WritableReg, + }, + MulHi { + size: OperandSize, + signed: bool, + src1: Reg, + src2: RegMem, + dst_lo: WritableReg, + dst_hi: WritableReg, + }, + CheckedDivOrRemSeq { + kind: DivOrRemKind, + size: OperandSize, + dividend: Reg, + divisor: WritableReg, + dst_quotient: WritableReg, + dst_remainder: WritableReg, + tmp: OptionWritableReg, + }, + SignExtendData { + size: OperandSize, + src: Reg, + dst: WritableReg, + }, + Imm { + dst_size: OperandSize, + simm64: u64, + dst: WritableReg, + }, + MovRR { + size: OperandSize, + src: Reg, + dst: WritableReg, + }, + MovzxRmR { + ext_mode: ExtMode, + src: RegMem, + dst: WritableReg, + }, + Mov64MR { + src: SyntheticAmode, + dst: WritableReg, + }, + LoadEffectiveAddress { + addr: SyntheticAmode, + dst: WritableGpr, + }, + MovsxRmR { + ext_mode: ExtMode, + src: RegMem, + dst: WritableReg, + }, + MovRM { + size: OperandSize, + src: Reg, + dst: SyntheticAmode, + }, + ShiftR { + size: OperandSize, + kind: ShiftKind, + src: Reg, + num_bits: Imm8Reg, + dst: WritableReg, + }, + XmmRmiReg { + opcode: SseOpcode, + src1: Xmm, + src2: XmmMemImm, + dst: WritableXmm, + }, + CmpRmiR { + size: OperandSize, + opcode: CmpOpcode, + src: RegMemImm, + dst: Reg, + }, + Setcc { + cc: CC, + dst: WritableReg, + }, + Cmove { + size: OperandSize, + cc: CC, + consequent: RegMem, + alternative: Reg, + dst: WritableReg, + }, + Push64 { + src: RegMemImm, + }, + Pop64 { + dst: WritableReg, + }, + XmmRmR { + op: SseOpcode, + src1: Xmm, + src2: XmmMem, + dst: WritableXmm, + }, + XmmRmREvex { + op: Avx512Opcode, + src1: XmmMem, + src2: Xmm, + dst: WritableXmm, + }, + XmmUnaryRmR { + op: SseOpcode, + src: XmmMem, + dst: WritableXmm, + }, + XmmUnaryRmREvex { + op: Avx512Opcode, + src: XmmMem, + dst: WritableXmm, + }, + XmmMovRM { + op: SseOpcode, + src: Reg, + dst: SyntheticAmode, + }, + XmmLoadConst { + src: VCodeConstant, + dst: WritableReg, + ty: Type, + }, + XmmToGpr { + op: SseOpcode, + src: Xmm, + dst: WritableGpr, + dst_size: OperandSize, + }, + GprToXmm { + op: SseOpcode, + src: RegMem, + dst: WritableXmm, + src_size: OperandSize, + }, + CvtUint64ToFloatSeq { + dst_size: OperandSize, + src: WritableGpr, + dst: WritableXmm, + tmp_gpr1: WritableGpr, + tmp_gpr2: WritableGpr, + }, + CvtFloatToSintSeq { + dst_size: OperandSize, + src_size: OperandSize, + is_saturating: bool, + src: WritableXmm, + dst: WritableGpr, + tmp_gpr: WritableGpr, + tmp_xmm: WritableXmm, + }, + CvtFloatToUintSeq { + dst_size: OperandSize, + src_size: OperandSize, + is_saturating: bool, + src: WritableXmm, + dst: WritableGpr, + tmp_gpr: WritableGpr, + tmp_xmm: WritableXmm, + }, + XmmMinMaxSeq { + size: OperandSize, + is_min: bool, + lhs: Reg, + rhs_dst: WritableReg, + }, + XmmCmove { + size: OperandSize, + cc: CC, + src: RegMem, + dst: WritableReg, + }, + XmmCmpRmR { + op: SseOpcode, + src: RegMem, + dst: Reg, + }, + XmmRmRImm { + op: SseOpcode, + src1: Reg, + src2: RegMem, + dst: WritableReg, + imm: u8, + size: OperandSize, + }, + CallKnown { + dest: ExternalName, + uses: VecReg, + defs: VecWritableReg, + opcode: Opcode, + }, + CallUnknown { + dest: RegMem, + uses: VecReg, + defs: VecWritableReg, + opcode: Opcode, + }, + Ret, + EpiloguePlaceholder, + JmpKnown { + dst: MachLabel, + }, + JmpIf { + cc: CC, + taken: MachLabel, + }, + JmpCond { + cc: CC, + taken: MachLabel, + not_taken: MachLabel, + }, + JmpTableSeq { + idx: Reg, + tmp1: WritableReg, + tmp2: WritableReg, + default_target: MachLabel, + targets: VecMachLabel, + targets_for_term: VecMachLabel, + }, + JmpUnknown { + target: RegMem, + }, + TrapIf { + cc: CC, + trap_code: TrapCode, + }, + Hlt, + Ud2 { + trap_code: TrapCode, + }, + LoadExtName { + dst: WritableReg, + name: BoxExternalName, + offset: i64, + }, + LockCmpxchg { + ty: Type, + replacement: Reg, + expected: Reg, + mem: SyntheticAmode, + dst_old: WritableReg, + }, + AtomicRmwSeq { + ty: Type, + op: AtomicRmwOp, + address: Reg, + operand: Reg, + temp: WritableReg, + dst_old: WritableReg, + }, + Fence { + kind: FenceKind, + }, + VirtualSPOffsetAdj { + offset: i64, + }, + XmmUninitializedValue { + dst: WritableXmm, + }, + ElfTlsGetAddr { + symbol: ExternalName, + }, + MachOTlsGetAddr { + symbol: ExternalName, + }, + ValueLabelMarker { + reg: Reg, + label: ValueLabel, + }, + Unwind { + inst: UnwindInst, + }, +} + +/// Internal type ExtendKind: defined at src/isa/x64/inst.isle line 1051. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum ExtendKind { Sign, Zero, @@ -120,7 +451,7 @@ pub enum ExtendKind { // Generated as internal constructor for term temp_reg. pub fn constructor_temp_reg(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; - // Rule at src/prelude.isle line 70. + // Rule at src/prelude.isle line 73. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); return Some(expr1_0); @@ -129,7 +460,7 @@ pub fn constructor_temp_reg(ctx: &mut C, arg0: Type) -> Option // Generated as internal constructor for term lo_reg. pub fn constructor_lo_reg(ctx: &mut C, arg0: Value) -> Option { let pattern0_0 = arg0; - // Rule at src/prelude.isle line 105. + // Rule at src/prelude.isle line 108. let expr0_0 = C::put_in_regs(ctx, pattern0_0); let expr1_0: usize = 0; let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); @@ -146,7 +477,7 @@ pub fn constructor_value_regs_none( inst: ref pattern1_0, } = pattern0_0 { - // Rule at src/prelude.isle line 287. + // Rule at src/prelude.isle line 299. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::value_regs_invalid(ctx); return Some(expr1_0); @@ -172,7 +503,7 @@ pub fn constructor_with_flags( result: pattern3_1, } = pattern2_0 { - // Rule at src/prelude.isle line 308. + // Rule at src/prelude.isle line 320. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern3_0); let expr2_0 = C::value_regs(ctx, pattern1_1, pattern3_1); @@ -200,7 +531,7 @@ pub fn constructor_with_flags_1( result: pattern3_1, } = pattern2_0 { - // Rule at src/prelude.isle line 316. + // Rule at src/prelude.isle line 328. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern3_0); return Some(pattern3_1); @@ -234,7 +565,7 @@ pub fn constructor_with_flags_2( result: pattern5_1, } = pattern4_0 { - // Rule at src/prelude.isle line 326. + // Rule at src/prelude.isle line 338. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern5_0); let expr2_0 = C::emit(ctx, &pattern3_0); @@ -251,22 +582,22 @@ pub fn constructor_operand_size_bits(ctx: &mut C, arg0: &OperandSize let pattern0_0 = arg0; match pattern0_0 { &OperandSize::Size8 => { - // Rule at src/isa/x64/inst.isle line 98. + // Rule at src/isa/x64/inst.isle line 509. let expr0_0: u16 = 8; return Some(expr0_0); } &OperandSize::Size16 => { - // Rule at src/isa/x64/inst.isle line 99. + // Rule at src/isa/x64/inst.isle line 510. let expr0_0: u16 = 16; return Some(expr0_0); } &OperandSize::Size32 => { - // Rule at src/isa/x64/inst.isle line 100. + // Rule at src/isa/x64/inst.isle line 511. let expr0_0: u16 = 32; return Some(expr0_0); } &OperandSize::Size64 => { - // Rule at src/isa/x64/inst.isle line 101. + // Rule at src/isa/x64/inst.isle line 512. let expr0_0: u16 = 64; return Some(expr0_0); } @@ -275,6 +606,89 @@ pub fn constructor_operand_size_bits(ctx: &mut C, arg0: &OperandSize return None; } +// Generated as internal constructor for term put_in_gpr. +pub fn constructor_put_in_gpr(ctx: &mut C, arg0: Value) -> Option { + let pattern0_0 = arg0; + // Rule at src/isa/x64/inst.isle line 934. + let expr0_0 = C::put_in_reg(ctx, pattern0_0); + let expr1_0 = C::gpr_new(ctx, expr0_0); + return Some(expr1_0); +} + +// Generated as internal constructor for term put_in_gpr_mem. +pub fn constructor_put_in_gpr_mem(ctx: &mut C, arg0: Value) -> Option { + let pattern0_0 = arg0; + // Rule at src/isa/x64/inst.isle line 941. + let expr0_0 = C::put_in_reg_mem(ctx, pattern0_0); + let expr1_0 = C::gpr_mem_new(ctx, &expr0_0); + return Some(expr1_0); +} + +// Generated as internal constructor for term put_in_gpr_mem_imm. +pub fn constructor_put_in_gpr_mem_imm(ctx: &mut C, arg0: Value) -> Option { + let pattern0_0 = arg0; + // Rule at src/isa/x64/inst.isle line 948. + let expr0_0 = C::put_in_reg_mem_imm(ctx, pattern0_0); + let expr1_0 = C::gpr_mem_imm_new(ctx, &expr0_0); + return Some(expr1_0); +} + +// Generated as internal constructor for term put_in_xmm. +pub fn constructor_put_in_xmm(ctx: &mut C, arg0: Value) -> Option { + let pattern0_0 = arg0; + // Rule at src/isa/x64/inst.isle line 955. + let expr0_0 = C::put_in_reg(ctx, pattern0_0); + let expr1_0 = C::xmm_new(ctx, expr0_0); + return Some(expr1_0); +} + +// Generated as internal constructor for term put_in_xmm_mem. +pub fn constructor_put_in_xmm_mem(ctx: &mut C, arg0: Value) -> Option { + let pattern0_0 = arg0; + // Rule at src/isa/x64/inst.isle line 962. + let expr0_0 = C::put_in_reg_mem(ctx, pattern0_0); + let expr1_0 = C::xmm_mem_new(ctx, &expr0_0); + return Some(expr1_0); +} + +// Generated as internal constructor for term put_in_xmm_mem_imm. +pub fn constructor_put_in_xmm_mem_imm(ctx: &mut C, arg0: Value) -> Option { + let pattern0_0 = arg0; + // Rule at src/isa/x64/inst.isle line 969. + let expr0_0 = C::put_in_reg_mem_imm(ctx, pattern0_0); + let expr1_0 = C::xmm_mem_imm_new(ctx, &expr0_0); + return Some(expr1_0); +} + +// Generated as internal constructor for term value_gpr. +pub fn constructor_value_gpr(ctx: &mut C, arg0: Gpr) -> Option { + let pattern0_0 = arg0; + // Rule at src/isa/x64/inst.isle line 974. + let expr0_0 = C::gpr_to_reg(ctx, pattern0_0); + let expr1_0 = C::value_reg(ctx, expr0_0); + return Some(expr1_0); +} + +// Generated as internal constructor for term value_gprs. +pub fn constructor_value_gprs(ctx: &mut C, arg0: Gpr, arg1: Gpr) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 979. + let expr0_0 = C::gpr_to_reg(ctx, pattern0_0); + let expr1_0 = C::gpr_to_reg(ctx, pattern1_0); + let expr2_0 = C::value_regs(ctx, expr0_0, expr1_0); + return Some(expr2_0); +} + +// Generated as internal constructor for term value_xmm. +pub fn constructor_value_xmm(ctx: &mut C, arg0: Xmm) -> Option { + let pattern0_0 = arg0; + // Rule at src/isa/x64/inst.isle line 984. + let expr0_0 = C::xmm_to_reg(ctx, pattern0_0); + let expr1_0 = C::value_reg(ctx, expr0_0); + return Some(expr1_0); +} + // Generated as internal constructor for term extend_to_reg. pub fn constructor_extend_to_reg( ctx: &mut C, @@ -287,12 +701,12 @@ pub fn constructor_extend_to_reg( let pattern2_0 = arg1; if pattern2_0 == pattern1_0 { let pattern4_0 = arg2; - // Rule at src/isa/x64/inst.isle line 476. + // Rule at src/isa/x64/inst.isle line 1063. let expr0_0 = C::put_in_reg(ctx, pattern0_0); return Some(expr0_0); } let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 479. + // Rule at src/isa/x64/inst.isle line 1066. let expr0_0 = C::ty_bits_u16(ctx, pattern1_0); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern2_0); let expr2_0 = constructor_operand_size_bits(ctx, &expr1_0)?; @@ -316,7 +730,7 @@ pub fn constructor_extend( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/inst.isle line 499. + // Rule at src/isa/x64/inst.isle line 1086. let expr0_0 = constructor_movsx(ctx, pattern2_0, pattern3_0, pattern4_0)?; return Some(expr0_0); } @@ -324,7 +738,7 @@ pub fn constructor_extend( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/inst.isle line 495. + // Rule at src/isa/x64/inst.isle line 1082. let expr0_0 = constructor_movzx(ctx, pattern2_0, pattern3_0, pattern4_0)?; return Some(expr0_0); } @@ -337,17 +751,17 @@ pub fn constructor_extend( pub fn constructor_sse_xor_op(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32X4 { - // Rule at src/isa/x64/inst.isle line 506. + // Rule at src/isa/x64/inst.isle line 1093. let expr0_0 = SseOpcode::Xorps; return Some(expr0_0); } if pattern0_0 == F64X2 { - // Rule at src/isa/x64/inst.isle line 507. + // Rule at src/isa/x64/inst.isle line 1094. let expr0_0 = SseOpcode::Xorpd; return Some(expr0_0); } if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { - // Rule at src/isa/x64/inst.isle line 508. + // Rule at src/isa/x64/inst.isle line 1095. let expr0_0 = SseOpcode::Pxor; return Some(expr0_0); } @@ -358,13 +772,13 @@ pub fn constructor_sse_xor_op(ctx: &mut C, arg0: Type) -> Option( ctx: &mut C, arg0: Type, - arg1: Reg, - arg2: &RegMem, -) -> Option { + arg1: Xmm, + arg2: &XmmMem, +) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 512. + // Rule at src/isa/x64/inst.isle line 1099. let expr0_0 = constructor_sse_xor_op(ctx, pattern0_0)?; let expr1_0 = constructor_xmm_rm_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -374,40 +788,40 @@ pub fn constructor_sse_xor( pub fn constructor_sse_cmp_op(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32X4 { - // Rule at src/isa/x64/inst.isle line 521. + // Rule at src/isa/x64/inst.isle line 1108. let expr0_0 = SseOpcode::Cmpps; return Some(expr0_0); } if pattern0_0 == F64X2 { - // Rule at src/isa/x64/inst.isle line 522. + // Rule at src/isa/x64/inst.isle line 1109. let expr0_0 = SseOpcode::Cmppd; return Some(expr0_0); } if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { if pattern1_0 == 8 { if pattern1_1 == 16 { - // Rule at src/isa/x64/inst.isle line 517. + // Rule at src/isa/x64/inst.isle line 1104. let expr0_0 = SseOpcode::Pcmpeqb; return Some(expr0_0); } } if pattern1_0 == 16 { if pattern1_1 == 8 { - // Rule at src/isa/x64/inst.isle line 518. + // Rule at src/isa/x64/inst.isle line 1105. let expr0_0 = SseOpcode::Pcmpeqw; return Some(expr0_0); } } if pattern1_0 == 32 { if pattern1_1 == 4 { - // Rule at src/isa/x64/inst.isle line 519. + // Rule at src/isa/x64/inst.isle line 1106. let expr0_0 = SseOpcode::Pcmpeqd; return Some(expr0_0); } } if pattern1_0 == 64 { if pattern1_1 == 2 { - // Rule at src/isa/x64/inst.isle line 520. + // Rule at src/isa/x64/inst.isle line 1107. let expr0_0 = SseOpcode::Pcmpeqq; return Some(expr0_0); } @@ -417,14 +831,14 @@ pub fn constructor_sse_cmp_op(ctx: &mut C, arg0: Type) -> Option(ctx: &mut C, arg0: Type) -> Option { +pub fn constructor_vector_all_ones(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 536. - let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); - let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); + // Rule at src/isa/x64/inst.isle line 1123. + let expr0_0 = C::temp_writable_xmm(ctx); + let expr1_0 = C::writable_xmm_to_xmm(ctx, expr0_0); let expr2_0: Type = I32X4; let expr3_0 = constructor_sse_cmp_op(ctx, expr2_0)?; - let expr4_0 = RegMem::Reg { reg: expr1_0 }; + let expr4_0 = C::xmm_to_xmm_mem(ctx, expr1_0); let expr5_0 = MInst::XmmRmR { op: expr3_0, src1: expr1_0, @@ -438,70 +852,71 @@ pub fn constructor_vector_all_ones(ctx: &mut C, arg0: Type) -> Optio // Generated as internal constructor for term make_i64x2_from_lanes. pub fn constructor_make_i64x2_from_lanes( ctx: &mut C, - arg0: &RegMem, - arg1: &RegMem, -) -> Option { + arg0: &GprMem, + arg1: &GprMem, +) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 547. - let expr0_0: Type = I64X2; - let expr1_0 = C::temp_writable_reg(ctx, expr0_0); - let expr2_0 = C::writable_reg_to_reg(ctx, expr1_0); - let expr3_0 = MInst::XmmUninitializedValue { dst: expr1_0 }; - let expr4_0 = C::emit(ctx, &expr3_0); - let expr5_0 = SseOpcode::Pinsrd; - let expr6_0: u8 = 0; - let expr7_0 = OperandSize::Size64; - let expr8_0 = MInst::XmmRmRImm { - op: expr5_0, - src1: expr2_0, - src2: pattern0_0.clone(), + // Rule at src/isa/x64/inst.isle line 1134. + let expr0_0 = C::temp_writable_xmm(ctx); + let expr1_0 = C::writable_xmm_to_reg(ctx, expr0_0); + let expr2_0 = C::writable_xmm_to_xmm(ctx, expr0_0); + let expr3_0 = C::xmm_to_reg(ctx, expr2_0); + let expr4_0 = MInst::XmmUninitializedValue { dst: expr0_0 }; + let expr5_0 = C::emit(ctx, &expr4_0); + let expr6_0 = SseOpcode::Pinsrd; + let expr7_0 = C::gpr_mem_to_reg_mem(ctx, pattern0_0); + let expr8_0: u8 = 0; + let expr9_0 = OperandSize::Size64; + let expr10_0 = MInst::XmmRmRImm { + op: expr6_0, + src1: expr3_0, + src2: expr7_0, dst: expr1_0, - imm: expr6_0, - size: expr7_0, + imm: expr8_0, + size: expr9_0, }; - let expr9_0 = C::emit(ctx, &expr8_0); - let expr10_0 = SseOpcode::Pinsrd; - let expr11_0: u8 = 1; - let expr12_0 = OperandSize::Size64; - let expr13_0 = MInst::XmmRmRImm { - op: expr10_0, - src1: expr2_0, - src2: pattern1_0.clone(), + let expr11_0 = C::emit(ctx, &expr10_0); + let expr12_0 = SseOpcode::Pinsrd; + let expr13_0 = C::gpr_mem_to_reg_mem(ctx, pattern1_0); + let expr14_0: u8 = 1; + let expr15_0 = OperandSize::Size64; + let expr16_0 = MInst::XmmRmRImm { + op: expr12_0, + src1: expr3_0, + src2: expr13_0, dst: expr1_0, - imm: expr11_0, - size: expr12_0, + imm: expr14_0, + size: expr15_0, }; - let expr14_0 = C::emit(ctx, &expr13_0); + let expr17_0 = C::emit(ctx, &expr16_0); return Some(expr2_0); } -// Generated as internal constructor for term reg_mem_imm_to_xmm. -pub fn constructor_reg_mem_imm_to_xmm( - ctx: &mut C, - arg0: &RegMemImm, -) -> Option { +// Generated as internal constructor for term mov_rmi_to_xmm. +pub fn constructor_mov_rmi_to_xmm(ctx: &mut C, arg0: &RegMemImm) -> Option { let pattern0_0 = arg0; match pattern0_0 { &RegMemImm::Imm { simm32: pattern1_0 } => { - // Rule at src/isa/x64/inst.isle line 568. - return Some(pattern0_0.clone()); + // Rule at src/isa/x64/inst.isle line 1157. + let expr0_0 = C::xmm_mem_imm_new(ctx, pattern0_0); + return Some(expr0_0); } &RegMemImm::Reg { reg: pattern1_0 } => { - // Rule at src/isa/x64/inst.isle line 569. - let expr0_0: Type = I8X16; - let expr1_0 = SseOpcode::Movd; - let expr2_0 = RegMem::Reg { reg: pattern1_0 }; - let expr3_0 = OperandSize::Size32; - let expr4_0 = constructor_gpr_to_xmm(ctx, expr0_0, &expr1_0, &expr2_0, &expr3_0)?; - let expr5_0 = RegMemImm::Reg { reg: expr4_0 }; - return Some(expr5_0); + // Rule at src/isa/x64/inst.isle line 1158. + let expr0_0 = SseOpcode::Movd; + let expr1_0 = C::reg_to_gpr_mem(ctx, pattern1_0); + let expr2_0 = OperandSize::Size32; + let expr3_0 = constructor_gpr_to_xmm(ctx, &expr0_0, &expr1_0, &expr2_0)?; + let expr4_0 = C::xmm_to_xmm_mem_imm(ctx, expr3_0); + return Some(expr4_0); } &RegMemImm::Mem { addr: ref pattern1_0, } => { - // Rule at src/isa/x64/inst.isle line 567. - return Some(pattern0_0.clone()); + // Rule at src/isa/x64/inst.isle line 1156. + let expr0_0 = C::xmm_mem_imm_new(ctx, pattern0_0); + return Some(expr0_0); } _ => {} } @@ -519,7 +934,7 @@ pub fn constructor_x64_load( if pattern0_0 == I64 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 585. + // Rule at src/isa/x64/inst.isle line 1173. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::Mov64MR { @@ -533,53 +948,63 @@ pub fn constructor_x64_load( if pattern0_0 == F32 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 590. + // Rule at src/isa/x64/inst.isle line 1178. let expr0_0 = SseOpcode::Movss; let expr1_0 = C::synthetic_amode_to_reg_mem(ctx, pattern2_0); - let expr2_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, &expr1_0)?; - return Some(expr2_0); + let expr2_0 = C::xmm_mem_new(ctx, &expr1_0); + let expr3_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, &expr2_0)?; + let expr4_0 = C::xmm_to_reg(ctx, expr3_0); + return Some(expr4_0); } if pattern0_0 == F64 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 594. + // Rule at src/isa/x64/inst.isle line 1182. let expr0_0 = SseOpcode::Movsd; let expr1_0 = C::synthetic_amode_to_reg_mem(ctx, pattern2_0); - let expr2_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, &expr1_0)?; - return Some(expr2_0); + let expr2_0 = C::xmm_mem_new(ctx, &expr1_0); + let expr3_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, &expr2_0)?; + let expr4_0 = C::xmm_to_reg(ctx, expr3_0); + return Some(expr4_0); } if pattern0_0 == F32X4 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 598. + // Rule at src/isa/x64/inst.isle line 1186. let expr0_0 = SseOpcode::Movups; let expr1_0 = C::synthetic_amode_to_reg_mem(ctx, pattern2_0); - let expr2_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, &expr1_0)?; - return Some(expr2_0); + let expr2_0 = C::xmm_mem_new(ctx, &expr1_0); + let expr3_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, &expr2_0)?; + let expr4_0 = C::xmm_to_reg(ctx, expr3_0); + return Some(expr4_0); } if pattern0_0 == F64X2 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 602. + // Rule at src/isa/x64/inst.isle line 1190. let expr0_0 = SseOpcode::Movupd; let expr1_0 = C::synthetic_amode_to_reg_mem(ctx, pattern2_0); - let expr2_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, &expr1_0)?; - return Some(expr2_0); + let expr2_0 = C::xmm_mem_new(ctx, &expr1_0); + let expr3_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, &expr2_0)?; + let expr4_0 = C::xmm_to_reg(ctx, expr3_0); + return Some(expr4_0); } if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 606. + // Rule at src/isa/x64/inst.isle line 1194. let expr0_0 = SseOpcode::Movdqu; let expr1_0 = C::synthetic_amode_to_reg_mem(ctx, pattern2_0); - let expr2_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, &expr1_0)?; - return Some(expr2_0); + let expr2_0 = C::xmm_mem_new(ctx, &expr1_0); + let expr3_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, &expr2_0)?; + let expr4_0 = C::xmm_to_reg(ctx, expr3_0); + return Some(expr4_0); } if let Some(pattern1_0) = C::fits_in_32(ctx, pattern0_0) { let pattern2_0 = arg1; let pattern3_0 = arg2; if let &ExtKind::SignExtend = pattern3_0 { - // Rule at src/isa/x64/inst.isle line 580. + // Rule at src/isa/x64/inst.isle line 1168. let expr0_0 = C::ty_bytes(ctx, pattern1_0); let expr1_0: u16 = 8; let expr2_0 = C::ext_mode(ctx, expr0_0, expr1_0); @@ -603,7 +1028,7 @@ pub fn constructor_alu_rmi_r( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 625. + // Rule at src/isa/x64/inst.isle line 1213. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = MInst::AluRmiR { @@ -628,7 +1053,7 @@ pub fn constructor_add( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 633. + // Rule at src/isa/x64/inst.isle line 1221. let expr0_0 = AluRmiROpcode::Add; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -644,7 +1069,7 @@ pub fn constructor_add_with_flags( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 641. + // Rule at src/isa/x64/inst.isle line 1229. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = AluRmiROpcode::Add; @@ -673,7 +1098,7 @@ pub fn constructor_adc( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 652. + // Rule at src/isa/x64/inst.isle line 1240. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = AluRmiROpcode::Adc; @@ -702,7 +1127,7 @@ pub fn constructor_sub( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 663. + // Rule at src/isa/x64/inst.isle line 1251. let expr0_0 = AluRmiROpcode::Sub; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -718,7 +1143,7 @@ pub fn constructor_sub_with_flags( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 671. + // Rule at src/isa/x64/inst.isle line 1259. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = AluRmiROpcode::Sub; @@ -747,7 +1172,7 @@ pub fn constructor_sbb( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 682. + // Rule at src/isa/x64/inst.isle line 1270. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = AluRmiROpcode::Sbb; @@ -776,7 +1201,7 @@ pub fn constructor_mul( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 693. + // Rule at src/isa/x64/inst.isle line 1281. let expr0_0 = AluRmiROpcode::Mul; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -792,7 +1217,7 @@ pub fn constructor_x64_and( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 704. + // Rule at src/isa/x64/inst.isle line 1292. let expr0_0 = AluRmiROpcode::And; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -808,7 +1233,7 @@ pub fn constructor_or( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 712. + // Rule at src/isa/x64/inst.isle line 1300. let expr0_0 = AluRmiROpcode::Or; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -824,7 +1249,7 @@ pub fn constructor_xor( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 720. + // Rule at src/isa/x64/inst.isle line 1308. let expr0_0 = AluRmiROpcode::Xor; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -836,7 +1261,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option if pattern0_0 == I64 { let pattern2_0 = arg1; if let Some(pattern3_0) = C::nonzero_u64_fits_in_u32(ctx, pattern2_0) { - // Rule at src/isa/x64/inst.isle line 749. + // Rule at src/isa/x64/inst.isle line 1341. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = OperandSize::Size32; @@ -853,67 +1278,69 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option if pattern0_0 == F32 { let pattern2_0 = arg1; if pattern2_0 == 0 { - // Rule at src/isa/x64/inst.isle line 778. - let expr0_0: Type = F32; - let expr1_0 = C::temp_writable_reg(ctx, expr0_0); - let expr2_0 = C::writable_reg_to_reg(ctx, expr1_0); - let expr3_0 = SseOpcode::Xorps; - let expr4_0 = RegMem::Reg { reg: expr2_0 }; - let expr5_0 = MInst::XmmRmR { - op: expr3_0, - src1: expr2_0, - src2: expr4_0, - dst: expr1_0, + // Rule at src/isa/x64/inst.isle line 1370. + let expr0_0 = C::temp_writable_xmm(ctx); + let expr1_0 = C::writable_xmm_to_xmm(ctx, expr0_0); + let expr2_0 = SseOpcode::Xorps; + let expr3_0 = C::xmm_to_xmm_mem(ctx, expr1_0); + let expr4_0 = MInst::XmmRmR { + op: expr2_0, + src1: expr1_0, + src2: expr3_0, + dst: expr0_0, }; - let expr6_0 = C::emit(ctx, &expr5_0); - return Some(expr2_0); + let expr5_0 = C::emit(ctx, &expr4_0); + let expr6_0 = C::xmm_to_reg(ctx, expr1_0); + return Some(expr6_0); } - // Rule at src/isa/x64/inst.isle line 737. - let expr0_0: Type = F32; - let expr1_0 = SseOpcode::Movd; - let expr2_0: Type = I32; - let expr3_0 = constructor_imm(ctx, expr2_0, pattern2_0)?; - let expr4_0 = RegMem::Reg { reg: expr3_0 }; + // Rule at src/isa/x64/inst.isle line 1325. + let expr0_0 = SseOpcode::Movd; + let expr1_0: Type = I32; + let expr2_0 = constructor_imm(ctx, expr1_0, pattern2_0)?; + let expr3_0 = RegMem::Reg { reg: expr2_0 }; + let expr4_0 = C::gpr_mem_new(ctx, &expr3_0); let expr5_0 = OperandSize::Size32; - let expr6_0 = constructor_gpr_to_xmm(ctx, expr0_0, &expr1_0, &expr4_0, &expr5_0)?; - return Some(expr6_0); + let expr6_0 = constructor_gpr_to_xmm(ctx, &expr0_0, &expr4_0, &expr5_0)?; + let expr7_0 = C::xmm_to_reg(ctx, expr6_0); + return Some(expr7_0); } if pattern0_0 == F64 { let pattern2_0 = arg1; if pattern2_0 == 0 { - // Rule at src/isa/x64/inst.isle line 790. - let expr0_0: Type = F64; - let expr1_0 = C::temp_writable_reg(ctx, expr0_0); - let expr2_0 = C::writable_reg_to_reg(ctx, expr1_0); - let expr3_0 = SseOpcode::Xorpd; - let expr4_0 = RegMem::Reg { reg: expr2_0 }; - let expr5_0 = MInst::XmmRmR { - op: expr3_0, - src1: expr2_0, - src2: expr4_0, - dst: expr1_0, + // Rule at src/isa/x64/inst.isle line 1382. + let expr0_0 = C::temp_writable_xmm(ctx); + let expr1_0 = C::writable_xmm_to_xmm(ctx, expr0_0); + let expr2_0 = SseOpcode::Xorpd; + let expr3_0 = C::xmm_to_xmm_mem(ctx, expr1_0); + let expr4_0 = MInst::XmmRmR { + op: expr2_0, + src1: expr1_0, + src2: expr3_0, + dst: expr0_0, }; - let expr6_0 = C::emit(ctx, &expr5_0); - return Some(expr2_0); + let expr5_0 = C::emit(ctx, &expr4_0); + let expr6_0 = C::xmm_to_reg(ctx, expr1_0); + return Some(expr6_0); } - // Rule at src/isa/x64/inst.isle line 741. - let expr0_0: Type = F64; - let expr1_0 = SseOpcode::Movq; - let expr2_0: Type = I64; - let expr3_0 = constructor_imm(ctx, expr2_0, pattern2_0)?; - let expr4_0 = RegMem::Reg { reg: expr3_0 }; + // Rule at src/isa/x64/inst.isle line 1331. + let expr0_0 = SseOpcode::Movq; + let expr1_0: Type = I64; + let expr2_0 = constructor_imm(ctx, expr1_0, pattern2_0)?; + let expr3_0 = RegMem::Reg { reg: expr2_0 }; + let expr4_0 = C::gpr_mem_new(ctx, &expr3_0); let expr5_0 = OperandSize::Size64; - let expr6_0 = constructor_gpr_to_xmm(ctx, expr0_0, &expr1_0, &expr4_0, &expr5_0)?; - return Some(expr6_0); + let expr6_0 = constructor_gpr_to_xmm(ctx, &expr0_0, &expr4_0, &expr5_0)?; + let expr7_0 = C::xmm_to_reg(ctx, expr6_0); + return Some(expr7_0); } if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { let pattern2_0 = arg1; if pattern2_0 == 0 { - // Rule at src/isa/x64/inst.isle line 768. - let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); - let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); + // Rule at src/isa/x64/inst.isle line 1360. + let expr0_0 = C::temp_writable_xmm(ctx); + let expr1_0 = C::writable_xmm_to_xmm(ctx, expr0_0); let expr2_0 = constructor_sse_xor_op(ctx, pattern0_0)?; - let expr3_0 = RegMem::Reg { reg: expr1_0 }; + let expr3_0 = C::xmm_to_xmm_mem(ctx, expr1_0); let expr4_0 = MInst::XmmRmR { op: expr2_0, src1: expr1_0, @@ -921,13 +1348,14 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option dst: expr0_0, }; let expr5_0 = C::emit(ctx, &expr4_0); - return Some(expr1_0); + let expr6_0 = C::xmm_to_reg(ctx, expr1_0); + return Some(expr6_0); } } if let Some(pattern1_0) = C::fits_in_64(ctx, pattern0_0) { let pattern2_0 = arg1; if pattern2_0 == 0 { - // Rule at src/isa/x64/inst.isle line 755. + // Rule at src/isa/x64/inst.isle line 1347. let expr0_0 = C::temp_writable_reg(ctx, pattern1_0); let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); let expr2_0 = C::operand_size_of_type_32_64(ctx, pattern1_0); @@ -943,7 +1371,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option let expr6_0 = C::emit(ctx, &expr5_0); return Some(expr1_0); } - // Rule at src/isa/x64/inst.isle line 730. + // Rule at src/isa/x64/inst.isle line 1318. let expr0_0 = C::temp_writable_reg(ctx, pattern1_0); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern1_0); let expr2_0 = MInst::Imm { @@ -970,7 +1398,7 @@ pub fn constructor_shift_r( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 803. + // Rule at src/isa/x64/inst.isle line 1395. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::raw_operand_size_of_type(ctx, pattern0_0); let expr2_0 = MInst::ShiftR { @@ -995,7 +1423,7 @@ pub fn constructor_x64_rotl( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 814. + // Rule at src/isa/x64/inst.isle line 1406. let expr0_0 = ShiftKind::RotateLeft; let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -1011,7 +1439,7 @@ pub fn constructor_x64_rotr( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 820. + // Rule at src/isa/x64/inst.isle line 1412. let expr0_0 = ShiftKind::RotateRight; let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -1027,7 +1455,7 @@ pub fn constructor_shl( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 825. + // Rule at src/isa/x64/inst.isle line 1417. let expr0_0 = ShiftKind::ShiftLeft; let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -1043,7 +1471,7 @@ pub fn constructor_shr( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 830. + // Rule at src/isa/x64/inst.isle line 1422. let expr0_0 = ShiftKind::ShiftRightLogical; let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -1059,7 +1487,7 @@ pub fn constructor_sar( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 835. + // Rule at src/isa/x64/inst.isle line 1427. let expr0_0 = ShiftKind::ShiftRightArithmetic; let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -1077,7 +1505,7 @@ pub fn constructor_cmp_rmi_r( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 840. + // Rule at src/isa/x64/inst.isle line 1432. let expr0_0 = MInst::CmpRmiR { size: pattern0_0.clone(), opcode: pattern1_0.clone(), @@ -1102,7 +1530,7 @@ pub fn constructor_cmp( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 849. + // Rule at src/isa/x64/inst.isle line 1441. let expr0_0 = CmpOpcode::Cmp; let expr1_0 = constructor_cmp_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -1118,7 +1546,7 @@ pub fn constructor_test( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 854. + // Rule at src/isa/x64/inst.isle line 1446. let expr0_0 = CmpOpcode::Test; let expr1_0 = constructor_cmp_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -1136,7 +1564,7 @@ pub fn constructor_cmove( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 859. + // Rule at src/isa/x64/inst.isle line 1451. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = MInst::Cmove { @@ -1164,7 +1592,7 @@ pub fn constructor_movzx( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 867. + // Rule at src/isa/x64/inst.isle line 1459. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::MovzxRmR { ext_mode: pattern1_0.clone(), @@ -1186,7 +1614,7 @@ pub fn constructor_movsx( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 874. + // Rule at src/isa/x64/inst.isle line 1466. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::MovsxRmR { ext_mode: pattern1_0.clone(), @@ -1203,15 +1631,15 @@ pub fn constructor_xmm_rm_r( ctx: &mut C, arg0: Type, arg1: &SseOpcode, - arg2: Reg, - arg3: &RegMem, -) -> Option { + arg2: Xmm, + arg3: &XmmMem, +) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 881. - let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); + // Rule at src/isa/x64/inst.isle line 1473. + let expr0_0 = C::temp_writable_xmm(ctx); let expr1_0 = MInst::XmmRmR { op: pattern1_0.clone(), src1: pattern2_0, @@ -1219,15 +1647,15 @@ pub fn constructor_xmm_rm_r( dst: expr0_0, }; let expr2_0 = C::emit(ctx, &expr1_0); - let expr3_0 = C::writable_reg_to_reg(ctx, expr0_0); + let expr3_0 = C::writable_xmm_to_xmm(ctx, expr0_0); return Some(expr3_0); } // Generated as internal constructor for term paddb. -pub fn constructor_paddb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_paddb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 888. + // Rule at src/isa/x64/inst.isle line 1480. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Paddb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1235,10 +1663,10 @@ pub fn constructor_paddb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O } // Generated as internal constructor for term paddw. -pub fn constructor_paddw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_paddw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 893. + // Rule at src/isa/x64/inst.isle line 1485. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Paddw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1246,10 +1674,10 @@ pub fn constructor_paddw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O } // Generated as internal constructor for term paddd. -pub fn constructor_paddd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_paddd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 898. + // Rule at src/isa/x64/inst.isle line 1490. let expr0_0: Type = I32X4; let expr1_0 = SseOpcode::Paddd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1257,10 +1685,10 @@ pub fn constructor_paddd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O } // Generated as internal constructor for term paddq. -pub fn constructor_paddq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_paddq(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 903. + // Rule at src/isa/x64/inst.isle line 1495. let expr0_0: Type = I64X2; let expr1_0 = SseOpcode::Paddq; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1268,10 +1696,10 @@ pub fn constructor_paddq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O } // Generated as internal constructor for term paddsb. -pub fn constructor_paddsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_paddsb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 908. + // Rule at src/isa/x64/inst.isle line 1500. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Paddsb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1279,10 +1707,10 @@ pub fn constructor_paddsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term paddsw. -pub fn constructor_paddsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_paddsw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 913. + // Rule at src/isa/x64/inst.isle line 1505. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Paddsw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1290,10 +1718,10 @@ pub fn constructor_paddsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term paddusb. -pub fn constructor_paddusb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_paddusb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 918. + // Rule at src/isa/x64/inst.isle line 1510. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Paddusb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1301,10 +1729,10 @@ pub fn constructor_paddusb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term paddusw. -pub fn constructor_paddusw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_paddusw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 923. + // Rule at src/isa/x64/inst.isle line 1515. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Paddusw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1312,10 +1740,10 @@ pub fn constructor_paddusw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term psubb. -pub fn constructor_psubb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_psubb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 928. + // Rule at src/isa/x64/inst.isle line 1520. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Psubb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1323,10 +1751,10 @@ pub fn constructor_psubb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O } // Generated as internal constructor for term psubw. -pub fn constructor_psubw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_psubw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 933. + // Rule at src/isa/x64/inst.isle line 1525. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Psubw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1334,10 +1762,10 @@ pub fn constructor_psubw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O } // Generated as internal constructor for term psubd. -pub fn constructor_psubd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_psubd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 938. + // Rule at src/isa/x64/inst.isle line 1530. let expr0_0: Type = I32X4; let expr1_0 = SseOpcode::Psubd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1345,10 +1773,10 @@ pub fn constructor_psubd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O } // Generated as internal constructor for term psubq. -pub fn constructor_psubq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_psubq(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 943. + // Rule at src/isa/x64/inst.isle line 1535. let expr0_0: Type = I64X2; let expr1_0 = SseOpcode::Psubq; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1356,10 +1784,10 @@ pub fn constructor_psubq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O } // Generated as internal constructor for term psubsb. -pub fn constructor_psubsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_psubsb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 948. + // Rule at src/isa/x64/inst.isle line 1540. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Psubsb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1367,10 +1795,10 @@ pub fn constructor_psubsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term psubsw. -pub fn constructor_psubsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_psubsw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 953. + // Rule at src/isa/x64/inst.isle line 1545. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Psubsw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1378,10 +1806,10 @@ pub fn constructor_psubsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term psubusb. -pub fn constructor_psubusb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_psubusb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 958. + // Rule at src/isa/x64/inst.isle line 1550. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Psubusb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1389,10 +1817,10 @@ pub fn constructor_psubusb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term psubusw. -pub fn constructor_psubusw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_psubusw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 963. + // Rule at src/isa/x64/inst.isle line 1555. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Psubusw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1400,10 +1828,10 @@ pub fn constructor_psubusw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term pavgb. -pub fn constructor_pavgb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pavgb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 968. + // Rule at src/isa/x64/inst.isle line 1560. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pavgb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1411,10 +1839,10 @@ pub fn constructor_pavgb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O } // Generated as internal constructor for term pavgw. -pub fn constructor_pavgw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pavgw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 973. + // Rule at src/isa/x64/inst.isle line 1565. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pavgw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1422,10 +1850,10 @@ pub fn constructor_pavgw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O } // Generated as internal constructor for term pand. -pub fn constructor_pand(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pand(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 978. + // Rule at src/isa/x64/inst.isle line 1570. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Pand; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1433,10 +1861,10 @@ pub fn constructor_pand(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Op } // Generated as internal constructor for term andps. -pub fn constructor_andps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_andps(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 983. + // Rule at src/isa/x64/inst.isle line 1575. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Andps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1444,10 +1872,10 @@ pub fn constructor_andps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O } // Generated as internal constructor for term andpd. -pub fn constructor_andpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_andpd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 988. + // Rule at src/isa/x64/inst.isle line 1580. let expr0_0: Type = F64X2; let expr1_0 = SseOpcode::Andpd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1455,10 +1883,10 @@ pub fn constructor_andpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O } // Generated as internal constructor for term por. -pub fn constructor_por(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_por(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 993. + // Rule at src/isa/x64/inst.isle line 1585. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Por; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1466,10 +1894,10 @@ pub fn constructor_por(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Opt } // Generated as internal constructor for term orps. -pub fn constructor_orps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_orps(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 998. + // Rule at src/isa/x64/inst.isle line 1590. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Orps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1477,10 +1905,10 @@ pub fn constructor_orps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Op } // Generated as internal constructor for term orpd. -pub fn constructor_orpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_orpd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1003. + // Rule at src/isa/x64/inst.isle line 1595. let expr0_0: Type = F64X2; let expr1_0 = SseOpcode::Orpd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1488,10 +1916,10 @@ pub fn constructor_orpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Op } // Generated as internal constructor for term pxor. -pub fn constructor_pxor(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pxor(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1008. + // Rule at src/isa/x64/inst.isle line 1600. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pxor; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1499,10 +1927,10 @@ pub fn constructor_pxor(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Op } // Generated as internal constructor for term xorps. -pub fn constructor_xorps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_xorps(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1013. + // Rule at src/isa/x64/inst.isle line 1605. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Xorps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1510,10 +1938,10 @@ pub fn constructor_xorps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O } // Generated as internal constructor for term xorpd. -pub fn constructor_xorpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_xorpd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1018. + // Rule at src/isa/x64/inst.isle line 1610. let expr0_0: Type = F64X2; let expr1_0 = SseOpcode::Xorpd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1521,10 +1949,10 @@ pub fn constructor_xorpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O } // Generated as internal constructor for term pmullw. -pub fn constructor_pmullw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pmullw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1023. + // Rule at src/isa/x64/inst.isle line 1615. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pmullw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1532,10 +1960,10 @@ pub fn constructor_pmullw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term pmulld. -pub fn constructor_pmulld(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pmulld(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1028. + // Rule at src/isa/x64/inst.isle line 1620. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pmulld; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1543,10 +1971,10 @@ pub fn constructor_pmulld(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term pmulhw. -pub fn constructor_pmulhw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pmulhw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1033. + // Rule at src/isa/x64/inst.isle line 1625. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pmulhw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1554,10 +1982,10 @@ pub fn constructor_pmulhw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term pmulhuw. -pub fn constructor_pmulhuw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pmulhuw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1038. + // Rule at src/isa/x64/inst.isle line 1630. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pmulhuw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1565,10 +1993,10 @@ pub fn constructor_pmulhuw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term pmuldq. -pub fn constructor_pmuldq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pmuldq(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1043. + // Rule at src/isa/x64/inst.isle line 1635. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pmuldq; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1576,10 +2004,10 @@ pub fn constructor_pmuldq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term pmuludq. -pub fn constructor_pmuludq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pmuludq(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1048. + // Rule at src/isa/x64/inst.isle line 1640. let expr0_0: Type = I64X2; let expr1_0 = SseOpcode::Pmuludq; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1587,10 +2015,10 @@ pub fn constructor_pmuludq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term punpckhwd. -pub fn constructor_punpckhwd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_punpckhwd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1053. + // Rule at src/isa/x64/inst.isle line 1645. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Punpckhwd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1598,10 +2026,10 @@ pub fn constructor_punpckhwd(ctx: &mut C, arg0: Reg, arg1: &RegMem) } // Generated as internal constructor for term punpcklwd. -pub fn constructor_punpcklwd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_punpcklwd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1058. + // Rule at src/isa/x64/inst.isle line 1650. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Punpcklwd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1609,10 +2037,10 @@ pub fn constructor_punpcklwd(ctx: &mut C, arg0: Reg, arg1: &RegMem) } // Generated as internal constructor for term andnps. -pub fn constructor_andnps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_andnps(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1063. + // Rule at src/isa/x64/inst.isle line 1655. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Andnps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1620,10 +2048,10 @@ pub fn constructor_andnps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term andnpd. -pub fn constructor_andnpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_andnpd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1068. + // Rule at src/isa/x64/inst.isle line 1660. let expr0_0: Type = F64X2; let expr1_0 = SseOpcode::Andnpd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1631,10 +2059,10 @@ pub fn constructor_andnpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term pandn. -pub fn constructor_pandn(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pandn(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1073. + // Rule at src/isa/x64/inst.isle line 1665. let expr0_0: Type = F64X2; let expr1_0 = SseOpcode::Pandn; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1645,17 +2073,17 @@ pub fn constructor_pandn(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_sse_blend_op(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32X4 { - // Rule at src/isa/x64/inst.isle line 1077. + // Rule at src/isa/x64/inst.isle line 1669. let expr0_0 = SseOpcode::Blendvps; return Some(expr0_0); } if pattern0_0 == F64X2 { - // Rule at src/isa/x64/inst.isle line 1078. + // Rule at src/isa/x64/inst.isle line 1670. let expr0_0 = SseOpcode::Blendvpd; return Some(expr0_0); } if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { - // Rule at src/isa/x64/inst.isle line 1079. + // Rule at src/isa/x64/inst.isle line 1671. let expr0_0 = SseOpcode::Pblendvb; return Some(expr0_0); } @@ -1666,17 +2094,17 @@ pub fn constructor_sse_blend_op(ctx: &mut C, arg0: Type) -> Option(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32X4 { - // Rule at src/isa/x64/inst.isle line 1082. + // Rule at src/isa/x64/inst.isle line 1674. let expr0_0 = SseOpcode::Movaps; return Some(expr0_0); } if pattern0_0 == F64X2 { - // Rule at src/isa/x64/inst.isle line 1083. + // Rule at src/isa/x64/inst.isle line 1675. let expr0_0 = SseOpcode::Movapd; return Some(expr0_0); } if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { - // Rule at src/isa/x64/inst.isle line 1084. + // Rule at src/isa/x64/inst.isle line 1676. let expr0_0 = SseOpcode::Movdqa; return Some(expr0_0); } @@ -1687,15 +2115,15 @@ pub fn constructor_sse_mov_op(ctx: &mut C, arg0: Type) -> Option( ctx: &mut C, arg0: Type, - arg1: &RegMem, - arg2: &RegMem, - arg3: Reg, -) -> Option { + arg1: &XmmMem, + arg2: &XmmMem, + arg3: Xmm, +) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1088. + // Rule at src/isa/x64/inst.isle line 1680. let expr0_0 = C::xmm0(ctx); let expr1_0 = constructor_sse_mov_op(ctx, pattern0_0)?; let expr2_0 = MInst::XmmUnaryRmR { @@ -1712,17 +2140,17 @@ pub fn constructor_sse_blend( // Generated as internal constructor for term blendvpd. pub fn constructor_blendvpd( ctx: &mut C, - arg0: Reg, - arg1: &RegMem, - arg2: Reg, -) -> Option { + arg0: Xmm, + arg1: &XmmMem, + arg2: Xmm, +) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1100. + // Rule at src/isa/x64/inst.isle line 1694. let expr0_0 = C::xmm0(ctx); let expr1_0 = SseOpcode::Movapd; - let expr2_0 = RegMem::Reg { reg: pattern2_0 }; + let expr2_0 = C::xmm_to_xmm_mem(ctx, pattern2_0); let expr3_0 = MInst::XmmUnaryRmR { op: expr1_0, src: expr2_0, @@ -1736,10 +2164,10 @@ pub fn constructor_blendvpd( } // Generated as internal constructor for term movsd. -pub fn constructor_movsd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_movsd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1112. + // Rule at src/isa/x64/inst.isle line 1708. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Movsd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1747,10 +2175,10 @@ pub fn constructor_movsd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O } // Generated as internal constructor for term movlhps. -pub fn constructor_movlhps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_movlhps(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1117. + // Rule at src/isa/x64/inst.isle line 1713. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Movlhps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1758,10 +2186,10 @@ pub fn constructor_movlhps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term pmaxsb. -pub fn constructor_pmaxsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pmaxsb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1122. + // Rule at src/isa/x64/inst.isle line 1718. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxsb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1769,10 +2197,10 @@ pub fn constructor_pmaxsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term pmaxsw. -pub fn constructor_pmaxsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pmaxsw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1127. + // Rule at src/isa/x64/inst.isle line 1723. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxsw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1780,10 +2208,10 @@ pub fn constructor_pmaxsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term pmaxsd. -pub fn constructor_pmaxsd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pmaxsd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1132. + // Rule at src/isa/x64/inst.isle line 1728. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxsd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1791,10 +2219,10 @@ pub fn constructor_pmaxsd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term pminsb. -pub fn constructor_pminsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pminsb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1137. + // Rule at src/isa/x64/inst.isle line 1733. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pminsb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1802,10 +2230,10 @@ pub fn constructor_pminsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term pminsw. -pub fn constructor_pminsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pminsw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1142. + // Rule at src/isa/x64/inst.isle line 1738. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pminsw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1813,10 +2241,10 @@ pub fn constructor_pminsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term pminsd. -pub fn constructor_pminsd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pminsd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1147. + // Rule at src/isa/x64/inst.isle line 1743. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pminsd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1824,10 +2252,10 @@ pub fn constructor_pminsd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term pmaxub. -pub fn constructor_pmaxub(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pmaxub(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1152. + // Rule at src/isa/x64/inst.isle line 1748. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxub; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1835,10 +2263,10 @@ pub fn constructor_pmaxub(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term pmaxuw. -pub fn constructor_pmaxuw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pmaxuw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1157. + // Rule at src/isa/x64/inst.isle line 1753. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxuw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1846,10 +2274,10 @@ pub fn constructor_pmaxuw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term pmaxud. -pub fn constructor_pmaxud(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pmaxud(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1162. + // Rule at src/isa/x64/inst.isle line 1758. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxud; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1857,10 +2285,10 @@ pub fn constructor_pmaxud(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term pminub. -pub fn constructor_pminub(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pminub(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1167. + // Rule at src/isa/x64/inst.isle line 1763. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pminub; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1868,10 +2296,10 @@ pub fn constructor_pminub(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term pminuw. -pub fn constructor_pminuw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pminuw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1172. + // Rule at src/isa/x64/inst.isle line 1768. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pminuw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1879,10 +2307,10 @@ pub fn constructor_pminuw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term pminud. -pub fn constructor_pminud(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pminud(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1177. + // Rule at src/isa/x64/inst.isle line 1773. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pminud; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1890,10 +2318,10 @@ pub fn constructor_pminud(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term punpcklbw. -pub fn constructor_punpcklbw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_punpcklbw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1182. + // Rule at src/isa/x64/inst.isle line 1778. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Punpcklbw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1901,10 +2329,10 @@ pub fn constructor_punpcklbw(ctx: &mut C, arg0: Reg, arg1: &RegMem) } // Generated as internal constructor for term punpckhbw. -pub fn constructor_punpckhbw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_punpckhbw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1187. + // Rule at src/isa/x64/inst.isle line 1783. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Punpckhbw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1912,10 +2340,10 @@ pub fn constructor_punpckhbw(ctx: &mut C, arg0: Reg, arg1: &RegMem) } // Generated as internal constructor for term packsswb. -pub fn constructor_packsswb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_packsswb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1192. + // Rule at src/isa/x64/inst.isle line 1788. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Packsswb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1930,15 +2358,15 @@ pub fn constructor_xmm_rm_r_imm( arg2: &RegMem, arg3: u8, arg4: &OperandSize, -) -> Option { +) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; let pattern4_0 = arg4; - // Rule at src/isa/x64/inst.isle line 1197. - let expr0_0: Type = I8X16; - let expr1_0 = C::temp_writable_reg(ctx, expr0_0); + // Rule at src/isa/x64/inst.isle line 1793. + let expr0_0 = C::temp_writable_xmm(ctx); + let expr1_0 = C::writable_xmm_to_reg(ctx, expr0_0); let expr2_0 = MInst::XmmRmRImm { op: pattern0_0.clone(), src1: pattern1_0, @@ -1948,118 +2376,218 @@ pub fn constructor_xmm_rm_r_imm( size: pattern4_0.clone(), }; let expr3_0 = C::emit(ctx, &expr2_0); - let expr4_0 = C::writable_reg_to_reg(ctx, expr1_0); + let expr4_0 = C::writable_xmm_to_xmm(ctx, expr0_0); return Some(expr4_0); } // Generated as internal constructor for term palignr. pub fn constructor_palignr( ctx: &mut C, - arg0: Reg, - arg1: &RegMem, + arg0: Xmm, + arg1: &XmmMem, arg2: u8, arg3: &OperandSize, -) -> Option { +) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1209. + // Rule at src/isa/x64/inst.isle line 1805. let expr0_0 = SseOpcode::Palignr; - let expr1_0 = constructor_xmm_rm_r_imm( - ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0, pattern3_0, - )?; - return Some(expr1_0); + let expr1_0 = C::xmm_to_reg(ctx, pattern0_0); + let expr2_0 = C::xmm_mem_to_reg_mem(ctx, pattern1_0); + let expr3_0 = + constructor_xmm_rm_r_imm(ctx, &expr0_0, expr1_0, &expr2_0, pattern2_0, pattern3_0)?; + return Some(expr3_0); +} + +// Generated as internal constructor for term cmpps. +pub fn constructor_cmpps( + ctx: &mut C, + arg0: Xmm, + arg1: &XmmMem, + arg2: &FcmpImm, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + // Rule at src/isa/x64/inst.isle line 1814. + let expr0_0 = SseOpcode::Cmpps; + let expr1_0 = C::xmm_to_reg(ctx, pattern0_0); + let expr2_0 = C::xmm_mem_to_reg_mem(ctx, pattern1_0); + let expr3_0 = C::encode_fcmp_imm(ctx, pattern2_0); + let expr4_0 = OperandSize::Size32; + let expr5_0 = constructor_xmm_rm_r_imm(ctx, &expr0_0, expr1_0, &expr2_0, expr3_0, &expr4_0)?; + return Some(expr5_0); +} + +// Generated as internal constructor for term pinsrb. +pub fn constructor_pinsrb( + ctx: &mut C, + arg0: Xmm, + arg1: &GprMem, + arg2: u8, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + // Rule at src/isa/x64/inst.isle line 1823. + let expr0_0 = SseOpcode::Pinsrb; + let expr1_0 = C::xmm_to_reg(ctx, pattern0_0); + let expr2_0 = C::gpr_mem_to_reg_mem(ctx, pattern1_0); + let expr3_0 = OperandSize::Size32; + let expr4_0 = constructor_xmm_rm_r_imm(ctx, &expr0_0, expr1_0, &expr2_0, pattern2_0, &expr3_0)?; + return Some(expr4_0); +} + +// Generated as internal constructor for term pinsrw. +pub fn constructor_pinsrw( + ctx: &mut C, + arg0: Xmm, + arg1: &GprMem, + arg2: u8, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + // Rule at src/isa/x64/inst.isle line 1832. + let expr0_0 = SseOpcode::Pinsrw; + let expr1_0 = C::xmm_to_reg(ctx, pattern0_0); + let expr2_0 = C::gpr_mem_to_reg_mem(ctx, pattern1_0); + let expr3_0 = OperandSize::Size32; + let expr4_0 = constructor_xmm_rm_r_imm(ctx, &expr0_0, expr1_0, &expr2_0, pattern2_0, &expr3_0)?; + return Some(expr4_0); +} + +// Generated as internal constructor for term pinsrd. +pub fn constructor_pinsrd( + ctx: &mut C, + arg0: Xmm, + arg1: &GprMem, + arg2: u8, + arg3: &OperandSize, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + let pattern3_0 = arg3; + // Rule at src/isa/x64/inst.isle line 1841. + let expr0_0 = SseOpcode::Pinsrd; + let expr1_0 = C::xmm_to_reg(ctx, pattern0_0); + let expr2_0 = C::gpr_mem_to_reg_mem(ctx, pattern1_0); + let expr3_0 = + constructor_xmm_rm_r_imm(ctx, &expr0_0, expr1_0, &expr2_0, pattern2_0, pattern3_0)?; + return Some(expr3_0); +} + +// Generated as internal constructor for term insertps. +pub fn constructor_insertps( + ctx: &mut C, + arg0: Xmm, + arg1: &XmmMem, + arg2: u8, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + // Rule at src/isa/x64/inst.isle line 1850. + let expr0_0 = SseOpcode::Insertps; + let expr1_0 = C::xmm_to_reg(ctx, pattern0_0); + let expr2_0 = C::xmm_mem_to_reg_mem(ctx, pattern1_0); + let expr3_0 = OperandSize::Size32; + let expr4_0 = constructor_xmm_rm_r_imm(ctx, &expr0_0, expr1_0, &expr2_0, pattern2_0, &expr3_0)?; + return Some(expr4_0); } // Generated as internal constructor for term pshufd. pub fn constructor_pshufd( ctx: &mut C, - arg0: &RegMem, + arg0: &XmmMem, arg1: u8, arg2: &OperandSize, -) -> Option { +) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1218. - let expr0_0: Type = I8X16; - let expr1_0 = C::temp_writable_reg(ctx, expr0_0); - let expr2_0 = C::writable_reg_to_reg(ctx, expr1_0); - let expr3_0 = SseOpcode::Pshufd; - let expr4_0 = MInst::XmmRmRImm { - op: expr3_0, - src1: expr2_0, - src2: pattern0_0.clone(), - dst: expr1_0, + // Rule at src/isa/x64/inst.isle line 1859. + let expr0_0 = C::temp_writable_xmm(ctx); + let expr1_0 = C::writable_xmm_to_xmm(ctx, expr0_0); + let expr2_0 = SseOpcode::Pshufd; + let expr3_0 = C::xmm_to_reg(ctx, expr1_0); + let expr4_0 = C::xmm_mem_to_reg_mem(ctx, pattern0_0); + let expr5_0 = C::writable_xmm_to_reg(ctx, expr0_0); + let expr6_0 = MInst::XmmRmRImm { + op: expr2_0, + src1: expr3_0, + src2: expr4_0, + dst: expr5_0, imm: pattern1_0, size: pattern2_0.clone(), }; - let expr5_0 = C::emit(ctx, &expr4_0); - return Some(expr2_0); + let expr7_0 = C::emit(ctx, &expr6_0); + return Some(expr1_0); } // Generated as internal constructor for term xmm_unary_rm_r. pub fn constructor_xmm_unary_rm_r( ctx: &mut C, arg0: &SseOpcode, - arg1: &RegMem, -) -> Option { + arg1: &XmmMem, +) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1231. - let expr0_0: Type = I8X16; - let expr1_0 = C::temp_writable_reg(ctx, expr0_0); - let expr2_0 = MInst::XmmUnaryRmR { + // Rule at src/isa/x64/inst.isle line 1872. + let expr0_0 = C::temp_writable_xmm(ctx); + let expr1_0 = MInst::XmmUnaryRmR { op: pattern0_0.clone(), src: pattern1_0.clone(), - dst: expr1_0, + dst: expr0_0, }; - let expr3_0 = C::emit(ctx, &expr2_0); - let expr4_0 = C::writable_reg_to_reg(ctx, expr1_0); - return Some(expr4_0); + let expr2_0 = C::emit(ctx, &expr1_0); + let expr3_0 = C::writable_xmm_to_xmm(ctx, expr0_0); + return Some(expr3_0); } // Generated as internal constructor for term pmovsxbw. -pub fn constructor_pmovsxbw(ctx: &mut C, arg0: &RegMem) -> Option { +pub fn constructor_pmovsxbw(ctx: &mut C, arg0: &XmmMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1238. + // Rule at src/isa/x64/inst.isle line 1879. let expr0_0 = SseOpcode::Pmovsxbw; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); } // Generated as internal constructor for term pmovzxbw. -pub fn constructor_pmovzxbw(ctx: &mut C, arg0: &RegMem) -> Option { +pub fn constructor_pmovzxbw(ctx: &mut C, arg0: &XmmMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1243. + // Rule at src/isa/x64/inst.isle line 1884. let expr0_0 = SseOpcode::Pmovzxbw; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); } // Generated as internal constructor for term pabsb. -pub fn constructor_pabsb(ctx: &mut C, arg0: &RegMem) -> Option { +pub fn constructor_pabsb(ctx: &mut C, arg0: &XmmMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1248. + // Rule at src/isa/x64/inst.isle line 1889. let expr0_0 = SseOpcode::Pabsb; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); } // Generated as internal constructor for term pabsw. -pub fn constructor_pabsw(ctx: &mut C, arg0: &RegMem) -> Option { +pub fn constructor_pabsw(ctx: &mut C, arg0: &XmmMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1253. + // Rule at src/isa/x64/inst.isle line 1894. let expr0_0 = SseOpcode::Pabsw; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); } // Generated as internal constructor for term pabsd. -pub fn constructor_pabsd(ctx: &mut C, arg0: &RegMem) -> Option { +pub fn constructor_pabsd(ctx: &mut C, arg0: &XmmMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1258. + // Rule at src/isa/x64/inst.isle line 1899. let expr0_0 = SseOpcode::Pabsd; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -2069,27 +2597,26 @@ pub fn constructor_pabsd(ctx: &mut C, arg0: &RegMem) -> Option pub fn constructor_xmm_unary_rm_r_evex( ctx: &mut C, arg0: &Avx512Opcode, - arg1: &RegMem, -) -> Option { + arg1: &XmmMem, +) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1263. - let expr0_0: Type = I8X16; - let expr1_0 = C::temp_writable_reg(ctx, expr0_0); - let expr2_0 = MInst::XmmUnaryRmREvex { + // Rule at src/isa/x64/inst.isle line 1904. + let expr0_0 = C::temp_writable_xmm(ctx); + let expr1_0 = MInst::XmmUnaryRmREvex { op: pattern0_0.clone(), src: pattern1_0.clone(), - dst: expr1_0, + dst: expr0_0, }; - let expr3_0 = C::emit(ctx, &expr2_0); - let expr4_0 = C::writable_reg_to_reg(ctx, expr1_0); - return Some(expr4_0); + let expr2_0 = C::emit(ctx, &expr1_0); + let expr3_0 = C::writable_xmm_to_xmm(ctx, expr0_0); + return Some(expr3_0); } // Generated as internal constructor for term vpabsq. -pub fn constructor_vpabsq(ctx: &mut C, arg0: &RegMem) -> Option { +pub fn constructor_vpabsq(ctx: &mut C, arg0: &XmmMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1270. + // Rule at src/isa/x64/inst.isle line 1911. let expr0_0 = Avx512Opcode::Vpabsq; let expr1_0 = constructor_xmm_unary_rm_r_evex(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -2099,140 +2626,35 @@ pub fn constructor_vpabsq(ctx: &mut C, arg0: &RegMem) -> Option pub fn constructor_xmm_rm_r_evex( ctx: &mut C, arg0: &Avx512Opcode, - arg1: &RegMem, - arg2: Reg, -) -> Option { + arg1: &XmmMem, + arg2: Xmm, +) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1275. - let expr0_0: Type = I8X16; - let expr1_0 = C::temp_writable_reg(ctx, expr0_0); - let expr2_0 = MInst::XmmRmREvex { + // Rule at src/isa/x64/inst.isle line 1916. + let expr0_0 = C::temp_writable_xmm(ctx); + let expr1_0 = MInst::XmmRmREvex { op: pattern0_0.clone(), src1: pattern1_0.clone(), src2: pattern2_0, - dst: expr1_0, + dst: expr0_0, }; - let expr3_0 = C::emit(ctx, &expr2_0); - let expr4_0 = C::writable_reg_to_reg(ctx, expr1_0); - return Some(expr4_0); + let expr2_0 = C::emit(ctx, &expr1_0); + let expr3_0 = C::writable_xmm_to_xmm(ctx, expr0_0); + return Some(expr3_0); } // Generated as internal constructor for term vpmullq. -pub fn constructor_vpmullq(ctx: &mut C, arg0: &RegMem, arg1: Reg) -> Option { +pub fn constructor_vpmullq(ctx: &mut C, arg0: &XmmMem, arg1: Xmm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1287. + // Rule at src/isa/x64/inst.isle line 1928. let expr0_0 = Avx512Opcode::Vpmullq; let expr1_0 = constructor_xmm_rm_r_evex(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); } -// Generated as internal constructor for term xmm_rmi_reg. -pub fn constructor_xmm_rmi_reg( - ctx: &mut C, - arg0: &SseOpcode, - arg1: Reg, - arg2: &RegMemImm, -) -> Option { - let pattern0_0 = arg0; - let pattern1_0 = arg1; - let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1294. - let expr0_0: Type = I8X16; - let expr1_0 = C::temp_writable_reg(ctx, expr0_0); - let expr2_0 = MInst::XmmRmiReg { - opcode: pattern0_0.clone(), - src1: pattern1_0, - src2: pattern2_0.clone(), - dst: expr1_0, - }; - let expr3_0 = C::emit(ctx, &expr2_0); - let expr4_0 = C::writable_reg_to_reg(ctx, expr1_0); - return Some(expr4_0); -} - -// Generated as internal constructor for term psllw. -pub fn constructor_psllw(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option { - let pattern0_0 = arg0; - let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1304. - let expr0_0 = SseOpcode::Psllw; - let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; - return Some(expr1_0); -} - -// Generated as internal constructor for term pslld. -pub fn constructor_pslld(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option { - let pattern0_0 = arg0; - let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1309. - let expr0_0 = SseOpcode::Pslld; - let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; - return Some(expr1_0); -} - -// Generated as internal constructor for term psllq. -pub fn constructor_psllq(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option { - let pattern0_0 = arg0; - let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1314. - let expr0_0 = SseOpcode::Psllq; - let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; - return Some(expr1_0); -} - -// Generated as internal constructor for term psrlw. -pub fn constructor_psrlw(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option { - let pattern0_0 = arg0; - let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1319. - let expr0_0 = SseOpcode::Psrlw; - let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; - return Some(expr1_0); -} - -// Generated as internal constructor for term psrld. -pub fn constructor_psrld(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option { - let pattern0_0 = arg0; - let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1324. - let expr0_0 = SseOpcode::Psrld; - let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; - return Some(expr1_0); -} - -// Generated as internal constructor for term psrlq. -pub fn constructor_psrlq(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option { - let pattern0_0 = arg0; - let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1329. - let expr0_0 = SseOpcode::Psrlq; - let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; - return Some(expr1_0); -} - -// Generated as internal constructor for term psraw. -pub fn constructor_psraw(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option { - let pattern0_0 = arg0; - let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1334. - let expr0_0 = SseOpcode::Psraw; - let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; - return Some(expr1_0); -} - -// Generated as internal constructor for term psrad. -pub fn constructor_psrad(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option { - let pattern0_0 = arg0; - let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1339. - let expr0_0 = SseOpcode::Psrad; - let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; - return Some(expr1_0); -} - // Generated as internal constructor for term mul_hi. pub fn constructor_mul_hi( ctx: &mut C, @@ -2245,7 +2667,7 @@ pub fn constructor_mul_hi( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1346. + // Rule at src/isa/x64/inst.isle line 1937. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::temp_writable_reg(ctx, pattern0_0); let expr2_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); @@ -2274,179 +2696,192 @@ pub fn constructor_mulhi_u( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1362. + // Rule at src/isa/x64/inst.isle line 1953. let expr0_0: bool = false; let expr1_0 = constructor_mul_hi(ctx, pattern0_0, expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); } -// Generated as internal constructor for term cmpps. -pub fn constructor_cmpps( +// Generated as internal constructor for term xmm_rmi_xmm. +pub fn constructor_xmm_rmi_xmm( ctx: &mut C, - arg0: Reg, - arg1: &RegMem, - arg2: &FcmpImm, -) -> Option { + arg0: &SseOpcode, + arg1: Xmm, + arg2: &XmmMemImm, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + // Rule at src/isa/x64/inst.isle line 1958. + let expr0_0 = C::temp_writable_xmm(ctx); + let expr1_0 = MInst::XmmRmiReg { + opcode: pattern0_0.clone(), + src1: pattern1_0, + src2: pattern2_0.clone(), + dst: expr0_0, + }; + let expr2_0 = C::emit(ctx, &expr1_0); + let expr3_0 = C::writable_xmm_to_xmm(ctx, expr0_0); + return Some(expr3_0); +} + +// Generated as internal constructor for term psllw. +pub fn constructor_psllw(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 1968. + let expr0_0 = SseOpcode::Psllw; + let expr1_0 = constructor_xmm_rmi_xmm(ctx, &expr0_0, pattern0_0, pattern1_0)?; + return Some(expr1_0); +} + +// Generated as internal constructor for term pslld. +pub fn constructor_pslld(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 1973. + let expr0_0 = SseOpcode::Pslld; + let expr1_0 = constructor_xmm_rmi_xmm(ctx, &expr0_0, pattern0_0, pattern1_0)?; + return Some(expr1_0); +} + +// Generated as internal constructor for term psllq. +pub fn constructor_psllq(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 1978. + let expr0_0 = SseOpcode::Psllq; + let expr1_0 = constructor_xmm_rmi_xmm(ctx, &expr0_0, pattern0_0, pattern1_0)?; + return Some(expr1_0); +} + +// Generated as internal constructor for term psrlw. +pub fn constructor_psrlw(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1367. - let expr0_0 = SseOpcode::Cmpps; - let expr1_0 = C::encode_fcmp_imm(ctx, pattern2_0); - let expr2_0 = OperandSize::Size32; - let expr3_0 = - constructor_xmm_rm_r_imm(ctx, &expr0_0, pattern0_0, pattern1_0, expr1_0, &expr2_0)?; - return Some(expr3_0); + // Rule at src/isa/x64/inst.isle line 1983. + let expr0_0 = SseOpcode::Psrlw; + let expr1_0 = constructor_xmm_rmi_xmm(ctx, &expr0_0, pattern0_0, pattern1_0)?; + return Some(expr1_0); } -// Generated as internal constructor for term cmppd. -pub fn constructor_cmppd( - ctx: &mut C, - arg0: Reg, - arg1: &RegMem, - arg2: &FcmpImm, -) -> Option { +// Generated as internal constructor for term psrld. +pub fn constructor_psrld(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1380. - let expr0_0 = SseOpcode::Cmppd; - let expr1_0 = C::encode_fcmp_imm(ctx, pattern2_0); - let expr2_0 = OperandSize::Size32; - let expr3_0 = - constructor_xmm_rm_r_imm(ctx, &expr0_0, pattern0_0, pattern1_0, expr1_0, &expr2_0)?; - return Some(expr3_0); + // Rule at src/isa/x64/inst.isle line 1988. + let expr0_0 = SseOpcode::Psrld; + let expr1_0 = constructor_xmm_rmi_xmm(ctx, &expr0_0, pattern0_0, pattern1_0)?; + return Some(expr1_0); } -// Generated as internal constructor for term gpr_to_xmm. -pub fn constructor_gpr_to_xmm( - ctx: &mut C, - arg0: Type, - arg1: &SseOpcode, - arg2: &RegMem, - arg3: &OperandSize, -) -> Option { +// Generated as internal constructor for term psrlq. +pub fn constructor_psrlq(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - let pattern2_0 = arg2; - let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1389. - let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); - let expr1_0 = MInst::GprToXmm { - op: pattern1_0.clone(), - src: pattern2_0.clone(), - dst: expr0_0, - src_size: pattern3_0.clone(), - }; - let expr2_0 = C::emit(ctx, &expr1_0); - let expr3_0 = C::writable_reg_to_reg(ctx, expr0_0); - return Some(expr3_0); + // Rule at src/isa/x64/inst.isle line 1993. + let expr0_0 = SseOpcode::Psrlq; + let expr1_0 = constructor_xmm_rmi_xmm(ctx, &expr0_0, pattern0_0, pattern1_0)?; + return Some(expr1_0); } -// Generated as internal constructor for term pinsrb. -pub fn constructor_pinsrb( - ctx: &mut C, - arg0: Reg, - arg1: &RegMem, - arg2: u8, -) -> Option { +// Generated as internal constructor for term psraw. +pub fn constructor_psraw(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1396. - let expr0_0 = SseOpcode::Pinsrb; - let expr1_0 = OperandSize::Size32; - let expr2_0 = - constructor_xmm_rm_r_imm(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0, &expr1_0)?; - return Some(expr2_0); + // Rule at src/isa/x64/inst.isle line 1998. + let expr0_0 = SseOpcode::Psraw; + let expr1_0 = constructor_xmm_rmi_xmm(ctx, &expr0_0, pattern0_0, pattern1_0)?; + return Some(expr1_0); } -// Generated as internal constructor for term pinsrw. -pub fn constructor_pinsrw( - ctx: &mut C, - arg0: Reg, - arg1: &RegMem, - arg2: u8, -) -> Option { +// Generated as internal constructor for term psrad. +pub fn constructor_psrad(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1401. - let expr0_0 = SseOpcode::Pinsrw; - let expr1_0 = OperandSize::Size32; - let expr2_0 = - constructor_xmm_rm_r_imm(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0, &expr1_0)?; - return Some(expr2_0); + // Rule at src/isa/x64/inst.isle line 2003. + let expr0_0 = SseOpcode::Psrad; + let expr1_0 = constructor_xmm_rmi_xmm(ctx, &expr0_0, pattern0_0, pattern1_0)?; + return Some(expr1_0); } -// Generated as internal constructor for term pinsrd. -pub fn constructor_pinsrd( - ctx: &mut C, - arg0: Reg, - arg1: &RegMem, - arg2: u8, - arg3: &OperandSize, -) -> Option { +// Generated as internal constructor for term pextrd. +pub fn constructor_pextrd(ctx: &mut C, arg0: Type, arg1: Xmm, arg2: u8) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1406. - let expr0_0 = SseOpcode::Pinsrd; - let expr1_0 = constructor_xmm_rm_r_imm( - ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0, pattern3_0, - )?; + // Rule at src/isa/x64/inst.isle line 2008. + let expr0_0 = C::temp_writable_gpr(ctx); + let expr1_0 = C::writable_gpr_to_gpr(ctx, expr0_0); + let expr2_0 = SseOpcode::Pextrd; + let expr3_0 = C::gpr_to_reg(ctx, expr1_0); + let expr4_0 = C::xmm_to_reg(ctx, pattern1_0); + let expr5_0 = RegMem::Reg { reg: expr4_0 }; + let expr6_0 = C::writable_gpr_to_reg(ctx, expr0_0); + let expr7_0 = C::lane_type(ctx, pattern0_0); + let expr8_0 = C::operand_size_of_type_32_64(ctx, expr7_0); + let expr9_0 = MInst::XmmRmRImm { + op: expr2_0, + src1: expr3_0, + src2: expr5_0, + dst: expr6_0, + imm: pattern2_0, + size: expr8_0, + }; + let expr10_0 = C::emit(ctx, &expr9_0); return Some(expr1_0); } -// Generated as internal constructor for term insertps. -pub fn constructor_insertps( +// Generated as internal constructor for term cmppd. +pub fn constructor_cmppd( ctx: &mut C, - arg0: Reg, - arg1: &RegMem, - arg2: u8, -) -> Option { + arg0: Xmm, + arg1: &XmmMem, + arg2: &FcmpImm, +) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1411. - let expr0_0 = SseOpcode::Insertps; - let expr1_0 = OperandSize::Size32; - let expr2_0 = - constructor_xmm_rm_r_imm(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0, &expr1_0)?; - return Some(expr2_0); + // Rule at src/isa/x64/inst.isle line 2025. + let expr0_0 = SseOpcode::Cmppd; + let expr1_0 = C::xmm_to_reg(ctx, pattern0_0); + let expr2_0 = C::xmm_mem_to_reg_mem(ctx, pattern1_0); + let expr3_0 = C::encode_fcmp_imm(ctx, pattern2_0); + let expr4_0 = OperandSize::Size32; + let expr5_0 = constructor_xmm_rm_r_imm(ctx, &expr0_0, expr1_0, &expr2_0, expr3_0, &expr4_0)?; + return Some(expr5_0); } -// Generated as internal constructor for term pextrd. -pub fn constructor_pextrd(ctx: &mut C, arg0: Type, arg1: Reg, arg2: u8) -> Option { +// Generated as internal constructor for term gpr_to_xmm. +pub fn constructor_gpr_to_xmm( + ctx: &mut C, + arg0: &SseOpcode, + arg1: &GprMem, + arg2: &OperandSize, +) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1416. - let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); - let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); - let expr2_0 = SseOpcode::Pextrd; - let expr3_0 = RegMem::Reg { reg: pattern1_0 }; - let expr4_0 = C::lane_type(ctx, pattern0_0); - let expr5_0 = C::operand_size_of_type_32_64(ctx, expr4_0); - let expr6_0 = MInst::XmmRmRImm { - op: expr2_0, - src1: expr1_0, - src2: expr3_0, + // Rule at src/isa/x64/inst.isle line 2034. + let expr0_0 = C::temp_writable_xmm(ctx); + let expr1_0 = C::gpr_mem_to_reg_mem(ctx, pattern1_0); + let expr2_0 = MInst::GprToXmm { + op: pattern0_0.clone(), + src: expr1_0, dst: expr0_0, - imm: pattern2_0, - size: expr5_0, + src_size: pattern2_0.clone(), }; - let expr7_0 = C::emit(ctx, &expr6_0); - return Some(expr1_0); + let expr3_0 = C::emit(ctx, &expr2_0); + let expr4_0 = C::writable_xmm_to_xmm(ctx, expr0_0); + return Some(expr4_0); } // Generated as internal constructor for term not. -pub fn constructor_not(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { +pub fn constructor_not(ctx: &mut C, arg0: Type, arg1: Gpr) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1429. - let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); + // Rule at src/isa/x64/inst.isle line 2041. + let expr0_0 = C::temp_writable_gpr(ctx); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = MInst::Not { size: expr1_0, @@ -2454,16 +2889,16 @@ pub fn constructor_not(ctx: &mut C, arg0: Type, arg1: Reg) -> Option dst: expr0_0, }; let expr3_0 = C::emit(ctx, &expr2_0); - let expr4_0 = C::writable_reg_to_reg(ctx, expr0_0); + let expr4_0 = C::writable_gpr_to_gpr(ctx, expr0_0); return Some(expr4_0); } // Generated as internal constructor for term neg. -pub fn constructor_neg(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { +pub fn constructor_neg(ctx: &mut C, arg0: Type, arg1: Gpr) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1437. - let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); + // Rule at src/isa/x64/inst.isle line 2049. + let expr0_0 = C::temp_writable_gpr(ctx); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = MInst::Neg { size: expr1_0, @@ -2471,29 +2906,28 @@ pub fn constructor_neg(ctx: &mut C, arg0: Type, arg1: Reg) -> Option dst: expr0_0, }; let expr3_0 = C::emit(ctx, &expr2_0); - let expr4_0 = C::writable_reg_to_reg(ctx, expr0_0); + let expr4_0 = C::writable_gpr_to_gpr(ctx, expr0_0); return Some(expr4_0); } // Generated as internal constructor for term lea. -pub fn constructor_lea(ctx: &mut C, arg0: &SyntheticAmode) -> Option { +pub fn constructor_lea(ctx: &mut C, arg0: &SyntheticAmode) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1444. - let expr0_0: Type = I64; - let expr1_0 = C::temp_writable_reg(ctx, expr0_0); - let expr2_0 = MInst::LoadEffectiveAddress { + // Rule at src/isa/x64/inst.isle line 2056. + let expr0_0 = C::temp_writable_gpr(ctx); + let expr1_0 = MInst::LoadEffectiveAddress { addr: pattern0_0.clone(), - dst: expr1_0, + dst: expr0_0, }; - let expr3_0 = C::emit(ctx, &expr2_0); - let expr4_0 = C::writable_reg_to_reg(ctx, expr1_0); - return Some(expr4_0); + let expr2_0 = C::emit(ctx, &expr1_0); + let expr3_0 = C::writable_gpr_to_gpr(ctx, expr0_0); + return Some(expr3_0); } // Generated as internal constructor for term ud2. pub fn constructor_ud2(ctx: &mut C, arg0: &TrapCode) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1451. + // Rule at src/isa/x64/inst.isle line 2063. let expr0_0 = MInst::Ud2 { trap_code: pattern0_0.clone(), }; @@ -2538,13 +2972,13 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { match &pattern2_0 { &Opcode::Trap => { - // Rule at src/isa/x64/lower.isle line 1438. + // Rule at src/isa/x64/lower.isle line 1463. let expr0_0 = constructor_ud2(ctx, &pattern2_1)?; let expr1_0 = constructor_value_regs_none(ctx, &expr0_0)?; return Some(expr1_0); } &Opcode::ResumableTrap => { - // Rule at src/isa/x64/lower.isle line 1443. + // Rule at src/isa/x64/lower.isle line 1468. let expr0_0 = constructor_ud2(ctx, &pattern2_1)?; let expr1_0 = constructor_value_regs_none(ctx, &expr0_0)?; return Some(expr1_0); @@ -2561,12 +2995,12 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 384. + // Rule at src/isa/x64/lower.isle line 386. let expr0_0 = C::put_in_regs(ctx, pattern7_0); let expr1_0: usize = 0; let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); @@ -2630,7 +3064,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 458. + // Rule at src/isa/x64/lower.isle line 462. let expr0_0 = C::put_in_regs(ctx, pattern7_0); let expr1_0: usize = 0; let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); @@ -2646,7 +3080,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 524. + // Rule at src/isa/x64/lower.isle line 528. let expr0_0 = C::put_in_regs(ctx, pattern7_0); let expr1_0: usize = 0; let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); @@ -2667,7 +3101,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::Bnot = &pattern5_0 { - // Rule at src/isa/x64/lower.isle line 1304. + // Rule at src/isa/x64/lower.isle line 1321. let expr0_0 = constructor_i128_not(ctx, pattern5_1)?; return Some(expr0_0); } @@ -2750,7 +3184,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 992. + // Rule at src/isa/x64/lower.isle line 1007. let expr0_0 = C::put_in_regs(ctx, pattern7_0); let expr1_0: usize = 0; let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); @@ -2786,7 +3220,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 374. + // Rule at src/isa/x64/lower.isle line 376. let expr0_0 = C::put_in_regs(ctx, pattern7_0); let expr1_0: usize = 0; let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); @@ -2809,7 +3243,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 455. + // Rule at src/isa/x64/lower.isle line 459. let expr0_0 = C::put_in_regs(ctx, pattern7_0); let expr1_0 = C::put_in_regs(ctx, pattern7_1); let expr2_0 = constructor_or_i128(ctx, expr0_0, expr1_0)?; @@ -2818,7 +3252,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 514. + // Rule at src/isa/x64/lower.isle line 518. let expr0_0 = C::put_in_regs(ctx, pattern7_0); let expr1_0: usize = 0; let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); @@ -2841,7 +3275,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 860. + // Rule at src/isa/x64/lower.isle line 875. let expr0_0 = C::put_in_regs(ctx, pattern7_0); let expr1_0 = constructor_lo_reg(ctx, pattern7_1)?; let expr2_0 = constructor_shl_i128(ctx, expr0_0, expr1_0)?; @@ -2858,7 +3292,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 900. + // Rule at src/isa/x64/lower.isle line 915. let expr0_0 = C::put_in_regs(ctx, pattern7_0); let expr1_0 = constructor_lo_reg(ctx, pattern7_1)?; let expr2_0 = constructor_shr_i128(ctx, expr0_0, expr1_0)?; @@ -2875,7 +3309,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 568. + // Rule at src/isa/x64/lower.isle line 572. let expr0_0 = constructor_lo_reg(ctx, pattern7_1)?; let expr1_0 = C::put_in_regs(ctx, pattern7_0); let expr2_0 = constructor_shl_i128(ctx, expr1_0, expr0_0)?; @@ -2884,7 +3318,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 664. + // Rule at src/isa/x64/lower.isle line 671. let expr0_0 = constructor_lo_reg(ctx, pattern7_1)?; let expr1_0 = C::put_in_regs(ctx, pattern7_0); let expr2_0 = constructor_shr_i128(ctx, expr1_0, expr0_0)?; @@ -2893,7 +3327,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 761. + // Rule at src/isa/x64/lower.isle line 773. let expr0_0 = constructor_lo_reg(ctx, pattern7_1)?; let expr1_0 = C::put_in_regs(ctx, pattern7_0); let expr2_0 = constructor_sar_i128(ctx, expr1_0, expr0_0)?; @@ -2907,7 +3341,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::Bnot = &pattern5_0 { - // Rule at src/isa/x64/lower.isle line 1301. + // Rule at src/isa/x64/lower.isle line 1318. let expr0_0 = constructor_i128_not(ctx, pattern5_1)?; return Some(expr0_0); } @@ -2952,50 +3386,50 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1405. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); + // Rule at src/isa/x64/lower.isle line 1430. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_pminsb(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::Umin => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1427. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); + // Rule at src/isa/x64/lower.isle line 1452. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_pminub(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::Imax => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1394. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); + // Rule at src/isa/x64/lower.isle line 1419. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_pmaxsb(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::Umax => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1416. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); + // Rule at src/isa/x64/lower.isle line 1441. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_pmaxub(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::Ishl => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 580. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); + // Rule at src/isa/x64/lower.isle line 584. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = C::put_in_reg_mem_imm(ctx, pattern7_1); - let expr2_0 = constructor_reg_mem_imm_to_xmm(ctx, &expr1_0)?; + let expr2_0 = constructor_mov_rmi_to_xmm(ctx, &expr1_0)?; let expr3_0 = constructor_psllw(ctx, expr0_0, &expr2_0)?; let expr4_0 = constructor_ishl_i8x16_mask(ctx, &expr1_0)?; let expr5_0: Type = I8X16; @@ -3003,17 +3437,18 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 674. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); + // Rule at src/isa/x64/lower.isle line 681. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = C::put_in_reg_mem_imm(ctx, pattern7_1); - let expr2_0 = constructor_reg_mem_imm_to_xmm(ctx, &expr1_0)?; + let expr2_0 = constructor_mov_rmi_to_xmm(ctx, &expr1_0)?; let expr3_0 = constructor_psrlw(ctx, expr0_0, &expr2_0)?; let expr4_0 = constructor_ushr_i8x16_mask(ctx, &expr1_0)?; let expr5_0: Type = I8X16; @@ -3021,28 +3456,29 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); let pattern8_0 = C::value_type(ctx, pattern7_1); - // Rule at src/isa/x64/lower.isle line 782. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); - let expr1_0 = RegMem::Reg { reg: expr0_0 }; + // Rule at src/isa/x64/lower.isle line 794. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; + let expr1_0 = C::xmm_to_xmm_mem(ctx, expr0_0); let expr2_0 = constructor_punpcklbw(ctx, expr0_0, &expr1_0)?; - let expr3_0 = RegMem::Reg { reg: expr0_0 }; + let expr3_0 = C::xmm_to_xmm_mem(ctx, expr0_0); let expr4_0 = constructor_punpckhbw(ctx, expr0_0, &expr3_0)?; let expr5_0 = C::put_in_reg_mem_imm(ctx, pattern7_1); let expr6_0 = constructor_sshr_i8x16_bigger_shift(ctx, pattern8_0, &expr5_0)?; let expr7_0 = constructor_psraw(ctx, expr2_0, &expr6_0)?; let expr8_0 = constructor_psraw(ctx, expr4_0, &expr6_0)?; - let expr9_0 = RegMem::Reg { reg: expr8_0 }; + let expr9_0 = C::xmm_to_xmm_mem(ctx, expr8_0); let expr10_0 = constructor_packsswb(ctx, expr7_0, &expr9_0)?; - let expr11_0 = C::value_reg(ctx, expr10_0); + let expr11_0 = constructor_value_xmm(ctx, expr10_0)?; return Some(expr11_0); } _ => {} @@ -3054,20 +3490,21 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { match &pattern5_0 { &Opcode::Ineg => { - // Rule at src/isa/x64/lower.isle line 917. + // Rule at src/isa/x64/lower.isle line 932. let expr0_0: Type = I8X16; let expr1_0: u64 = 0; let expr2_0 = constructor_imm(ctx, expr0_0, expr1_0)?; - let expr3_0 = C::put_in_reg_mem(ctx, pattern5_1); - let expr4_0 = constructor_psubb(ctx, expr2_0, &expr3_0)?; - let expr5_0 = C::value_reg(ctx, expr4_0); - return Some(expr5_0); + let expr3_0 = C::xmm_new(ctx, expr2_0); + let expr4_0 = constructor_put_in_xmm_mem(ctx, pattern5_1)?; + let expr5_0 = constructor_psubb(ctx, expr3_0, &expr4_0)?; + let expr6_0 = constructor_value_xmm(ctx, expr5_0)?; + return Some(expr6_0); } &Opcode::Iabs => { - // Rule at src/isa/x64/lower.isle line 1247. - let expr0_0 = C::put_in_reg_mem(ctx, pattern5_1); + // Rule at src/isa/x64/lower.isle line 1262. + let expr0_0 = constructor_put_in_xmm_mem(ctx, pattern5_1)?; let expr1_0 = constructor_pabsb(ctx, &expr0_0)?; - let expr2_0 = C::value_reg(ctx, expr1_0); + let expr2_0 = constructor_value_xmm(ctx, expr1_0)?; return Some(expr2_0); } _ => {} @@ -3087,74 +3524,74 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1408. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); + // Rule at src/isa/x64/lower.isle line 1433. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_pminsw(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::Umin => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1430. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); + // Rule at src/isa/x64/lower.isle line 1455. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_pminuw(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::Imax => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1397. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); + // Rule at src/isa/x64/lower.isle line 1422. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_pmaxsw(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::Umax => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1419. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); + // Rule at src/isa/x64/lower.isle line 1444. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_pmaxuw(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::Ishl => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 621. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); + // Rule at src/isa/x64/lower.isle line 626. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = C::put_in_reg_mem_imm(ctx, pattern7_1); - let expr2_0 = constructor_reg_mem_imm_to_xmm(ctx, &expr1_0)?; + let expr2_0 = constructor_mov_rmi_to_xmm(ctx, &expr1_0)?; let expr3_0 = constructor_psllw(ctx, expr0_0, &expr2_0)?; - let expr4_0 = C::value_reg(ctx, expr3_0); + let expr4_0 = constructor_value_xmm(ctx, expr3_0)?; return Some(expr4_0); } &Opcode::Ushr => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 715. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); + // Rule at src/isa/x64/lower.isle line 725. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = C::put_in_reg_mem_imm(ctx, pattern7_1); - let expr2_0 = constructor_reg_mem_imm_to_xmm(ctx, &expr1_0)?; + let expr2_0 = constructor_mov_rmi_to_xmm(ctx, &expr1_0)?; let expr3_0 = constructor_psrlw(ctx, expr0_0, &expr2_0)?; - let expr4_0 = C::value_reg(ctx, expr3_0); + let expr4_0 = constructor_value_xmm(ctx, expr3_0)?; return Some(expr4_0); } &Opcode::Sshr => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 804. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); + // Rule at src/isa/x64/lower.isle line 817. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = C::put_in_reg_mem_imm(ctx, pattern7_1); - let expr2_0 = constructor_reg_mem_imm_to_xmm(ctx, &expr1_0)?; + let expr2_0 = constructor_mov_rmi_to_xmm(ctx, &expr1_0)?; let expr3_0 = constructor_psraw(ctx, expr0_0, &expr2_0)?; - let expr4_0 = C::value_reg(ctx, expr3_0); + let expr4_0 = constructor_value_xmm(ctx, expr3_0)?; return Some(expr4_0); } _ => {} @@ -3166,20 +3603,21 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { match &pattern5_0 { &Opcode::Ineg => { - // Rule at src/isa/x64/lower.isle line 921. + // Rule at src/isa/x64/lower.isle line 936. let expr0_0: Type = I16X8; let expr1_0: u64 = 0; let expr2_0 = constructor_imm(ctx, expr0_0, expr1_0)?; - let expr3_0 = C::put_in_reg_mem(ctx, pattern5_1); - let expr4_0 = constructor_psubw(ctx, expr2_0, &expr3_0)?; - let expr5_0 = C::value_reg(ctx, expr4_0); - return Some(expr5_0); + let expr3_0 = C::xmm_new(ctx, expr2_0); + let expr4_0 = constructor_put_in_xmm_mem(ctx, pattern5_1)?; + let expr5_0 = constructor_psubw(ctx, expr3_0, &expr4_0)?; + let expr6_0 = constructor_value_xmm(ctx, expr5_0)?; + return Some(expr6_0); } &Opcode::Iabs => { - // Rule at src/isa/x64/lower.isle line 1250. - let expr0_0 = C::put_in_reg_mem(ctx, pattern5_1); + // Rule at src/isa/x64/lower.isle line 1265. + let expr0_0 = constructor_put_in_xmm_mem(ctx, pattern5_1)?; let expr1_0 = constructor_pabsw(ctx, &expr0_0)?; - let expr2_0 = C::value_reg(ctx, expr1_0); + let expr2_0 = constructor_value_xmm(ctx, expr1_0)?; return Some(expr2_0); } _ => {} @@ -3199,74 +3637,74 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1411. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); + // Rule at src/isa/x64/lower.isle line 1436. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_pminsd(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::Umin => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1433. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); + // Rule at src/isa/x64/lower.isle line 1458. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_pminud(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::Imax => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1400. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); + // Rule at src/isa/x64/lower.isle line 1425. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_pmaxsd(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::Umax => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1422. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); + // Rule at src/isa/x64/lower.isle line 1447. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_pmaxud(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::Ishl => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 624. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); + // Rule at src/isa/x64/lower.isle line 630. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = C::put_in_reg_mem_imm(ctx, pattern7_1); - let expr2_0 = constructor_reg_mem_imm_to_xmm(ctx, &expr1_0)?; + let expr2_0 = constructor_mov_rmi_to_xmm(ctx, &expr1_0)?; let expr3_0 = constructor_pslld(ctx, expr0_0, &expr2_0)?; - let expr4_0 = C::value_reg(ctx, expr3_0); + let expr4_0 = constructor_value_xmm(ctx, expr3_0)?; return Some(expr4_0); } &Opcode::Ushr => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 718. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); + // Rule at src/isa/x64/lower.isle line 729. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = C::put_in_reg_mem_imm(ctx, pattern7_1); - let expr2_0 = constructor_reg_mem_imm_to_xmm(ctx, &expr1_0)?; + let expr2_0 = constructor_mov_rmi_to_xmm(ctx, &expr1_0)?; let expr3_0 = constructor_psrld(ctx, expr0_0, &expr2_0)?; - let expr4_0 = C::value_reg(ctx, expr3_0); + let expr4_0 = constructor_value_xmm(ctx, expr3_0)?; return Some(expr4_0); } &Opcode::Sshr => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 807. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); + // Rule at src/isa/x64/lower.isle line 821. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = C::put_in_reg_mem_imm(ctx, pattern7_1); - let expr2_0 = constructor_reg_mem_imm_to_xmm(ctx, &expr1_0)?; + let expr2_0 = constructor_mov_rmi_to_xmm(ctx, &expr1_0)?; let expr3_0 = constructor_psrad(ctx, expr0_0, &expr2_0)?; - let expr4_0 = C::value_reg(ctx, expr3_0); + let expr4_0 = constructor_value_xmm(ctx, expr3_0)?; return Some(expr4_0); } _ => {} @@ -3278,20 +3716,21 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { match &pattern5_0 { &Opcode::Ineg => { - // Rule at src/isa/x64/lower.isle line 925. + // Rule at src/isa/x64/lower.isle line 940. let expr0_0: Type = I32X4; let expr1_0: u64 = 0; let expr2_0 = constructor_imm(ctx, expr0_0, expr1_0)?; - let expr3_0 = C::put_in_reg_mem(ctx, pattern5_1); - let expr4_0 = constructor_psubd(ctx, expr2_0, &expr3_0)?; - let expr5_0 = C::value_reg(ctx, expr4_0); - return Some(expr5_0); + let expr3_0 = C::xmm_new(ctx, expr2_0); + let expr4_0 = constructor_put_in_xmm_mem(ctx, pattern5_1)?; + let expr5_0 = constructor_psubd(ctx, expr3_0, &expr4_0)?; + let expr6_0 = constructor_value_xmm(ctx, expr5_0)?; + return Some(expr6_0); } &Opcode::Iabs => { - // Rule at src/isa/x64/lower.isle line 1253. - let expr0_0 = C::put_in_reg_mem(ctx, pattern5_1); + // Rule at src/isa/x64/lower.isle line 1268. + let expr0_0 = constructor_put_in_xmm_mem(ctx, pattern5_1)?; let expr1_0 = constructor_pabsd(ctx, &expr0_0)?; - let expr2_0 = C::value_reg(ctx, expr1_0); + let expr2_0 = constructor_value_xmm(ctx, expr1_0)?; return Some(expr2_0); } _ => {} @@ -3311,30 +3750,30 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 627. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); + // Rule at src/isa/x64/lower.isle line 634. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = C::put_in_reg_mem_imm(ctx, pattern7_1); - let expr2_0 = constructor_reg_mem_imm_to_xmm(ctx, &expr1_0)?; + let expr2_0 = constructor_mov_rmi_to_xmm(ctx, &expr1_0)?; let expr3_0 = constructor_psllq(ctx, expr0_0, &expr2_0)?; - let expr4_0 = C::value_reg(ctx, expr3_0); + let expr4_0 = constructor_value_xmm(ctx, expr3_0)?; return Some(expr4_0); } &Opcode::Ushr => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 721. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); + // Rule at src/isa/x64/lower.isle line 733. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = C::put_in_reg_mem_imm(ctx, pattern7_1); - let expr2_0 = constructor_reg_mem_imm_to_xmm(ctx, &expr1_0)?; + let expr2_0 = constructor_mov_rmi_to_xmm(ctx, &expr1_0)?; let expr3_0 = constructor_psrlq(ctx, expr0_0, &expr2_0)?; - let expr4_0 = C::value_reg(ctx, expr3_0); + let expr4_0 = constructor_value_xmm(ctx, expr3_0)?; return Some(expr4_0); } &Opcode::Sshr => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 819. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); + // Rule at src/isa/x64/lower.isle line 833. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0: Type = I64; let expr2_0: u8 = 0; let expr3_0 = constructor_pextrd(ctx, expr1_0, expr0_0, expr2_0)?; @@ -3344,15 +3783,19 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option {} } @@ -3363,27 +3806,29 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { match &pattern5_0 { &Opcode::Ineg => { - // Rule at src/isa/x64/lower.isle line 929. + // Rule at src/isa/x64/lower.isle line 944. let expr0_0: Type = I64X2; let expr1_0: u64 = 0; let expr2_0 = constructor_imm(ctx, expr0_0, expr1_0)?; - let expr3_0 = C::put_in_reg_mem(ctx, pattern5_1); - let expr4_0 = constructor_psubq(ctx, expr2_0, &expr3_0)?; - let expr5_0 = C::value_reg(ctx, expr4_0); - return Some(expr5_0); + let expr3_0 = C::xmm_new(ctx, expr2_0); + let expr4_0 = constructor_put_in_xmm_mem(ctx, pattern5_1)?; + let expr5_0 = constructor_psubq(ctx, expr3_0, &expr4_0)?; + let expr6_0 = constructor_value_xmm(ctx, expr5_0)?; + return Some(expr6_0); } &Opcode::Iabs => { - // Rule at src/isa/x64/lower.isle line 1267. - let expr0_0 = C::put_in_reg(ctx, pattern5_1); + // Rule at src/isa/x64/lower.isle line 1282. + let expr0_0 = constructor_put_in_xmm(ctx, pattern5_1)?; let expr1_0: Type = I64X2; let expr2_0: u64 = 0; let expr3_0 = constructor_imm(ctx, expr1_0, expr2_0)?; - let expr4_0 = RegMem::Reg { reg: expr0_0 }; - let expr5_0 = constructor_psubq(ctx, expr3_0, &expr4_0)?; - let expr6_0 = RegMem::Reg { reg: expr0_0 }; - let expr7_0 = constructor_blendvpd(ctx, expr5_0, &expr6_0, expr5_0)?; - let expr8_0 = C::value_reg(ctx, expr7_0); - return Some(expr8_0); + let expr4_0 = C::xmm_new(ctx, expr3_0); + let expr5_0 = C::xmm_to_xmm_mem(ctx, expr0_0); + let expr6_0 = constructor_psubq(ctx, expr4_0, &expr5_0)?; + let expr7_0 = C::xmm_to_xmm_mem(ctx, expr0_0); + let expr8_0 = constructor_blendvpd(ctx, expr6_0, &expr7_0, expr6_0)?; + let expr9_0 = constructor_value_xmm(ctx, expr8_0)?; + return Some(expr9_0); } _ => {} } @@ -3399,17 +3844,18 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::BandNot = &pattern4_0 { let (pattern6_0, pattern6_1) = C::unpack_value_array_2(ctx, &pattern4_1); - // Rule at src/isa/x64/lower.isle line 1240. - let expr0_0 = C::put_in_reg(ctx, pattern6_1); - let expr1_0 = C::put_in_reg_mem(ctx, pattern6_0); + // Rule at src/isa/x64/lower.isle line 1255. + let expr0_0 = constructor_put_in_xmm(ctx, pattern6_1)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern6_0)?; let expr2_0 = constructor_sse_and_not(ctx, pattern2_0, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } } @@ -3478,11 +3925,11 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { let (pattern9_0, pattern9_1) = C::unpack_value_array_2(ctx, &pattern7_1); - // Rule at src/isa/x64/lower.isle line 935. - let expr0_0 = C::put_in_reg(ctx, pattern9_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); + // Rule at src/isa/x64/lower.isle line 950. + let expr0_0 = constructor_put_in_xmm(ctx, pattern9_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern9_1)?; let expr2_0 = constructor_pavgb(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::UaddSat => { let (pattern9_0, pattern9_1) = C::unpack_value_array_2(ctx, &pattern7_1); // Rule at src/isa/x64/lower.isle line 144. - let expr0_0 = C::put_in_reg(ctx, pattern9_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); + let expr0_0 = constructor_put_in_xmm(ctx, pattern9_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern9_1)?; let expr2_0 = constructor_paddusb(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::SaddSat => { let (pattern9_0, pattern9_1) = C::unpack_value_array_2(ctx, &pattern7_1); // Rule at src/isa/x64/lower.isle line 132. - let expr0_0 = C::put_in_reg(ctx, pattern9_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); + let expr0_0 = constructor_put_in_xmm(ctx, pattern9_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern9_1)?; let expr2_0 = constructor_paddsb(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::UsubSat => { let (pattern9_0, pattern9_1) = C::unpack_value_array_2(ctx, &pattern7_1); // Rule at src/isa/x64/lower.isle line 313. - let expr0_0 = C::put_in_reg(ctx, pattern9_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); + let expr0_0 = constructor_put_in_xmm(ctx, pattern9_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern9_1)?; let expr2_0 = constructor_psubusb(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::SsubSat => { let (pattern9_0, pattern9_1) = C::unpack_value_array_2(ctx, &pattern7_1); // Rule at src/isa/x64/lower.isle line 301. - let expr0_0 = C::put_in_reg(ctx, pattern9_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); + let expr0_0 = constructor_put_in_xmm(ctx, pattern9_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern9_1)?; let expr2_0 = constructor_psubsb(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::Iadd => { let (pattern9_0, pattern9_1) = C::unpack_value_array_2(ctx, &pattern7_1); // Rule at src/isa/x64/lower.isle line 96. - let expr0_0 = C::put_in_reg(ctx, pattern9_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); + let expr0_0 = constructor_put_in_xmm(ctx, pattern9_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern9_1)?; let expr2_0 = constructor_paddb(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::Isub => { let (pattern9_0, pattern9_1) = C::unpack_value_array_2(ctx, &pattern7_1); // Rule at src/isa/x64/lower.isle line 265. - let expr0_0 = C::put_in_reg(ctx, pattern9_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); + let expr0_0 = constructor_put_in_xmm(ctx, pattern9_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern9_1)?; let expr2_0 = constructor_psubb(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } _ => {} @@ -3606,71 +4053,71 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern9_0, pattern9_1) = C::unpack_value_array_2(ctx, &pattern7_1); - // Rule at src/isa/x64/lower.isle line 939. - let expr0_0 = C::put_in_reg(ctx, pattern9_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); + // Rule at src/isa/x64/lower.isle line 954. + let expr0_0 = constructor_put_in_xmm(ctx, pattern9_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern9_1)?; let expr2_0 = constructor_pavgw(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::UaddSat => { let (pattern9_0, pattern9_1) = C::unpack_value_array_2(ctx, &pattern7_1); // Rule at src/isa/x64/lower.isle line 149. - let expr0_0 = C::put_in_reg(ctx, pattern9_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); + let expr0_0 = constructor_put_in_xmm(ctx, pattern9_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern9_1)?; let expr2_0 = constructor_paddusw(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::SaddSat => { let (pattern9_0, pattern9_1) = C::unpack_value_array_2(ctx, &pattern7_1); // Rule at src/isa/x64/lower.isle line 137. - let expr0_0 = C::put_in_reg(ctx, pattern9_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); + let expr0_0 = constructor_put_in_xmm(ctx, pattern9_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern9_1)?; let expr2_0 = constructor_paddsw(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::UsubSat => { let (pattern9_0, pattern9_1) = C::unpack_value_array_2(ctx, &pattern7_1); // Rule at src/isa/x64/lower.isle line 318. - let expr0_0 = C::put_in_reg(ctx, pattern9_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); + let expr0_0 = constructor_put_in_xmm(ctx, pattern9_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern9_1)?; let expr2_0 = constructor_psubusw(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::SsubSat => { let (pattern9_0, pattern9_1) = C::unpack_value_array_2(ctx, &pattern7_1); // Rule at src/isa/x64/lower.isle line 306. - let expr0_0 = C::put_in_reg(ctx, pattern9_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); + let expr0_0 = constructor_put_in_xmm(ctx, pattern9_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern9_1)?; let expr2_0 = constructor_psubsw(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::Iadd => { let (pattern9_0, pattern9_1) = C::unpack_value_array_2(ctx, &pattern7_1); // Rule at src/isa/x64/lower.isle line 101. - let expr0_0 = C::put_in_reg(ctx, pattern9_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); + let expr0_0 = constructor_put_in_xmm(ctx, pattern9_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern9_1)?; let expr2_0 = constructor_paddw(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::Isub => { let (pattern9_0, pattern9_1) = C::unpack_value_array_2(ctx, &pattern7_1); // Rule at src/isa/x64/lower.isle line 270. - let expr0_0 = C::put_in_reg(ctx, pattern9_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); + let expr0_0 = constructor_put_in_xmm(ctx, pattern9_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern9_1)?; let expr2_0 = constructor_psubw(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::Imul => { @@ -3719,16 +4166,14 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option {} @@ -3981,20 +4404,20 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern9_0, pattern9_1) = C::unpack_value_array_2(ctx, &pattern7_1); // Rule at src/isa/x64/lower.isle line 275. - let expr0_0 = C::put_in_reg(ctx, pattern9_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); + let expr0_0 = constructor_put_in_xmm(ctx, pattern9_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern9_1)?; let expr2_0 = constructor_psubd(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::Imul => { @@ -4043,22 +4466,16 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option {} @@ -4293,20 +4692,20 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern9_0, pattern9_1) = C::unpack_value_array_2(ctx, &pattern7_1); // Rule at src/isa/x64/lower.isle line 280. - let expr0_0 = C::put_in_reg(ctx, pattern9_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); + let expr0_0 = constructor_put_in_xmm(ctx, pattern9_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern9_1)?; let expr2_0 = constructor_psubq(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::Imul => { @@ -4355,20 +4754,18 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option {} } @@ -4613,30 +5007,30 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 440. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); + // Rule at src/isa/x64/lower.isle line 442. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_sse_or(ctx, pattern2_0, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::Bxor => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 509. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); + // Rule at src/isa/x64/lower.isle line 513. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_sse_xor(ctx, pattern2_0, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } _ => {} @@ -4650,30 +5044,30 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1, pattern7_2) = C::unpack_value_array_3(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1314. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); - let expr1_0 = C::put_in_reg(ctx, pattern7_1); - let expr2_0 = RegMem::Reg { reg: expr0_0 }; + // Rule at src/isa/x64/lower.isle line 1331. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; + let expr1_0 = constructor_put_in_xmm(ctx, pattern7_1)?; + let expr2_0 = C::xmm_to_xmm_mem(ctx, expr0_0); let expr3_0 = constructor_sse_and(ctx, pattern2_0, expr1_0, &expr2_0)?; - let expr4_0 = C::put_in_reg_mem(ctx, pattern7_2); + let expr4_0 = constructor_put_in_xmm_mem(ctx, pattern7_2)?; let expr5_0 = constructor_sse_and_not(ctx, pattern2_0, expr0_0, &expr4_0)?; - let expr6_0 = RegMem::Reg { reg: expr3_0 }; + let expr6_0 = C::xmm_to_xmm_mem(ctx, expr3_0); let expr7_0 = constructor_sse_or(ctx, pattern2_0, expr5_0, &expr6_0)?; - let expr8_0 = C::value_reg(ctx, expr7_0); + let expr8_0 = constructor_value_xmm(ctx, expr7_0)?; return Some(expr8_0); } &Opcode::Vselect => { let (pattern7_0, pattern7_1, pattern7_2) = C::unpack_value_array_3(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1328. - let expr0_0 = C::put_in_reg_mem(ctx, pattern7_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); - let expr2_0 = C::put_in_reg(ctx, pattern7_2); + // Rule at src/isa/x64/lower.isle line 1345. + let expr0_0 = constructor_put_in_xmm_mem(ctx, pattern7_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; + let expr2_0 = constructor_put_in_xmm(ctx, pattern7_2)?; let expr3_0 = constructor_sse_blend( ctx, pattern2_0, &expr0_0, &expr1_0, expr2_0, )?; - let expr4_0 = C::value_reg(ctx, expr3_0); + let expr4_0 = constructor_value_xmm(ctx, expr3_0)?; return Some(expr4_0); } _ => {} @@ -4684,12 +5078,12 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::Bnot = &pattern5_0 { - // Rule at src/isa/x64/lower.isle line 1309. - let expr0_0 = C::put_in_reg(ctx, pattern5_1); + // Rule at src/isa/x64/lower.isle line 1326. + let expr0_0 = constructor_put_in_xmm(ctx, pattern5_1)?; let expr1_0 = constructor_vector_all_ones(ctx, pattern2_0)?; - let expr2_0 = RegMem::Reg { reg: expr1_0 }; + let expr2_0 = C::xmm_to_xmm_mem(ctx, expr1_0); let expr3_0 = constructor_sse_xor(ctx, pattern2_0, expr0_0, &expr2_0)?; - let expr4_0 = C::value_reg(ctx, expr3_0); + let expr4_0 = constructor_value_xmm(ctx, expr3_0)?; return Some(expr4_0); } } @@ -4811,7 +5205,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 539. + // Rule at src/isa/x64/lower.isle line 543. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_masked_in_imm8_reg(ctx, pattern7_1, pattern3_0); let expr2_0 = constructor_shl(ctx, pattern3_0, expr0_0, &expr1_0)?; @@ -5052,7 +5446,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 635. + // Rule at src/isa/x64/lower.isle line 642. let expr0_0 = ExtendKind::Zero; let expr1_0 = constructor_extend_to_reg(ctx, pattern7_0, pattern3_0, &expr0_0)?; @@ -5064,7 +5458,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 729. + // Rule at src/isa/x64/lower.isle line 741. let expr0_0 = ExtendKind::Sign; let expr1_0 = constructor_extend_to_reg(ctx, pattern7_0, pattern3_0, &expr0_0)?; @@ -5082,17 +5476,17 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { match &pattern5_0 { &Opcode::Ineg => { - // Rule at src/isa/x64/lower.isle line 912. - let expr0_0 = C::put_in_reg(ctx, pattern5_1); + // Rule at src/isa/x64/lower.isle line 927. + let expr0_0 = constructor_put_in_gpr(ctx, pattern5_1)?; let expr1_0 = constructor_neg(ctx, pattern3_0, expr0_0)?; - let expr2_0 = C::value_reg(ctx, expr1_0); + let expr2_0 = constructor_value_gpr(ctx, expr1_0)?; return Some(expr2_0); } &Opcode::Bnot => { - // Rule at src/isa/x64/lower.isle line 1288. - let expr0_0 = C::put_in_reg(ctx, pattern5_1); + // Rule at src/isa/x64/lower.isle line 1305. + let expr0_0 = constructor_put_in_gpr(ctx, pattern5_1)?; let expr1_0 = constructor_not(ctx, pattern3_0, expr0_0)?; - let expr2_0 = C::value_reg(ctx, expr1_0); + let expr2_0 = constructor_value_gpr(ctx, expr1_0)?; return Some(expr2_0); } _ => {} @@ -5136,7 +5530,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option( ctx: &mut C, arg0: Type, - arg1: Reg, - arg2: &RegMem, -) -> Option { + arg1: Xmm, + arg2: &XmmMem, +) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32X4 { let pattern2_0 = arg1; @@ -5307,28 +5701,28 @@ pub fn constructor_sse_and( pub fn constructor_sse_or( ctx: &mut C, arg0: Type, - arg1: Reg, - arg2: &RegMem, -) -> Option { + arg1: Xmm, + arg2: &XmmMem, +) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32X4 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/lower.isle line 436. + // Rule at src/isa/x64/lower.isle line 438. let expr0_0 = constructor_orps(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if pattern0_0 == F64X2 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/lower.isle line 437. + // Rule at src/isa/x64/lower.isle line 439. let expr0_0 = constructor_orpd(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/lower.isle line 438. + // Rule at src/isa/x64/lower.isle line 440. let expr0_0 = constructor_por(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } @@ -5343,7 +5737,7 @@ pub fn constructor_or_i128( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/lower.isle line 447. + // Rule at src/isa/x64/lower.isle line 451. let expr0_0: usize = 0; let expr1_0 = C::value_regs_get(ctx, pattern0_0, expr0_0); let expr2_0: usize = 1; @@ -5370,7 +5764,7 @@ pub fn constructor_shl_i128( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/lower.isle line 545. + // Rule at src/isa/x64/lower.isle line 549. let expr0_0: usize = 0; let expr1_0 = C::value_regs_get(ctx, pattern0_0, expr0_0); let expr2_0: usize = 1; @@ -5429,12 +5823,12 @@ pub fn constructor_ishl_i8x16_mask( let pattern0_0 = arg0; match pattern0_0 { &RegMemImm::Imm { simm32: pattern1_0 } => { - // Rule at src/isa/x64/lower.isle line 600. + // Rule at src/isa/x64/lower.isle line 604. let expr0_0 = C::ishl_i8x16_mask_for_const(ctx, pattern1_0); return Some(expr0_0); } &RegMemImm::Reg { reg: pattern1_0 } => { - // Rule at src/isa/x64/lower.isle line 609. + // Rule at src/isa/x64/lower.isle line 613. let expr0_0 = C::ishl_i8x16_mask_table(ctx); let expr1_0 = constructor_lea(ctx, &expr0_0)?; let expr2_0: Type = I64; @@ -5442,15 +5836,16 @@ pub fn constructor_ishl_i8x16_mask( let expr4_0 = Imm8Reg::Imm8 { imm: expr3_0 }; let expr5_0 = constructor_shl(ctx, expr2_0, pattern1_0, &expr4_0)?; let expr6_0: u32 = 0; - let expr7_0: u8 = 0; - let expr8_0 = C::amode_imm_reg_reg_shift(ctx, expr6_0, expr1_0, expr5_0, expr7_0); - let expr9_0 = C::amode_to_synthetic_amode(ctx, &expr8_0); - return Some(expr9_0); + let expr7_0 = C::gpr_new(ctx, expr5_0); + let expr8_0: u8 = 0; + let expr9_0 = C::amode_imm_reg_reg_shift(ctx, expr6_0, expr1_0, expr7_0, expr8_0); + let expr10_0 = C::amode_to_synthetic_amode(ctx, &expr9_0); + return Some(expr10_0); } &RegMemImm::Mem { addr: ref pattern1_0, } => { - // Rule at src/isa/x64/lower.isle line 617. + // Rule at src/isa/x64/lower.isle line 621. let expr0_0: Type = I64; let expr1_0 = ExtKind::None; let expr2_0 = constructor_x64_load(ctx, expr0_0, &pattern1_0, &expr1_0)?; @@ -5471,7 +5866,7 @@ pub fn constructor_shr_i128( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/lower.isle line 642. + // Rule at src/isa/x64/lower.isle line 649. let expr0_0: usize = 0; let expr1_0 = C::value_regs_get(ctx, pattern0_0, expr0_0); let expr2_0: usize = 1; @@ -5533,12 +5928,12 @@ pub fn constructor_ushr_i8x16_mask( let pattern0_0 = arg0; match pattern0_0 { &RegMemImm::Imm { simm32: pattern1_0 } => { - // Rule at src/isa/x64/lower.isle line 694. + // Rule at src/isa/x64/lower.isle line 703. let expr0_0 = C::ushr_i8x16_mask_for_const(ctx, pattern1_0); return Some(expr0_0); } &RegMemImm::Reg { reg: pattern1_0 } => { - // Rule at src/isa/x64/lower.isle line 703. + // Rule at src/isa/x64/lower.isle line 712. let expr0_0 = C::ushr_i8x16_mask_table(ctx); let expr1_0 = constructor_lea(ctx, &expr0_0)?; let expr2_0: Type = I64; @@ -5546,15 +5941,16 @@ pub fn constructor_ushr_i8x16_mask( let expr4_0 = Imm8Reg::Imm8 { imm: expr3_0 }; let expr5_0 = constructor_shl(ctx, expr2_0, pattern1_0, &expr4_0)?; let expr6_0: u32 = 0; - let expr7_0: u8 = 0; - let expr8_0 = C::amode_imm_reg_reg_shift(ctx, expr6_0, expr1_0, expr5_0, expr7_0); - let expr9_0 = C::amode_to_synthetic_amode(ctx, &expr8_0); - return Some(expr9_0); + let expr7_0 = C::gpr_new(ctx, expr5_0); + let expr8_0: u8 = 0; + let expr9_0 = C::amode_imm_reg_reg_shift(ctx, expr6_0, expr1_0, expr7_0, expr8_0); + let expr10_0 = C::amode_to_synthetic_amode(ctx, &expr9_0); + return Some(expr10_0); } &RegMemImm::Mem { addr: ref pattern1_0, } => { - // Rule at src/isa/x64/lower.isle line 711. + // Rule at src/isa/x64/lower.isle line 720. let expr0_0: Type = I64; let expr1_0 = ExtKind::None; let expr2_0 = constructor_x64_load(ctx, expr0_0, &pattern1_0, &expr1_0)?; @@ -5575,7 +5971,7 @@ pub fn constructor_sar_i128( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/lower.isle line 736. + // Rule at src/isa/x64/lower.isle line 748. let expr0_0: usize = 0; let expr1_0 = C::value_regs_get(ctx, pattern0_0, expr0_0); let expr2_0: usize = 1; @@ -5635,35 +6031,36 @@ pub fn constructor_sshr_i8x16_bigger_shift( ctx: &mut C, arg0: Type, arg1: &RegMemImm, -) -> Option { +) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; match pattern1_0 { &RegMemImm::Imm { simm32: pattern2_0 } => { - // Rule at src/isa/x64/lower.isle line 795. + // Rule at src/isa/x64/lower.isle line 807. let expr0_0: u32 = 8; let expr1_0 = C::u32_add(ctx, pattern2_0, expr0_0); let expr2_0 = RegMemImm::Imm { simm32: expr1_0 }; - return Some(expr2_0); + let expr3_0 = C::xmm_mem_imm_new(ctx, &expr2_0); + return Some(expr3_0); } &RegMemImm::Reg { reg: pattern2_0 } => { - // Rule at src/isa/x64/lower.isle line 797. + // Rule at src/isa/x64/lower.isle line 809. let expr0_0: u32 = 8; let expr1_0 = RegMemImm::Imm { simm32: expr0_0 }; let expr2_0 = constructor_add(ctx, pattern0_0, pattern2_0, &expr1_0)?; let expr3_0 = RegMemImm::Reg { reg: expr2_0 }; - let expr4_0 = constructor_reg_mem_imm_to_xmm(ctx, &expr3_0)?; + let expr4_0 = constructor_mov_rmi_to_xmm(ctx, &expr3_0)?; return Some(expr4_0); } &RegMemImm::Mem { addr: ref pattern2_0, } => { - // Rule at src/isa/x64/lower.isle line 799. + // Rule at src/isa/x64/lower.isle line 811. let expr0_0: u64 = 8; let expr1_0 = constructor_imm(ctx, pattern0_0, expr0_0)?; let expr2_0 = constructor_add(ctx, pattern0_0, expr1_0, pattern1_0)?; let expr3_0 = RegMemImm::Reg { reg: expr2_0 }; - let expr4_0 = constructor_reg_mem_imm_to_xmm(ctx, &expr3_0)?; + let expr4_0 = constructor_mov_rmi_to_xmm(ctx, &expr3_0)?; return Some(expr4_0); } _ => {} @@ -5675,28 +6072,28 @@ pub fn constructor_sshr_i8x16_bigger_shift( pub fn constructor_sse_and_not( ctx: &mut C, arg0: Type, - arg1: Reg, - arg2: &RegMem, -) -> Option { + arg1: Xmm, + arg2: &XmmMem, +) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32X4 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/lower.isle line 1229. + // Rule at src/isa/x64/lower.isle line 1244. let expr0_0 = constructor_andnps(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if pattern0_0 == F64X2 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/lower.isle line 1230. + // Rule at src/isa/x64/lower.isle line 1245. let expr0_0 = constructor_andnpd(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/lower.isle line 1231. + // Rule at src/isa/x64/lower.isle line 1246. let expr0_0 = constructor_pandn(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } @@ -5706,71 +6103,78 @@ pub fn constructor_sse_and_not( // Generated as internal constructor for term i128_not. pub fn constructor_i128_not(ctx: &mut C, arg0: Value) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/lower.isle line 1294. + // Rule at src/isa/x64/lower.isle line 1311. let expr0_0 = C::put_in_regs(ctx, pattern0_0); let expr1_0: usize = 0; let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); - let expr3_0: usize = 1; - let expr4_0 = C::value_regs_get(ctx, expr0_0, expr3_0); - let expr5_0: Type = I64; - let expr6_0 = constructor_not(ctx, expr5_0, expr2_0)?; + let expr3_0 = C::gpr_new(ctx, expr2_0); + let expr4_0: usize = 1; + let expr5_0 = C::value_regs_get(ctx, expr0_0, expr4_0); + let expr6_0 = C::gpr_new(ctx, expr5_0); let expr7_0: Type = I64; - let expr8_0 = constructor_not(ctx, expr7_0, expr4_0)?; - let expr9_0 = C::value_regs(ctx, expr6_0, expr8_0); - return Some(expr9_0); + let expr8_0 = constructor_not(ctx, expr7_0, expr3_0)?; + let expr9_0: Type = I64; + let expr10_0 = constructor_not(ctx, expr9_0, expr6_0)?; + let expr11_0 = constructor_value_gprs(ctx, expr8_0, expr10_0)?; + return Some(expr11_0); } // Generated as internal constructor for term vec_insert_lane. pub fn constructor_vec_insert_lane( ctx: &mut C, arg0: Type, - arg1: Reg, + arg1: Xmm, arg2: &RegMem, arg3: u8, -) -> Option { +) -> Option { let pattern0_0 = arg0; if pattern0_0 == I8X16 { let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/lower.isle line 1348. - let expr0_0 = constructor_pinsrb(ctx, pattern2_0, pattern3_0, pattern4_0)?; - return Some(expr0_0); + // Rule at src/isa/x64/lower.isle line 1365. + let expr0_0 = C::gpr_mem_new(ctx, pattern3_0); + let expr1_0 = constructor_pinsrb(ctx, pattern2_0, &expr0_0, pattern4_0)?; + return Some(expr1_0); } if pattern0_0 == I16X8 { let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/lower.isle line 1351. - let expr0_0 = constructor_pinsrw(ctx, pattern2_0, pattern3_0, pattern4_0)?; - return Some(expr0_0); + // Rule at src/isa/x64/lower.isle line 1369. + let expr0_0 = C::gpr_mem_new(ctx, pattern3_0); + let expr1_0 = constructor_pinsrw(ctx, pattern2_0, &expr0_0, pattern4_0)?; + return Some(expr1_0); } if pattern0_0 == I32X4 { let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/lower.isle line 1354. - let expr0_0 = OperandSize::Size32; - let expr1_0 = constructor_pinsrd(ctx, pattern2_0, pattern3_0, pattern4_0, &expr0_0)?; - return Some(expr1_0); + // Rule at src/isa/x64/lower.isle line 1373. + let expr0_0 = C::gpr_mem_new(ctx, pattern3_0); + let expr1_0 = OperandSize::Size32; + let expr2_0 = constructor_pinsrd(ctx, pattern2_0, &expr0_0, pattern4_0, &expr1_0)?; + return Some(expr2_0); } if pattern0_0 == I64X2 { let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/lower.isle line 1357. - let expr0_0 = OperandSize::Size64; - let expr1_0 = constructor_pinsrd(ctx, pattern2_0, pattern3_0, pattern4_0, &expr0_0)?; - return Some(expr1_0); + // Rule at src/isa/x64/lower.isle line 1377. + let expr0_0 = C::gpr_mem_new(ctx, pattern3_0); + let expr1_0 = OperandSize::Size64; + let expr2_0 = constructor_pinsrd(ctx, pattern2_0, &expr0_0, pattern4_0, &expr1_0)?; + return Some(expr2_0); } if pattern0_0 == F32X4 { let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/lower.isle line 1360. - let expr0_0 = C::sse_insertps_lane_imm(ctx, pattern4_0); - let expr1_0 = constructor_insertps(ctx, pattern2_0, pattern3_0, expr0_0)?; - return Some(expr1_0); + // Rule at src/isa/x64/lower.isle line 1381. + let expr0_0 = C::xmm_mem_new(ctx, pattern3_0); + let expr1_0 = C::sse_insertps_lane_imm(ctx, pattern4_0); + let expr2_0 = constructor_insertps(ctx, pattern2_0, &expr0_0, expr1_0)?; + return Some(expr2_0); } if pattern0_0 == F64X2 { let pattern2_0 = arg1; @@ -5778,25 +6182,28 @@ pub fn constructor_vec_insert_lane( if let &RegMem::Reg { reg: pattern4_0 } = pattern3_0 { let pattern5_0 = arg3; if pattern5_0 == 0 { - // Rule at src/isa/x64/lower.isle line 1381. + // Rule at src/isa/x64/lower.isle line 1403. let expr0_0 = RegMem::Reg { reg: pattern4_0 }; - let expr1_0 = constructor_movsd(ctx, pattern2_0, &expr0_0)?; - return Some(expr1_0); + let expr1_0 = C::xmm_mem_new(ctx, &expr0_0); + let expr2_0 = constructor_movsd(ctx, pattern2_0, &expr1_0)?; + return Some(expr2_0); } } let pattern4_0 = arg3; if pattern4_0 == 0 { - // Rule at src/isa/x64/lower.isle line 1382. + // Rule at src/isa/x64/lower.isle line 1405. let expr0_0 = SseOpcode::Movsd; - let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern3_0)?; - let expr2_0 = RegMem::Reg { reg: expr1_0 }; - let expr3_0 = constructor_movsd(ctx, pattern2_0, &expr2_0)?; - return Some(expr3_0); + let expr1_0 = C::xmm_mem_new(ctx, pattern3_0); + let expr2_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, &expr1_0)?; + let expr3_0 = C::xmm_to_xmm_mem(ctx, expr2_0); + let expr4_0 = constructor_movsd(ctx, pattern2_0, &expr3_0)?; + return Some(expr4_0); } if pattern4_0 == 1 { - // Rule at src/isa/x64/lower.isle line 1390. - let expr0_0 = constructor_movlhps(ctx, pattern2_0, pattern3_0)?; - return Some(expr0_0); + // Rule at src/isa/x64/lower.isle line 1414. + let expr0_0 = C::xmm_mem_new(ctx, pattern3_0); + let expr1_0 = constructor_movlhps(ctx, pattern2_0, &expr0_0)?; + return Some(expr1_0); } } return None; diff --git a/cranelift/codegen/src/machinst/isle.rs b/cranelift/codegen/src/machinst/isle.rs index 369f3d1f6616..e33cbb4f084e 100644 --- a/cranelift/codegen/src/machinst/isle.rs +++ b/cranelift/codegen/src/machinst/isle.rs @@ -1,14 +1,25 @@ use crate::ir::{Inst, Value}; use crate::machinst::{get_output_reg, InsnOutput, LowerCtx, MachInst, RegRenamer}; +use alloc::boxed::Box; +use alloc::vec::Vec; use regalloc::{Reg, Writable}; use smallvec::SmallVec; +pub use super::MachLabel; +pub use crate::ir::ExternalName; +pub use crate::isa::unwind::UnwindInst; + pub type Unit = (); pub type ValueSlice<'a> = &'a [Value]; pub type ValueArray2 = [Value; 2]; pub type ValueArray3 = [Value; 3]; pub type WritableReg = Writable; +pub type OptionWritableReg = Option; +pub type VecReg = Vec; +pub type VecWritableReg = Vec; pub type ValueRegs = crate::machinst::ValueRegs; +pub type VecMachLabel = Vec; +pub type BoxExternalName = Box; /// Helper macro to define methods in `prelude.isle` within `impl Context for /// ...` for each backend. These methods are shared amongst all backends. diff --git a/cranelift/codegen/src/prelude.isle b/cranelift/codegen/src/prelude.isle index c5a037b6579c..8dfeaefb571c 100644 --- a/cranelift/codegen/src/prelude.isle +++ b/cranelift/codegen/src/prelude.isle @@ -48,6 +48,9 @@ (type Reg (primitive Reg)) (type WritableReg (primitive WritableReg)) +(type OptionWritableReg (primitive OptionWritableReg)) +(type VecReg extern (enum)) +(type VecWritableReg extern (enum)) ;; Construct a `ValueRegs` of one register. (decl value_reg (Reg) ValueRegs) @@ -106,6 +109,15 @@ (let ((regs ValueRegs (put_in_regs val))) (value_regs_get regs 0))) +;;;; Common Mach Types ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(type MachLabel (primitive MachLabel)) +(type VecMachLabel extern (enum)) +(type ValueLabel (primitive ValueLabel)) +(type UnwindInst (primitive UnwindInst)) +(type ExternalName (primitive ExternalName)) +(type BoxExternalName (primitive BoxExternalName)) + ;;;; Primitive Type Conversions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (decl u8_as_u64 (u8) u64) @@ -349,4 +361,3 @@ (decl avoid_div_traps () Type) (extern extractor avoid_div_traps avoid_div_traps) -