From 9cdf5a473cb5cb5b2d2c53337047ec9d3b3e475b Mon Sep 17 00:00:00 2001 From: Nick Fitzgerald Date: Fri, 21 Jan 2022 14:10:40 -0800 Subject: [PATCH] cranelift: Add newtype wrappers for x64 register classes This primary motivation of this large commit (apologies for its size!) is to introduce `Gpr` and `Xmm` newtypes over `Reg`. This should help catch difficult-to-diagnose register class mixup bugs in x64 lowerings. But having a newtype for `Gpr` and `Xmm` themselves isn't enough to catch all of our operand-with-wrong-register-class bugs, because about 50% of operands on x64 aren't just a register, but a register or memory address or even an immediate! So we have `{Gpr,Xmm}Mem[Imm]` newtypes as well. Unfortunately, `GprMem` et al can't be `enum`s and are therefore a little bit noisier to work with from ISLE. They need to maintain the invariant that their registers really are of the claimed register class, so they need to encapsulate the inner data. If they exposed the underlying `enum` variants, then anyone could just change register classes or construct a `GprMem` that holds an XMM register, defeating the whole point of these newtypes. So when working with these newtypes from ISLE, we rely on external constructors like `(gpr_to_gpr_mem my_gpr)` instead of `(GprMem.Gpr my_gpr)`. A bit of extra lines of code are included to add support for register mapping for all of these newtypes as well. Ultimately this is all a bit wordier than I'd hoped it would be when I first started authoring this commit, but I think it is all worth it nonetheless! In the process of adding these newtypes, I didn't want to have to update both the ISLE `extern` type definition of `MInst` and the Rust definition, so I move the definition fully into ISLE, similar as aarch64. Finally, this process isn't complete. I've introduced the newtypes here, and I've made most XMM-using instructions switch from `Reg` to `Xmm`, as well as register class-converting instructions, but I haven't moved all of the GPR-using instructions over to the newtypes yet. I figured this commit was big enough as it was, and I can continue the adoption of these newtypes in follow up commits. Part of #3685. --- Cargo.lock | 4 +- cranelift/codegen/Cargo.toml | 2 +- cranelift/codegen/build.rs | 2 + cranelift/codegen/src/isa/aarch64/inst.isle | 5 - .../lower/isle/generated_code.manifest | 4 +- .../isa/aarch64/lower/isle/generated_code.rs | 432 +-- cranelift/codegen/src/isa/s390x/inst.isle | 7 - .../s390x/lower/isle/generated_code.manifest | 4 +- .../isa/s390x/lower/isle/generated_code.rs | 910 +++--- cranelift/codegen/src/isa/x64/encoding/rex.rs | 6 +- cranelift/codegen/src/isa/x64/inst.isle | 1246 ++++++-- cranelift/codegen/src/isa/x64/inst/args.rs | 315 +- cranelift/codegen/src/isa/x64/inst/emit.rs | 240 +- cranelift/codegen/src/isa/x64/inst/mod.rs | 769 +---- cranelift/codegen/src/isa/x64/lower.isle | 449 +-- cranelift/codegen/src/isa/x64/lower.rs | 26 +- cranelift/codegen/src/isa/x64/lower/isle.rs | 138 +- .../x64/lower/isle/generated_code.manifest | 6 +- .../src/isa/x64/lower/isle/generated_code.rs | 2749 ++++++++++------- cranelift/codegen/src/machinst/isle.rs | 11 + cranelift/codegen/src/prelude.isle | 13 +- 21 files changed, 4231 insertions(+), 3107 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 488e4f7b3aa7..a2eda03a5480 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2350,9 +2350,9 @@ dependencies = [ [[package]] name = "regalloc" -version = "0.0.33" +version = "0.0.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d808cff91dfca7b239d40b972ba628add94892b1d9e19a842aedc5cfae8ab1a" +checksum = "62446b1d3ebf980bdc68837700af1d77b37bc430e524bf95319c6eada2a4cc02" dependencies = [ "log", "rustc-hash", diff --git a/cranelift/codegen/Cargo.toml b/cranelift/codegen/Cargo.toml index 3e20edcc8d56..27979797bf9d 100644 --- a/cranelift/codegen/Cargo.toml +++ b/cranelift/codegen/Cargo.toml @@ -23,7 +23,7 @@ serde = { version = "1.0.94", features = ["derive"], optional = true } bincode = { version = "1.2.1", optional = true } gimli = { version = "0.26.0", default-features = false, features = ["write"], optional = true } smallvec = { version = "1.6.1" } -regalloc = { version = "0.0.33" } +regalloc = "0.0.34" souper-ir = { version = "2.1.0", optional = true } # It is a goal of the cranelift-codegen crate to have minimal external dependencies. # Please don't add any unless they are essential to the task of creating binary diff --git a/cranelift/codegen/build.rs b/cranelift/codegen/build.rs index cb704cb0abbb..1b95bf02a834 100644 --- a/cranelift/codegen/build.rs +++ b/cranelift/codegen/build.rs @@ -406,6 +406,8 @@ fn rebuild_isle( ) -> Result<(), Box> { use cranelift_isle as isle; + println!("Rebuilding {}", compilation.output.display()); + // First, remove the manifest, if any; we will recreate it // below if the compilation is successful. Ignore error if no // manifest was present. diff --git a/cranelift/codegen/src/isa/aarch64/inst.isle b/cranelift/codegen/src/isa/aarch64/inst.isle index 839b16f7bfa7..c64922858736 100644 --- a/cranelift/codegen/src/isa/aarch64/inst.isle +++ b/cranelift/codegen/src/isa/aarch64/inst.isle @@ -871,15 +871,10 @@ (type BoxCallInfo (primitive BoxCallInfo)) (type BoxCallIndInfo (primitive BoxCallIndInfo)) -(type VecMachLabel (primitive VecMachLabel)) (type CondBrKind (primitive CondBrKind)) (type BranchTarget (primitive BranchTarget)) (type BoxJTSequenceInfo (primitive BoxJTSequenceInfo)) -(type BoxExternalName (primitive BoxExternalName)) (type CodeOffset (primitive CodeOffset)) -(type ExternalName (primitive ExternalName)) -(type ValueLabel (primitive ValueLabel)) -(type UnwindInst (primitive UnwindInst)) (type ExtendOp extern (enum diff --git a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest index 2b6effae74bd..831df6d3cc36 100644 --- a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest +++ b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest @@ -1,4 +1,4 @@ src/clif.isle 9ea75a6f790b5c03 -src/prelude.isle 51d2aef2566c1c96 -src/isa/aarch64/inst.isle f946561093de4ff5 +src/prelude.isle 3d00cfa50c4d52d9 +src/isa/aarch64/inst.isle 13b123492c3c859d src/isa/aarch64/lower.isle 2d2e1e076a0c8a23 diff --git a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs index 231c90bbc184..0f13e498acb0 100644 --- a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs +++ b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs @@ -95,19 +95,19 @@ pub trait Context { fn rotr_opposite_amount(&mut self, arg0: Type, arg1: ImmShift) -> ImmShift; } -/// Internal type SideEffectNoResult: defined at src/prelude.isle line 282. +/// Internal type SideEffectNoResult: defined at src/prelude.isle line 294. #[derive(Clone, Debug)] pub enum SideEffectNoResult { Inst { inst: MInst }, } -/// Internal type ProducesFlags: defined at src/prelude.isle line 295. +/// Internal type ProducesFlags: defined at src/prelude.isle line 307. #[derive(Clone, Debug)] pub enum ProducesFlags { ProducesFlags { inst: MInst, result: Reg }, } -/// Internal type ConsumesFlags: defined at src/prelude.isle line 298. +/// Internal type ConsumesFlags: defined at src/prelude.isle line 310. #[derive(Clone, Debug)] pub enum ConsumesFlags { ConsumesFlags { inst: MInst, result: Reg }, @@ -682,7 +682,7 @@ pub enum MInst { } /// Internal type ALUOp: defined at src/isa/aarch64/inst.isle line 789. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum ALUOp { Add32, Add64, @@ -729,7 +729,7 @@ pub enum ALUOp { } /// Internal type ALUOp3: defined at src/isa/aarch64/inst.isle line 850. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum ALUOp3 { MAdd32, MAdd64, @@ -737,8 +737,8 @@ pub enum ALUOp3 { MSub64, } -/// Internal type BitOp: defined at src/isa/aarch64/inst.isle line 898. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type BitOp: defined at src/isa/aarch64/inst.isle line 893. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum BitOp { RBit32, RBit64, @@ -748,8 +748,8 @@ pub enum BitOp { Cls64, } -/// Internal type FPUOp1: defined at src/isa/aarch64/inst.isle line 964. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type FPUOp1: defined at src/isa/aarch64/inst.isle line 959. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum FPUOp1 { Abs32, Abs64, @@ -761,8 +761,8 @@ pub enum FPUOp1 { Cvt64To32, } -/// Internal type FPUOp2: defined at src/isa/aarch64/inst.isle line 977. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type FPUOp2: defined at src/isa/aarch64/inst.isle line 972. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum FPUOp2 { Add32, Add64, @@ -782,15 +782,15 @@ pub enum FPUOp2 { Uqsub64, } -/// Internal type FPUOp3: defined at src/isa/aarch64/inst.isle line 1002. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type FPUOp3: defined at src/isa/aarch64/inst.isle line 997. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum FPUOp3 { MAdd32, MAdd64, } -/// Internal type FpuToIntOp: defined at src/isa/aarch64/inst.isle line 1009. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type FpuToIntOp: defined at src/isa/aarch64/inst.isle line 1004. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum FpuToIntOp { F32ToU32, F32ToI32, @@ -802,8 +802,8 @@ pub enum FpuToIntOp { F64ToI64, } -/// Internal type IntToFpuOp: defined at src/isa/aarch64/inst.isle line 1022. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type IntToFpuOp: defined at src/isa/aarch64/inst.isle line 1017. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum IntToFpuOp { U32ToF32, I32ToF32, @@ -815,8 +815,8 @@ pub enum IntToFpuOp { I64ToF64, } -/// Internal type FpuRoundMode: defined at src/isa/aarch64/inst.isle line 1036. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type FpuRoundMode: defined at src/isa/aarch64/inst.isle line 1031. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum FpuRoundMode { Minus32, Minus64, @@ -828,8 +828,8 @@ pub enum FpuRoundMode { Nearest64, } -/// Internal type VecExtendOp: defined at src/isa/aarch64/inst.isle line 1049. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type VecExtendOp: defined at src/isa/aarch64/inst.isle line 1044. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum VecExtendOp { Sxtl8, Sxtl16, @@ -839,8 +839,8 @@ pub enum VecExtendOp { Uxtl32, } -/// Internal type VecALUOp: defined at src/isa/aarch64/inst.isle line 1066. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type VecALUOp: defined at src/isa/aarch64/inst.isle line 1061. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum VecALUOp { Sqadd, Uqadd, @@ -881,8 +881,8 @@ pub enum VecALUOp { Sqrdmulh, } -/// Internal type VecMisc2: defined at src/isa/aarch64/inst.isle line 1145. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type VecMisc2: defined at src/isa/aarch64/inst.isle line 1140. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum VecMisc2 { Not, Neg, @@ -903,8 +903,8 @@ pub enum VecMisc2 { Cmeq0, } -/// Internal type VecRRLongOp: defined at src/isa/aarch64/inst.isle line 1184. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type VecRRLongOp: defined at src/isa/aarch64/inst.isle line 1179. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum VecRRLongOp { Fcvtl16, Fcvtl32, @@ -913,8 +913,8 @@ pub enum VecRRLongOp { Shll32, } -/// Internal type VecRRNarrowOp: defined at src/isa/aarch64/inst.isle line 1199. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type VecRRNarrowOp: defined at src/isa/aarch64/inst.isle line 1194. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum VecRRNarrowOp { Xtn16, Xtn32, @@ -932,8 +932,8 @@ pub enum VecRRNarrowOp { Fcvtn64, } -/// Internal type VecRRRLongOp: defined at src/isa/aarch64/inst.isle line 1231. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type VecRRRLongOp: defined at src/isa/aarch64/inst.isle line 1226. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum VecRRRLongOp { Smull8, Smull16, @@ -946,14 +946,14 @@ pub enum VecRRRLongOp { Umlal32, } -/// Internal type VecPairOp: defined at src/isa/aarch64/inst.isle line 1248. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type VecPairOp: defined at src/isa/aarch64/inst.isle line 1243. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum VecPairOp { Addp, } -/// Internal type VecRRPairLongOp: defined at src/isa/aarch64/inst.isle line 1256. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type VecRRPairLongOp: defined at src/isa/aarch64/inst.isle line 1251. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum VecRRPairLongOp { Saddlp8, Saddlp16, @@ -961,23 +961,23 @@ pub enum VecRRPairLongOp { Uaddlp16, } -/// Internal type VecLanesOp: defined at src/isa/aarch64/inst.isle line 1267. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type VecLanesOp: defined at src/isa/aarch64/inst.isle line 1262. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum VecLanesOp { Addv, Uminv, } -/// Internal type VecShiftImmOp: defined at src/isa/aarch64/inst.isle line 1276. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type VecShiftImmOp: defined at src/isa/aarch64/inst.isle line 1271. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum VecShiftImmOp { Shl, Ushr, Sshr, } -/// Internal type AtomicRMWOp: defined at src/isa/aarch64/inst.isle line 1287. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type AtomicRMWOp: defined at src/isa/aarch64/inst.isle line 1282. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum AtomicRMWOp { Add, Clr, @@ -992,7 +992,7 @@ pub enum AtomicRMWOp { // Generated as internal constructor for term temp_reg. pub fn constructor_temp_reg(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; - // Rule at src/prelude.isle line 70. + // Rule at src/prelude.isle line 73. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); return Some(expr1_0); @@ -1001,7 +1001,7 @@ pub fn constructor_temp_reg(ctx: &mut C, arg0: Type) -> Option // Generated as internal constructor for term lo_reg. pub fn constructor_lo_reg(ctx: &mut C, arg0: Value) -> Option { let pattern0_0 = arg0; - // Rule at src/prelude.isle line 105. + // Rule at src/prelude.isle line 108. let expr0_0 = C::put_in_regs(ctx, pattern0_0); let expr1_0: usize = 0; let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); @@ -1018,7 +1018,7 @@ pub fn constructor_value_regs_none( inst: ref pattern1_0, } = pattern0_0 { - // Rule at src/prelude.isle line 287. + // Rule at src/prelude.isle line 299. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::value_regs_invalid(ctx); return Some(expr1_0); @@ -1044,7 +1044,7 @@ pub fn constructor_with_flags( result: pattern3_1, } = pattern2_0 { - // Rule at src/prelude.isle line 308. + // Rule at src/prelude.isle line 320. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern3_0); let expr2_0 = C::value_regs(ctx, pattern1_1, pattern3_1); @@ -1072,7 +1072,7 @@ pub fn constructor_with_flags_1( result: pattern3_1, } = pattern2_0 { - // Rule at src/prelude.isle line 316. + // Rule at src/prelude.isle line 328. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern3_0); return Some(pattern3_1); @@ -1106,7 +1106,7 @@ pub fn constructor_with_flags_2( result: pattern5_1, } = pattern4_0 { - // Rule at src/prelude.isle line 326. + // Rule at src/prelude.isle line 338. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern5_0); let expr2_0 = C::emit(ctx, &pattern3_0); @@ -1124,28 +1124,28 @@ pub fn constructor_vector_size(ctx: &mut C, arg0: Type) -> Option( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1390. + // Rule at src/isa/aarch64/inst.isle line 1385. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::MovZ { @@ -1183,7 +1183,7 @@ pub fn constructor_movn( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1397. + // Rule at src/isa/aarch64/inst.isle line 1392. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::MovN { @@ -1206,7 +1206,7 @@ pub fn constructor_alu_rr_imm_logic( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1404. + // Rule at src/isa/aarch64/inst.isle line 1399. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::AluRRImmLogic { @@ -1230,7 +1230,7 @@ pub fn constructor_alu_rr_imm_shift( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1411. + // Rule at src/isa/aarch64/inst.isle line 1406. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::AluRRImmShift { @@ -1254,7 +1254,7 @@ pub fn constructor_alu_rrr( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1418. + // Rule at src/isa/aarch64/inst.isle line 1413. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::AluRRR { @@ -1280,7 +1280,7 @@ pub fn constructor_vec_rrr( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1425. + // Rule at src/isa/aarch64/inst.isle line 1420. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::VecRRR { @@ -1305,7 +1305,7 @@ pub fn constructor_vec_lanes( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1432. + // Rule at src/isa/aarch64/inst.isle line 1427. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::VecLanes { @@ -1323,7 +1323,7 @@ pub fn constructor_vec_lanes( pub fn constructor_vec_dup(ctx: &mut C, arg0: Reg, arg1: &VectorSize) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1439. + // Rule at src/isa/aarch64/inst.isle line 1434. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::VecDup { @@ -1346,7 +1346,7 @@ pub fn constructor_alu_rr_imm12( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1446. + // Rule at src/isa/aarch64/inst.isle line 1441. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::AluRRImm12 { @@ -1372,7 +1372,7 @@ pub fn constructor_alu_rrr_shift( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1453. + // Rule at src/isa/aarch64/inst.isle line 1448. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::AluRRRShift { @@ -1399,7 +1399,7 @@ pub fn constructor_alu_rrr_extend( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1460. + // Rule at src/isa/aarch64/inst.isle line 1455. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::AluRRRExtend { @@ -1424,7 +1424,7 @@ pub fn constructor_alu_rr_extend_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1468. + // Rule at src/isa/aarch64/inst.isle line 1463. let expr0_0 = C::put_extended_in_reg(ctx, pattern2_0); let expr1_0 = C::get_extended_op(ctx, pattern2_0); let expr2_0 = constructor_alu_rrr_extend(ctx, pattern0_0, pattern1_0, expr0_0, &expr1_0)?; @@ -1443,7 +1443,7 @@ pub fn constructor_alu_rrrr( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1475. + // Rule at src/isa/aarch64/inst.isle line 1470. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::AluRRRR { @@ -1462,7 +1462,7 @@ pub fn constructor_alu_rrrr( pub fn constructor_bit_rr(ctx: &mut C, arg0: &BitOp, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1482. + // Rule at src/isa/aarch64/inst.isle line 1477. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::BitRR { @@ -1483,7 +1483,7 @@ pub fn constructor_add64_with_flags( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1489. + // Rule at src/isa/aarch64/inst.isle line 1484. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = ALUOp::AddS64; @@ -1505,7 +1505,7 @@ pub fn constructor_add64_with_flags( pub fn constructor_adc64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1496. + // Rule at src/isa/aarch64/inst.isle line 1491. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = ALUOp::Adc64; @@ -1531,7 +1531,7 @@ pub fn constructor_sub64_with_flags( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1503. + // Rule at src/isa/aarch64/inst.isle line 1498. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = ALUOp::SubS64; @@ -1557,7 +1557,7 @@ pub fn constructor_cmp64_imm( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1509. + // Rule at src/isa/aarch64/inst.isle line 1504. let expr0_0 = ALUOp::SubS64; let expr1_0 = C::writable_zero_reg(ctx); let expr2_0 = MInst::AluRRImm12 { @@ -1578,7 +1578,7 @@ pub fn constructor_cmp64_imm( pub fn constructor_sbc64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1515. + // Rule at src/isa/aarch64/inst.isle line 1510. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = ALUOp::Sbc64; @@ -1606,7 +1606,7 @@ pub fn constructor_vec_misc( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1522. + // Rule at src/isa/aarch64/inst.isle line 1517. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::VecMisc { @@ -1632,7 +1632,7 @@ pub fn constructor_vec_rrr_long( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1529. + // Rule at src/isa/aarch64/inst.isle line 1524. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::VecRRRLong { @@ -1661,7 +1661,7 @@ pub fn constructor_vec_rrrr_long( let pattern2_0 = arg2; let pattern3_0 = arg3; let pattern4_0 = arg4; - // Rule at src/isa/aarch64/inst.isle line 1539. + // Rule at src/isa/aarch64/inst.isle line 1534. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::FpuMove128 { @@ -1691,7 +1691,7 @@ pub fn constructor_vec_rr_narrow( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1547. + // Rule at src/isa/aarch64/inst.isle line 1542. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::VecRRNarrow { @@ -1715,7 +1715,7 @@ pub fn constructor_vec_rr_long( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1554. + // Rule at src/isa/aarch64/inst.isle line 1549. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::VecRRLong { @@ -1737,7 +1737,7 @@ pub fn constructor_mov_to_fpu( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1561. + // Rule at src/isa/aarch64/inst.isle line 1556. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::MovToFpu { @@ -1762,7 +1762,7 @@ pub fn constructor_mov_to_vec( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1568. + // Rule at src/isa/aarch64/inst.isle line 1563. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::FpuMove128 { @@ -1791,7 +1791,7 @@ pub fn constructor_mov_from_vec( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1576. + // Rule at src/isa/aarch64/inst.isle line 1571. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::MovFromVec { @@ -1817,7 +1817,7 @@ pub fn constructor_mov_from_vec_signed( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1583. + // Rule at src/isa/aarch64/inst.isle line 1578. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::MovFromVecSigned { @@ -1844,7 +1844,7 @@ pub fn constructor_extend( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1590. + // Rule at src/isa/aarch64/inst.isle line 1585. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::Extend { @@ -1863,7 +1863,7 @@ pub fn constructor_extend( pub fn constructor_load_acquire(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1597. + // Rule at src/isa/aarch64/inst.isle line 1592. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::LoadAcquire { @@ -1884,7 +1884,7 @@ pub fn constructor_tst64_imm( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1607. + // Rule at src/isa/aarch64/inst.isle line 1602. let expr0_0 = ALUOp::AndS64; let expr1_0 = C::writable_zero_reg(ctx); let expr2_0 = MInst::AluRRImmLogic { @@ -1911,7 +1911,7 @@ pub fn constructor_csel( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1620. + // Rule at src/isa/aarch64/inst.isle line 1615. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::CSel { @@ -1934,14 +1934,14 @@ pub fn constructor_add(ctx: &mut C, arg0: Type, arg1: Reg, arg2: Reg if pattern0_0 == I64 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1629. + // Rule at src/isa/aarch64/inst.isle line 1624. let expr0_0 = constructor_add64(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if let Some(pattern1_0) = C::fits_in_32(ctx, pattern0_0) { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1628. + // Rule at src/isa/aarch64/inst.isle line 1623. let expr0_0 = constructor_add32(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } @@ -1952,7 +1952,7 @@ pub fn constructor_add(ctx: &mut C, arg0: Type, arg1: Reg, arg2: Reg pub fn constructor_add32(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1632. + // Rule at src/isa/aarch64/inst.isle line 1627. let expr0_0 = ALUOp::Add32; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -1962,7 +1962,7 @@ pub fn constructor_add32(ctx: &mut C, arg0: Reg, arg1: Reg) -> Optio pub fn constructor_add64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1635. + // Rule at src/isa/aarch64/inst.isle line 1630. let expr0_0 = ALUOp::Add64; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -1979,14 +1979,14 @@ pub fn constructor_add_imm( if pattern0_0 == I64 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1639. + // Rule at src/isa/aarch64/inst.isle line 1634. let expr0_0 = constructor_add64_imm(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if let Some(pattern1_0) = C::fits_in_32(ctx, pattern0_0) { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1638. + // Rule at src/isa/aarch64/inst.isle line 1633. let expr0_0 = constructor_add32_imm(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } @@ -1997,7 +1997,7 @@ pub fn constructor_add_imm( pub fn constructor_add32_imm(ctx: &mut C, arg0: Reg, arg1: Imm12) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1642. + // Rule at src/isa/aarch64/inst.isle line 1637. let expr0_0 = ALUOp::Add32; let expr1_0 = constructor_alu_rr_imm12(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2007,7 +2007,7 @@ pub fn constructor_add32_imm(ctx: &mut C, arg0: Reg, arg1: Imm12) -> pub fn constructor_add64_imm(ctx: &mut C, arg0: Reg, arg1: Imm12) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1645. + // Rule at src/isa/aarch64/inst.isle line 1640. let expr0_0 = ALUOp::Add64; let expr1_0 = constructor_alu_rr_imm12(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2024,14 +2024,14 @@ pub fn constructor_add_extend( if pattern0_0 == I64 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1649. + // Rule at src/isa/aarch64/inst.isle line 1644. let expr0_0 = constructor_add64_extend(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if let Some(pattern1_0) = C::fits_in_32(ctx, pattern0_0) { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1648. + // Rule at src/isa/aarch64/inst.isle line 1643. let expr0_0 = constructor_add32_extend(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } @@ -2046,7 +2046,7 @@ pub fn constructor_add32_extend( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1652. + // Rule at src/isa/aarch64/inst.isle line 1647. let expr0_0 = ALUOp::Add32; let expr1_0 = constructor_alu_rr_extend_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2060,7 +2060,7 @@ pub fn constructor_add64_extend( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1655. + // Rule at src/isa/aarch64/inst.isle line 1650. let expr0_0 = ALUOp::Add64; let expr1_0 = constructor_alu_rr_extend_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2079,7 +2079,7 @@ pub fn constructor_add_shift( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1659. + // Rule at src/isa/aarch64/inst.isle line 1654. let expr0_0 = constructor_add64_shift(ctx, pattern2_0, pattern3_0, pattern4_0)?; return Some(expr0_0); } @@ -2087,7 +2087,7 @@ pub fn constructor_add_shift( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1658. + // Rule at src/isa/aarch64/inst.isle line 1653. let expr0_0 = constructor_add32_shift(ctx, pattern2_0, pattern3_0, pattern4_0)?; return Some(expr0_0); } @@ -2104,7 +2104,7 @@ pub fn constructor_add32_shift( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1662. + // Rule at src/isa/aarch64/inst.isle line 1657. let expr0_0 = ALUOp::Add32; let expr1_0 = constructor_alu_rrr_shift(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2120,7 +2120,7 @@ pub fn constructor_add64_shift( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1665. + // Rule at src/isa/aarch64/inst.isle line 1660. let expr0_0 = ALUOp::Add64; let expr1_0 = constructor_alu_rrr_shift(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2136,7 +2136,7 @@ pub fn constructor_add_vec( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1668. + // Rule at src/isa/aarch64/inst.isle line 1663. let expr0_0 = VecALUOp::Add; let expr1_0 = constructor_vec_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2148,14 +2148,14 @@ pub fn constructor_sub(ctx: &mut C, arg0: Type, arg1: Reg, arg2: Reg if pattern0_0 == I64 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1674. + // Rule at src/isa/aarch64/inst.isle line 1669. let expr0_0 = constructor_sub64(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if let Some(pattern1_0) = C::fits_in_32(ctx, pattern0_0) { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1673. + // Rule at src/isa/aarch64/inst.isle line 1668. let expr0_0 = constructor_sub32(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } @@ -2166,7 +2166,7 @@ pub fn constructor_sub(ctx: &mut C, arg0: Type, arg1: Reg, arg2: Reg pub fn constructor_sub32(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1677. + // Rule at src/isa/aarch64/inst.isle line 1672. let expr0_0 = ALUOp::Sub32; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2176,7 +2176,7 @@ pub fn constructor_sub32(ctx: &mut C, arg0: Reg, arg1: Reg) -> Optio pub fn constructor_sub64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1680. + // Rule at src/isa/aarch64/inst.isle line 1675. let expr0_0 = ALUOp::Sub64; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2193,14 +2193,14 @@ pub fn constructor_sub_imm( if pattern0_0 == I64 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1684. + // Rule at src/isa/aarch64/inst.isle line 1679. let expr0_0 = constructor_sub64_imm(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if let Some(pattern1_0) = C::fits_in_32(ctx, pattern0_0) { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1683. + // Rule at src/isa/aarch64/inst.isle line 1678. let expr0_0 = constructor_sub32_imm(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } @@ -2211,7 +2211,7 @@ pub fn constructor_sub_imm( pub fn constructor_sub32_imm(ctx: &mut C, arg0: Reg, arg1: Imm12) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1687. + // Rule at src/isa/aarch64/inst.isle line 1682. let expr0_0 = ALUOp::Sub32; let expr1_0 = constructor_alu_rr_imm12(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2221,7 +2221,7 @@ pub fn constructor_sub32_imm(ctx: &mut C, arg0: Reg, arg1: Imm12) -> pub fn constructor_sub64_imm(ctx: &mut C, arg0: Reg, arg1: Imm12) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1690. + // Rule at src/isa/aarch64/inst.isle line 1685. let expr0_0 = ALUOp::Sub64; let expr1_0 = constructor_alu_rr_imm12(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2238,14 +2238,14 @@ pub fn constructor_sub_extend( if pattern0_0 == I64 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1694. + // Rule at src/isa/aarch64/inst.isle line 1689. let expr0_0 = constructor_sub64_extend(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if let Some(pattern1_0) = C::fits_in_32(ctx, pattern0_0) { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1693. + // Rule at src/isa/aarch64/inst.isle line 1688. let expr0_0 = constructor_sub32_extend(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } @@ -2260,7 +2260,7 @@ pub fn constructor_sub32_extend( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1697. + // Rule at src/isa/aarch64/inst.isle line 1692. let expr0_0 = ALUOp::Sub32; let expr1_0 = constructor_alu_rr_extend_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2274,7 +2274,7 @@ pub fn constructor_sub64_extend( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1700. + // Rule at src/isa/aarch64/inst.isle line 1695. let expr0_0 = ALUOp::Sub64; let expr1_0 = constructor_alu_rr_extend_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2293,7 +2293,7 @@ pub fn constructor_sub_shift( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1704. + // Rule at src/isa/aarch64/inst.isle line 1699. let expr0_0 = constructor_sub64_shift(ctx, pattern2_0, pattern3_0, pattern4_0)?; return Some(expr0_0); } @@ -2301,7 +2301,7 @@ pub fn constructor_sub_shift( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1703. + // Rule at src/isa/aarch64/inst.isle line 1698. let expr0_0 = constructor_sub32_shift(ctx, pattern2_0, pattern3_0, pattern4_0)?; return Some(expr0_0); } @@ -2318,7 +2318,7 @@ pub fn constructor_sub32_shift( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1707. + // Rule at src/isa/aarch64/inst.isle line 1702. let expr0_0 = ALUOp::Sub32; let expr1_0 = constructor_alu_rrr_shift(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2334,7 +2334,7 @@ pub fn constructor_sub64_shift( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1710. + // Rule at src/isa/aarch64/inst.isle line 1705. let expr0_0 = ALUOp::Sub64; let expr1_0 = constructor_alu_rrr_shift(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2350,7 +2350,7 @@ pub fn constructor_sub_vec( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1713. + // Rule at src/isa/aarch64/inst.isle line 1708. let expr0_0 = VecALUOp::Sub; let expr1_0 = constructor_vec_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2369,7 +2369,7 @@ pub fn constructor_madd( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1719. + // Rule at src/isa/aarch64/inst.isle line 1714. let expr0_0 = constructor_madd64(ctx, pattern2_0, pattern3_0, pattern4_0)?; return Some(expr0_0); } @@ -2377,7 +2377,7 @@ pub fn constructor_madd( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1718. + // Rule at src/isa/aarch64/inst.isle line 1713. let expr0_0 = constructor_madd32(ctx, pattern2_0, pattern3_0, pattern4_0)?; return Some(expr0_0); } @@ -2389,7 +2389,7 @@ pub fn constructor_madd32(ctx: &mut C, arg0: Reg, arg1: Reg, arg2: R let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1722. + // Rule at src/isa/aarch64/inst.isle line 1717. let expr0_0 = ALUOp3::MAdd32; let expr1_0 = constructor_alu_rrrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2400,7 +2400,7 @@ pub fn constructor_madd64(ctx: &mut C, arg0: Reg, arg1: Reg, arg2: R let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1725. + // Rule at src/isa/aarch64/inst.isle line 1720. let expr0_0 = ALUOp3::MAdd64; let expr1_0 = constructor_alu_rrrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2411,7 +2411,7 @@ pub fn constructor_msub64(ctx: &mut C, arg0: Reg, arg1: Reg, arg2: R let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1730. + // Rule at src/isa/aarch64/inst.isle line 1725. let expr0_0 = ALUOp3::MSub64; let expr1_0 = constructor_alu_rrrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2427,7 +2427,7 @@ pub fn constructor_uqadd( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1734. + // Rule at src/isa/aarch64/inst.isle line 1729. let expr0_0 = VecALUOp::Uqadd; let expr1_0 = constructor_vec_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2443,7 +2443,7 @@ pub fn constructor_sqadd( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1738. + // Rule at src/isa/aarch64/inst.isle line 1733. let expr0_0 = VecALUOp::Sqadd; let expr1_0 = constructor_vec_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2459,7 +2459,7 @@ pub fn constructor_uqsub( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1742. + // Rule at src/isa/aarch64/inst.isle line 1737. let expr0_0 = VecALUOp::Uqsub; let expr1_0 = constructor_vec_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2475,7 +2475,7 @@ pub fn constructor_sqsub( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1746. + // Rule at src/isa/aarch64/inst.isle line 1741. let expr0_0 = VecALUOp::Sqsub; let expr1_0 = constructor_vec_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2485,7 +2485,7 @@ pub fn constructor_sqsub( pub fn constructor_umulh(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1750. + // Rule at src/isa/aarch64/inst.isle line 1745. let expr0_0 = ALUOp::UMulH; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2495,7 +2495,7 @@ pub fn constructor_umulh(ctx: &mut C, arg0: Reg, arg1: Reg) -> Optio pub fn constructor_smulh(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1754. + // Rule at src/isa/aarch64/inst.isle line 1749. let expr0_0 = ALUOp::SMulH; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2511,7 +2511,7 @@ pub fn constructor_mul( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1758. + // Rule at src/isa/aarch64/inst.isle line 1753. let expr0_0 = VecALUOp::Mul; let expr1_0 = constructor_vec_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2521,7 +2521,7 @@ pub fn constructor_mul( pub fn constructor_neg(ctx: &mut C, arg0: Reg, arg1: &VectorSize) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1762. + // Rule at src/isa/aarch64/inst.isle line 1757. let expr0_0 = VecMisc2::Neg; let expr1_0 = constructor_vec_misc(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2531,7 +2531,7 @@ pub fn constructor_neg(ctx: &mut C, arg0: Reg, arg1: &VectorSize) -> pub fn constructor_rev64(ctx: &mut C, arg0: Reg, arg1: &VectorSize) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1766. + // Rule at src/isa/aarch64/inst.isle line 1761. let expr0_0 = VecMisc2::Rev64; let expr1_0 = constructor_vec_misc(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2541,7 +2541,7 @@ pub fn constructor_rev64(ctx: &mut C, arg0: Reg, arg1: &VectorSize) pub fn constructor_xtn64(ctx: &mut C, arg0: Reg, arg1: bool) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1770. + // Rule at src/isa/aarch64/inst.isle line 1765. let expr0_0 = VecRRNarrowOp::Xtn64; let expr1_0 = constructor_vec_rr_narrow(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2557,7 +2557,7 @@ pub fn constructor_addp( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1774. + // Rule at src/isa/aarch64/inst.isle line 1769. let expr0_0 = VecALUOp::Addp; let expr1_0 = constructor_vec_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2567,7 +2567,7 @@ pub fn constructor_addp( pub fn constructor_addv(ctx: &mut C, arg0: Reg, arg1: &VectorSize) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1778. + // Rule at src/isa/aarch64/inst.isle line 1773. let expr0_0 = VecLanesOp::Addv; let expr1_0 = constructor_vec_lanes(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2577,7 +2577,7 @@ pub fn constructor_addv(ctx: &mut C, arg0: Reg, arg1: &VectorSize) - pub fn constructor_shll32(ctx: &mut C, arg0: Reg, arg1: bool) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1782. + // Rule at src/isa/aarch64/inst.isle line 1777. let expr0_0 = VecRRLongOp::Shll32; let expr1_0 = constructor_vec_rr_long(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2595,7 +2595,7 @@ pub fn constructor_umlal32( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1786. + // Rule at src/isa/aarch64/inst.isle line 1781. let expr0_0 = VecRRRLongOp::Umlal32; let expr1_0 = constructor_vec_rrrr_long( ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0, pattern3_0, @@ -2613,7 +2613,7 @@ pub fn constructor_smull8( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1790. + // Rule at src/isa/aarch64/inst.isle line 1785. let expr0_0 = VecRRRLongOp::Smull8; let expr1_0 = constructor_vec_rrr_long(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2629,7 +2629,7 @@ pub fn constructor_umull8( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1794. + // Rule at src/isa/aarch64/inst.isle line 1789. let expr0_0 = VecRRRLongOp::Umull8; let expr1_0 = constructor_vec_rrr_long(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2645,7 +2645,7 @@ pub fn constructor_smull16( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1798. + // Rule at src/isa/aarch64/inst.isle line 1793. let expr0_0 = VecRRRLongOp::Smull16; let expr1_0 = constructor_vec_rrr_long(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2661,7 +2661,7 @@ pub fn constructor_umull16( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1802. + // Rule at src/isa/aarch64/inst.isle line 1797. let expr0_0 = VecRRRLongOp::Umull16; let expr1_0 = constructor_vec_rrr_long(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2677,7 +2677,7 @@ pub fn constructor_smull32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1806. + // Rule at src/isa/aarch64/inst.isle line 1801. let expr0_0 = VecRRRLongOp::Smull32; let expr1_0 = constructor_vec_rrr_long(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2693,7 +2693,7 @@ pub fn constructor_umull32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1810. + // Rule at src/isa/aarch64/inst.isle line 1805. let expr0_0 = VecRRRLongOp::Umull32; let expr1_0 = constructor_vec_rrr_long(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2703,7 +2703,7 @@ pub fn constructor_umull32( pub fn constructor_asr64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1814. + // Rule at src/isa/aarch64/inst.isle line 1809. let expr0_0 = ALUOp::Asr64; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2713,7 +2713,7 @@ pub fn constructor_asr64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Optio pub fn constructor_asr64_imm(ctx: &mut C, arg0: Reg, arg1: ImmShift) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1817. + // Rule at src/isa/aarch64/inst.isle line 1812. let expr0_0 = ALUOp::Asr64; let expr1_0 = constructor_alu_rr_imm_shift(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2723,7 +2723,7 @@ pub fn constructor_asr64_imm(ctx: &mut C, arg0: Reg, arg1: ImmShift) pub fn constructor_lsr32(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1821. + // Rule at src/isa/aarch64/inst.isle line 1816. let expr0_0 = ALUOp::Lsr32; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2733,7 +2733,7 @@ pub fn constructor_lsr32(ctx: &mut C, arg0: Reg, arg1: Reg) -> Optio pub fn constructor_lsr32_imm(ctx: &mut C, arg0: Reg, arg1: ImmShift) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1824. + // Rule at src/isa/aarch64/inst.isle line 1819. let expr0_0 = ALUOp::Lsr32; let expr1_0 = constructor_alu_rr_imm_shift(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2743,7 +2743,7 @@ pub fn constructor_lsr32_imm(ctx: &mut C, arg0: Reg, arg1: ImmShift) pub fn constructor_lsr64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1827. + // Rule at src/isa/aarch64/inst.isle line 1822. let expr0_0 = ALUOp::Lsr64; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2753,7 +2753,7 @@ pub fn constructor_lsr64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Optio pub fn constructor_lsr64_imm(ctx: &mut C, arg0: Reg, arg1: ImmShift) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1830. + // Rule at src/isa/aarch64/inst.isle line 1825. let expr0_0 = ALUOp::Lsr64; let expr1_0 = constructor_alu_rr_imm_shift(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2763,7 +2763,7 @@ pub fn constructor_lsr64_imm(ctx: &mut C, arg0: Reg, arg1: ImmShift) pub fn constructor_lsl32(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1834. + // Rule at src/isa/aarch64/inst.isle line 1829. let expr0_0 = ALUOp::Lsl32; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2773,7 +2773,7 @@ pub fn constructor_lsl32(ctx: &mut C, arg0: Reg, arg1: Reg) -> Optio pub fn constructor_lsl32_imm(ctx: &mut C, arg0: Reg, arg1: ImmShift) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1837. + // Rule at src/isa/aarch64/inst.isle line 1832. let expr0_0 = ALUOp::Lsl32; let expr1_0 = constructor_alu_rr_imm_shift(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2783,7 +2783,7 @@ pub fn constructor_lsl32_imm(ctx: &mut C, arg0: Reg, arg1: ImmShift) pub fn constructor_lsl64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1840. + // Rule at src/isa/aarch64/inst.isle line 1835. let expr0_0 = ALUOp::Lsl64; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2793,7 +2793,7 @@ pub fn constructor_lsl64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Optio pub fn constructor_lsl64_imm(ctx: &mut C, arg0: Reg, arg1: ImmShift) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1843. + // Rule at src/isa/aarch64/inst.isle line 1838. let expr0_0 = ALUOp::Lsl64; let expr1_0 = constructor_alu_rr_imm_shift(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2803,7 +2803,7 @@ pub fn constructor_lsl64_imm(ctx: &mut C, arg0: Reg, arg1: ImmShift) pub fn constructor_udiv64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1847. + // Rule at src/isa/aarch64/inst.isle line 1842. let expr0_0 = ALUOp::UDiv64; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2813,7 +2813,7 @@ pub fn constructor_udiv64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Opti pub fn constructor_sdiv64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1851. + // Rule at src/isa/aarch64/inst.isle line 1846. let expr0_0 = ALUOp::SDiv64; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2823,7 +2823,7 @@ pub fn constructor_sdiv64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Opti pub fn constructor_not(ctx: &mut C, arg0: Reg, arg1: &VectorSize) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1855. + // Rule at src/isa/aarch64/inst.isle line 1850. let expr0_0 = VecMisc2::Not; let expr1_0 = constructor_vec_misc(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2840,14 +2840,14 @@ pub fn constructor_orr_not( if pattern0_0 == I64 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1861. + // Rule at src/isa/aarch64/inst.isle line 1856. let expr0_0 = constructor_orr_not64(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if let Some(pattern1_0) = C::fits_in_32(ctx, pattern0_0) { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1860. + // Rule at src/isa/aarch64/inst.isle line 1855. let expr0_0 = constructor_orr_not32(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } @@ -2858,7 +2858,7 @@ pub fn constructor_orr_not( pub fn constructor_orr_not32(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1864. + // Rule at src/isa/aarch64/inst.isle line 1859. let expr0_0 = ALUOp::OrrNot32; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2868,7 +2868,7 @@ pub fn constructor_orr_not32(ctx: &mut C, arg0: Reg, arg1: Reg) -> O pub fn constructor_orr_not64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1867. + // Rule at src/isa/aarch64/inst.isle line 1862. let expr0_0 = ALUOp::OrrNot64; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2887,7 +2887,7 @@ pub fn constructor_orr_not_shift( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1871. + // Rule at src/isa/aarch64/inst.isle line 1866. let expr0_0 = constructor_orr_not_shift64(ctx, pattern2_0, pattern3_0, pattern4_0)?; return Some(expr0_0); } @@ -2895,7 +2895,7 @@ pub fn constructor_orr_not_shift( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1870. + // Rule at src/isa/aarch64/inst.isle line 1865. let expr0_0 = constructor_orr_not_shift32(ctx, pattern2_0, pattern3_0, pattern4_0)?; return Some(expr0_0); } @@ -2912,7 +2912,7 @@ pub fn constructor_orr_not_shift32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1874. + // Rule at src/isa/aarch64/inst.isle line 1869. let expr0_0 = ALUOp::OrrNot32; let expr1_0 = constructor_alu_rrr_shift(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2928,7 +2928,7 @@ pub fn constructor_orr_not_shift64( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1877. + // Rule at src/isa/aarch64/inst.isle line 1872. let expr0_0 = ALUOp::OrrNot64; let expr1_0 = constructor_alu_rrr_shift(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2938,7 +2938,7 @@ pub fn constructor_orr_not_shift64( pub fn constructor_orr32(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1882. + // Rule at src/isa/aarch64/inst.isle line 1877. let expr0_0 = ALUOp::Orr32; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2948,7 +2948,7 @@ pub fn constructor_orr32(ctx: &mut C, arg0: Reg, arg1: Reg) -> Optio pub fn constructor_orr32_imm(ctx: &mut C, arg0: Reg, arg1: ImmLogic) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1885. + // Rule at src/isa/aarch64/inst.isle line 1880. let expr0_0 = ALUOp::Orr32; let expr1_0 = constructor_alu_rr_imm_logic(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2958,7 +2958,7 @@ pub fn constructor_orr32_imm(ctx: &mut C, arg0: Reg, arg1: ImmLogic) pub fn constructor_orr64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1888. + // Rule at src/isa/aarch64/inst.isle line 1883. let expr0_0 = ALUOp::Orr64; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2968,7 +2968,7 @@ pub fn constructor_orr64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Optio pub fn constructor_orr64_imm(ctx: &mut C, arg0: Reg, arg1: ImmLogic) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1891. + // Rule at src/isa/aarch64/inst.isle line 1886. let expr0_0 = ALUOp::Orr64; let expr1_0 = constructor_alu_rr_imm_logic(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2984,7 +2984,7 @@ pub fn constructor_orr_vec( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1894. + // Rule at src/isa/aarch64/inst.isle line 1889. let expr0_0 = VecALUOp::Orr; let expr1_0 = constructor_vec_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2994,7 +2994,7 @@ pub fn constructor_orr_vec( pub fn constructor_and32_imm(ctx: &mut C, arg0: Reg, arg1: ImmLogic) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1899. + // Rule at src/isa/aarch64/inst.isle line 1894. let expr0_0 = ALUOp::And32; let expr1_0 = constructor_alu_rr_imm_logic(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -3010,7 +3010,7 @@ pub fn constructor_and_vec( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1902. + // Rule at src/isa/aarch64/inst.isle line 1897. let expr0_0 = VecALUOp::And; let expr1_0 = constructor_vec_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -3026,7 +3026,7 @@ pub fn constructor_eor_vec( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1906. + // Rule at src/isa/aarch64/inst.isle line 1901. let expr0_0 = VecALUOp::Eor; let expr1_0 = constructor_vec_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -3042,7 +3042,7 @@ pub fn constructor_bic_vec( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1910. + // Rule at src/isa/aarch64/inst.isle line 1905. let expr0_0 = VecALUOp::Bic; let expr1_0 = constructor_vec_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -3058,7 +3058,7 @@ pub fn constructor_sshl( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1914. + // Rule at src/isa/aarch64/inst.isle line 1909. let expr0_0 = VecALUOp::Sshl; let expr1_0 = constructor_vec_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -3074,7 +3074,7 @@ pub fn constructor_ushl( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1918. + // Rule at src/isa/aarch64/inst.isle line 1913. let expr0_0 = VecALUOp::Ushl; let expr1_0 = constructor_vec_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -3084,7 +3084,7 @@ pub fn constructor_ushl( pub fn constructor_rotr32(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1923. + // Rule at src/isa/aarch64/inst.isle line 1918. let expr0_0 = ALUOp::RotR32; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -3094,7 +3094,7 @@ pub fn constructor_rotr32(ctx: &mut C, arg0: Reg, arg1: Reg) -> Opti pub fn constructor_rotr32_imm(ctx: &mut C, arg0: Reg, arg1: ImmShift) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1926. + // Rule at src/isa/aarch64/inst.isle line 1921. let expr0_0 = ALUOp::RotR32; let expr1_0 = constructor_alu_rr_imm_shift(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -3104,7 +3104,7 @@ pub fn constructor_rotr32_imm(ctx: &mut C, arg0: Reg, arg1: ImmShift pub fn constructor_rotr64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1929. + // Rule at src/isa/aarch64/inst.isle line 1924. let expr0_0 = ALUOp::RotR64; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -3114,7 +3114,7 @@ pub fn constructor_rotr64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Opti pub fn constructor_rotr64_imm(ctx: &mut C, arg0: Reg, arg1: ImmShift) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1932. + // Rule at src/isa/aarch64/inst.isle line 1927. let expr0_0 = ALUOp::RotR64; let expr1_0 = constructor_alu_rr_imm_shift(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -3123,7 +3123,7 @@ pub fn constructor_rotr64_imm(ctx: &mut C, arg0: Reg, arg1: ImmShift // Generated as internal constructor for term rbit32. pub fn constructor_rbit32(ctx: &mut C, arg0: Reg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/aarch64/inst.isle line 1937. + // Rule at src/isa/aarch64/inst.isle line 1932. let expr0_0 = BitOp::RBit32; let expr1_0 = constructor_bit_rr(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -3132,7 +3132,7 @@ pub fn constructor_rbit32(ctx: &mut C, arg0: Reg) -> Option { // Generated as internal constructor for term rbit64. pub fn constructor_rbit64(ctx: &mut C, arg0: Reg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/aarch64/inst.isle line 1940. + // Rule at src/isa/aarch64/inst.isle line 1935. let expr0_0 = BitOp::RBit64; let expr1_0 = constructor_bit_rr(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -3141,7 +3141,7 @@ pub fn constructor_rbit64(ctx: &mut C, arg0: Reg) -> Option { // Generated as internal constructor for term clz32. pub fn constructor_clz32(ctx: &mut C, arg0: Reg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/aarch64/inst.isle line 1945. + // Rule at src/isa/aarch64/inst.isle line 1940. let expr0_0 = BitOp::Clz32; let expr1_0 = constructor_bit_rr(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -3150,7 +3150,7 @@ pub fn constructor_clz32(ctx: &mut C, arg0: Reg) -> Option { // Generated as internal constructor for term clz64. pub fn constructor_clz64(ctx: &mut C, arg0: Reg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/aarch64/inst.isle line 1948. + // Rule at src/isa/aarch64/inst.isle line 1943. let expr0_0 = BitOp::Clz64; let expr1_0 = constructor_bit_rr(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -3159,7 +3159,7 @@ pub fn constructor_clz64(ctx: &mut C, arg0: Reg) -> Option { // Generated as internal constructor for term cls32. pub fn constructor_cls32(ctx: &mut C, arg0: Reg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/aarch64/inst.isle line 1953. + // Rule at src/isa/aarch64/inst.isle line 1948. let expr0_0 = BitOp::Cls32; let expr1_0 = constructor_bit_rr(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -3168,7 +3168,7 @@ pub fn constructor_cls32(ctx: &mut C, arg0: Reg) -> Option { // Generated as internal constructor for term cls64. pub fn constructor_cls64(ctx: &mut C, arg0: Reg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/aarch64/inst.isle line 1956. + // Rule at src/isa/aarch64/inst.isle line 1951. let expr0_0 = BitOp::Cls64; let expr1_0 = constructor_bit_rr(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -3178,7 +3178,7 @@ pub fn constructor_cls64(ctx: &mut C, arg0: Reg) -> Option { pub fn constructor_eon32(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1961. + // Rule at src/isa/aarch64/inst.isle line 1956. let expr0_0 = ALUOp::EorNot32; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -3188,7 +3188,7 @@ pub fn constructor_eon32(ctx: &mut C, arg0: Reg, arg1: Reg) -> Optio pub fn constructor_eon64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1964. + // Rule at src/isa/aarch64/inst.isle line 1959. let expr0_0 = ALUOp::EorNot64; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -3198,7 +3198,7 @@ pub fn constructor_eon64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Optio pub fn constructor_vec_cnt(ctx: &mut C, arg0: Reg, arg1: &VectorSize) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1969. + // Rule at src/isa/aarch64/inst.isle line 1964. let expr0_0 = VecMisc2::Cnt; let expr1_0 = constructor_vec_misc(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -3215,25 +3215,25 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option }; if let Some(pattern3_0) = closure3() { if let Some(pattern4_0) = C::imm_logic_from_u64(ctx, pattern2_0, pattern3_0) { - // Rule at src/isa/aarch64/inst.isle line 1984. + // Rule at src/isa/aarch64/inst.isle line 1979. let expr0_0 = C::zero_reg(ctx); let expr1_0 = constructor_orr64_imm(ctx, expr0_0, pattern4_0)?; return Some(expr1_0); } } if let Some(pattern3_0) = C::move_wide_const_from_u64(ctx, pattern2_0) { - // Rule at src/isa/aarch64/inst.isle line 1976. + // Rule at src/isa/aarch64/inst.isle line 1971. let expr0_0 = OperandSize::Size64; let expr1_0 = constructor_movz(ctx, pattern3_0, &expr0_0)?; return Some(expr1_0); } if let Some(pattern3_0) = C::move_wide_const_from_negated_u64(ctx, pattern2_0) { - // Rule at src/isa/aarch64/inst.isle line 1980. + // Rule at src/isa/aarch64/inst.isle line 1975. let expr0_0 = OperandSize::Size64; let expr1_0 = constructor_movn(ctx, pattern3_0, &expr0_0)?; return Some(expr1_0); } - // Rule at src/isa/aarch64/inst.isle line 1991. + // Rule at src/isa/aarch64/inst.isle line 1986. let expr0_0 = C::load_constant64_full(ctx, pattern2_0); return Some(expr0_0); } @@ -3245,17 +3245,17 @@ pub fn constructor_put_in_reg_sext32(ctx: &mut C, arg0: Value) -> Op let pattern0_0 = arg0; let pattern1_0 = C::value_type(ctx, pattern0_0); if pattern1_0 == I32 { - // Rule at src/isa/aarch64/inst.isle line 2002. + // Rule at src/isa/aarch64/inst.isle line 1997. let expr0_0 = C::put_in_reg(ctx, pattern0_0); return Some(expr0_0); } if pattern1_0 == I64 { - // Rule at src/isa/aarch64/inst.isle line 2003. + // Rule at src/isa/aarch64/inst.isle line 1998. let expr0_0 = C::put_in_reg(ctx, pattern0_0); return Some(expr0_0); } if let Some(pattern2_0) = C::fits_in_32(ctx, pattern1_0) { - // Rule at src/isa/aarch64/inst.isle line 1998. + // Rule at src/isa/aarch64/inst.isle line 1993. let expr0_0 = C::put_in_reg(ctx, pattern0_0); let expr1_0: bool = true; let expr2_0 = C::ty_bits(ctx, pattern2_0); @@ -3271,17 +3271,17 @@ pub fn constructor_put_in_reg_zext32(ctx: &mut C, arg0: Value) -> Op let pattern0_0 = arg0; let pattern1_0 = C::value_type(ctx, pattern0_0); if pattern1_0 == I32 { - // Rule at src/isa/aarch64/inst.isle line 2011. + // Rule at src/isa/aarch64/inst.isle line 2006. let expr0_0 = C::put_in_reg(ctx, pattern0_0); return Some(expr0_0); } if pattern1_0 == I64 { - // Rule at src/isa/aarch64/inst.isle line 2012. + // Rule at src/isa/aarch64/inst.isle line 2007. let expr0_0 = C::put_in_reg(ctx, pattern0_0); return Some(expr0_0); } if let Some(pattern2_0) = C::fits_in_32(ctx, pattern1_0) { - // Rule at src/isa/aarch64/inst.isle line 2007. + // Rule at src/isa/aarch64/inst.isle line 2002. let expr0_0 = C::put_in_reg(ctx, pattern0_0); let expr1_0: bool = false; let expr2_0 = C::ty_bits(ctx, pattern2_0); @@ -3297,12 +3297,12 @@ pub fn constructor_put_in_reg_sext64(ctx: &mut C, arg0: Value) -> Op let pattern0_0 = arg0; let pattern1_0 = C::value_type(ctx, pattern0_0); if pattern1_0 == I64 { - // Rule at src/isa/aarch64/inst.isle line 2020. + // Rule at src/isa/aarch64/inst.isle line 2015. let expr0_0 = C::put_in_reg(ctx, pattern0_0); return Some(expr0_0); } if let Some(pattern2_0) = C::fits_in_32(ctx, pattern1_0) { - // Rule at src/isa/aarch64/inst.isle line 2016. + // Rule at src/isa/aarch64/inst.isle line 2011. let expr0_0 = C::put_in_reg(ctx, pattern0_0); let expr1_0: bool = true; let expr2_0 = C::ty_bits(ctx, pattern2_0); @@ -3318,12 +3318,12 @@ pub fn constructor_put_in_reg_zext64(ctx: &mut C, arg0: Value) -> Op let pattern0_0 = arg0; let pattern1_0 = C::value_type(ctx, pattern0_0); if pattern1_0 == I64 { - // Rule at src/isa/aarch64/inst.isle line 2028. + // Rule at src/isa/aarch64/inst.isle line 2023. let expr0_0 = C::put_in_reg(ctx, pattern0_0); return Some(expr0_0); } if let Some(pattern2_0) = C::fits_in_32(ctx, pattern1_0) { - // Rule at src/isa/aarch64/inst.isle line 2024. + // Rule at src/isa/aarch64/inst.isle line 2019. let expr0_0 = C::put_in_reg(ctx, pattern0_0); let expr1_0: bool = false; let expr2_0 = C::ty_bits(ctx, pattern2_0); @@ -3337,7 +3337,7 @@ pub fn constructor_put_in_reg_zext64(ctx: &mut C, arg0: Value) -> Op // Generated as internal constructor for term trap_if_zero_divisor. pub fn constructor_trap_if_zero_divisor(ctx: &mut C, arg0: Reg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/aarch64/inst.isle line 2033. + // Rule at src/isa/aarch64/inst.isle line 2028. let expr0_0 = C::cond_br_zero(ctx, pattern0_0); let expr1_0 = C::trap_code_division_by_zero(ctx); let expr2_0 = MInst::TrapIf { @@ -3352,12 +3352,12 @@ pub fn constructor_trap_if_zero_divisor(ctx: &mut C, arg0: Reg) -> O pub fn constructor_size_from_ty(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I64 { - // Rule at src/isa/aarch64/inst.isle line 2039. + // Rule at src/isa/aarch64/inst.isle line 2034. let expr0_0 = OperandSize::Size64; return Some(expr0_0); } if let Some(pattern1_0) = C::fits_in_32(ctx, pattern0_0) { - // Rule at src/isa/aarch64/inst.isle line 2038. + // Rule at src/isa/aarch64/inst.isle line 2033. let expr0_0 = OperandSize::Size32; return Some(expr0_0); } @@ -3374,7 +3374,7 @@ pub fn constructor_trap_if_div_overflow( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 2045. + // Rule at src/isa/aarch64/inst.isle line 2040. let expr0_0 = constructor_adds_op(ctx, pattern0_0)?; let expr1_0 = C::writable_zero_reg(ctx); let expr2_0: u8 = 1; @@ -3418,12 +3418,12 @@ pub fn constructor_trap_if_div_overflow( pub fn constructor_adds_op(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I64 { - // Rule at src/isa/aarch64/inst.isle line 2065. + // Rule at src/isa/aarch64/inst.isle line 2060. let expr0_0 = ALUOp::AddS64; return Some(expr0_0); } if let Some(pattern1_0) = C::fits_in_32(ctx, pattern0_0) { - // Rule at src/isa/aarch64/inst.isle line 2064. + // Rule at src/isa/aarch64/inst.isle line 2059. let expr0_0 = ALUOp::AddS32; return Some(expr0_0); } @@ -3457,7 +3457,7 @@ pub fn constructor_alu_rs_imm_logic_commutative( C::imm_logic_from_imm64(ctx, pattern5_1, pattern7_0) { let pattern9_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 2095. + // Rule at src/isa/aarch64/inst.isle line 2090. let expr0_0 = C::put_in_reg(ctx, pattern9_0); let expr1_0 = constructor_alu_rr_imm_logic(ctx, pattern0_0, expr0_0, pattern8_0)?; @@ -3488,7 +3488,7 @@ pub fn constructor_alu_rs_imm_logic_commutative( C::lshl_from_imm64(ctx, pattern10_1, pattern12_0) { let pattern14_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 2101. + // Rule at src/isa/aarch64/inst.isle line 2096. let expr0_0 = C::put_in_reg(ctx, pattern14_0); let expr1_0 = C::put_in_reg(ctx, pattern7_0); let expr2_0 = constructor_alu_rrr_shift( @@ -3525,7 +3525,7 @@ pub fn constructor_alu_rs_imm_logic_commutative( if let Some(pattern9_0) = C::imm_logic_from_imm64(ctx, pattern6_1, pattern8_0) { - // Rule at src/isa/aarch64/inst.isle line 2093. + // Rule at src/isa/aarch64/inst.isle line 2088. let expr0_0 = C::put_in_reg(ctx, pattern2_0); let expr1_0 = constructor_alu_rr_imm_logic(ctx, pattern0_0, expr0_0, pattern9_0)?; @@ -3555,7 +3555,7 @@ pub fn constructor_alu_rs_imm_logic_commutative( if let Some(pattern14_0) = C::lshl_from_imm64(ctx, pattern11_1, pattern13_0) { - // Rule at src/isa/aarch64/inst.isle line 2099. + // Rule at src/isa/aarch64/inst.isle line 2094. let expr0_0 = C::put_in_reg(ctx, pattern2_0); let expr1_0 = C::put_in_reg(ctx, pattern8_0); let expr2_0 = constructor_alu_rrr_shift( @@ -3576,7 +3576,7 @@ pub fn constructor_alu_rs_imm_logic_commutative( _ => {} } } - // Rule at src/isa/aarch64/inst.isle line 2089. + // Rule at src/isa/aarch64/inst.isle line 2084. let expr0_0 = C::put_in_reg(ctx, pattern2_0); let expr1_0 = C::put_in_reg(ctx, pattern3_0); let expr2_0 = constructor_alu_rrr(ctx, pattern0_0, expr0_0, expr1_0)?; @@ -3610,7 +3610,7 @@ pub fn constructor_alu_rs_imm_logic( if let Some(pattern9_0) = C::imm_logic_from_imm64(ctx, pattern6_1, pattern8_0) { - // Rule at src/isa/aarch64/inst.isle line 2109. + // Rule at src/isa/aarch64/inst.isle line 2104. let expr0_0 = C::put_in_reg(ctx, pattern2_0); let expr1_0 = constructor_alu_rr_imm_logic(ctx, pattern0_0, expr0_0, pattern9_0)?; @@ -3640,7 +3640,7 @@ pub fn constructor_alu_rs_imm_logic( if let Some(pattern14_0) = C::lshl_from_imm64(ctx, pattern11_1, pattern13_0) { - // Rule at src/isa/aarch64/inst.isle line 2111. + // Rule at src/isa/aarch64/inst.isle line 2106. let expr0_0 = C::put_in_reg(ctx, pattern2_0); let expr1_0 = C::put_in_reg(ctx, pattern8_0); let expr2_0 = constructor_alu_rrr_shift( @@ -3661,7 +3661,7 @@ pub fn constructor_alu_rs_imm_logic( _ => {} } } - // Rule at src/isa/aarch64/inst.isle line 2107. + // Rule at src/isa/aarch64/inst.isle line 2102. let expr0_0 = C::put_in_reg(ctx, pattern2_0); let expr1_0 = C::put_in_reg(ctx, pattern3_0); let expr2_0 = constructor_alu_rrr(ctx, pattern0_0, expr0_0, expr1_0)?; @@ -3678,7 +3678,7 @@ pub fn constructor_i128_alu_bitop( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 2119. + // Rule at src/isa/aarch64/inst.isle line 2114. let expr0_0 = C::put_in_regs(ctx, pattern1_0); let expr1_0: usize = 0; let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); diff --git a/cranelift/codegen/src/isa/s390x/inst.isle b/cranelift/codegen/src/isa/s390x/inst.isle index 377a68e7dd74..ae2e0303eb24 100644 --- a/cranelift/codegen/src/isa/s390x/inst.isle +++ b/cranelift/codegen/src/isa/s390x/inst.isle @@ -678,12 +678,7 @@ (type BoxCallInfo (primitive BoxCallInfo)) (type BoxCallIndInfo (primitive BoxCallIndInfo)) -(type MachLabel (primitive MachLabel)) -(type VecMachLabel (primitive VecMachLabel)) (type BoxJTSequenceInfo (primitive BoxJTSequenceInfo)) -(type BoxExternalName (primitive BoxExternalName)) -(type ValueLabel (primitive ValueLabel)) -(type UnwindInst (primitive UnwindInst)) ;; An ALU operation. (type ALUOp @@ -2908,5 +2903,3 @@ (decl fcmp_reg (Type Reg Reg) ProducesFlags) (rule (fcmp_reg $F32 src1 src2) (fpu_cmp32 src1 src2)) (rule (fcmp_reg $F64 src1 src2) (fpu_cmp64 src1 src2)) - - diff --git a/cranelift/codegen/src/isa/s390x/lower/isle/generated_code.manifest b/cranelift/codegen/src/isa/s390x/lower/isle/generated_code.manifest index f829980d23c7..092b167d3a6a 100644 --- a/cranelift/codegen/src/isa/s390x/lower/isle/generated_code.manifest +++ b/cranelift/codegen/src/isa/s390x/lower/isle/generated_code.manifest @@ -1,4 +1,4 @@ src/clif.isle 9ea75a6f790b5c03 -src/prelude.isle 51d2aef2566c1c96 -src/isa/s390x/inst.isle 17b77476355c4509 +src/prelude.isle 3d00cfa50c4d52d9 +src/isa/s390x/inst.isle ded9e2516999aa15 src/isa/s390x/lower.isle a0e21a567040bc33 diff --git a/cranelift/codegen/src/isa/s390x/lower/isle/generated_code.rs b/cranelift/codegen/src/isa/s390x/lower/isle/generated_code.rs index ea1edb2ebfb3..cf0ee2b052c7 100644 --- a/cranelift/codegen/src/isa/s390x/lower/isle/generated_code.rs +++ b/cranelift/codegen/src/isa/s390x/lower/isle/generated_code.rs @@ -124,19 +124,19 @@ pub trait Context { fn emit(&mut self, arg0: &MInst) -> Unit; } -/// Internal type SideEffectNoResult: defined at src/prelude.isle line 282. +/// Internal type SideEffectNoResult: defined at src/prelude.isle line 294. #[derive(Clone, Debug)] pub enum SideEffectNoResult { Inst { inst: MInst }, } -/// Internal type ProducesFlags: defined at src/prelude.isle line 295. +/// Internal type ProducesFlags: defined at src/prelude.isle line 307. #[derive(Clone, Debug)] pub enum ProducesFlags { ProducesFlags { inst: MInst, result: Reg }, } -/// Internal type ConsumesFlags: defined at src/prelude.isle line 298. +/// Internal type ConsumesFlags: defined at src/prelude.isle line 310. #[derive(Clone, Debug)] pub enum ConsumesFlags { ConsumesFlags { inst: MInst, result: Reg }, @@ -649,8 +649,8 @@ pub enum MInst { }, } -/// Internal type ALUOp: defined at src/isa/s390x/inst.isle line 689. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type ALUOp: defined at src/isa/s390x/inst.isle line 684. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum ALUOp { Add32, Add32Ext16, @@ -687,8 +687,8 @@ pub enum ALUOp { XorNot64, } -/// Internal type UnaryOp: defined at src/isa/s390x/inst.isle line 730. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type UnaryOp: defined at src/isa/s390x/inst.isle line 725. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum UnaryOp { Abs32, Abs64, @@ -700,8 +700,8 @@ pub enum UnaryOp { PopcntReg, } -/// Internal type ShiftOp: defined at src/isa/s390x/inst.isle line 743. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type ShiftOp: defined at src/isa/s390x/inst.isle line 738. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum ShiftOp { RotL32, RotL64, @@ -713,8 +713,8 @@ pub enum ShiftOp { AShR64, } -/// Internal type CmpOp: defined at src/isa/s390x/inst.isle line 756. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type CmpOp: defined at src/isa/s390x/inst.isle line 751. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum CmpOp { CmpS32, CmpS32Ext16, @@ -728,8 +728,8 @@ pub enum CmpOp { CmpL64Ext32, } -/// Internal type FPUOp1: defined at src/isa/s390x/inst.isle line 771. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type FPUOp1: defined at src/isa/s390x/inst.isle line 766. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum FPUOp1 { Abs32, Abs64, @@ -743,8 +743,8 @@ pub enum FPUOp1 { Cvt64To32, } -/// Internal type FPUOp2: defined at src/isa/s390x/inst.isle line 786. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type FPUOp2: defined at src/isa/s390x/inst.isle line 781. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum FPUOp2 { Add32, Add64, @@ -760,8 +760,8 @@ pub enum FPUOp2 { Min64, } -/// Internal type FPUOp3: defined at src/isa/s390x/inst.isle line 803. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type FPUOp3: defined at src/isa/s390x/inst.isle line 798. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum FPUOp3 { MAdd32, MAdd64, @@ -769,8 +769,8 @@ pub enum FPUOp3 { MSub64, } -/// Internal type FpuToIntOp: defined at src/isa/s390x/inst.isle line 812. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type FpuToIntOp: defined at src/isa/s390x/inst.isle line 807. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum FpuToIntOp { F32ToU32, F32ToI32, @@ -782,8 +782,8 @@ pub enum FpuToIntOp { F64ToI64, } -/// Internal type IntToFpuOp: defined at src/isa/s390x/inst.isle line 825. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type IntToFpuOp: defined at src/isa/s390x/inst.isle line 820. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum IntToFpuOp { U32ToF32, I32ToF32, @@ -795,8 +795,8 @@ pub enum IntToFpuOp { I64ToF64, } -/// Internal type FpuRoundMode: defined at src/isa/s390x/inst.isle line 839. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type FpuRoundMode: defined at src/isa/s390x/inst.isle line 834. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum FpuRoundMode { Minus32, Minus64, @@ -808,19 +808,19 @@ pub enum FpuRoundMode { Nearest64, } -/// Internal type WritableRegPair: defined at src/isa/s390x/inst.isle line 1236. +/// Internal type WritableRegPair: defined at src/isa/s390x/inst.isle line 1231. #[derive(Clone, Debug)] pub enum WritableRegPair { WritableRegPair { hi: WritableReg, lo: WritableReg }, } -/// Internal type RegPair: defined at src/isa/s390x/inst.isle line 1258. +/// Internal type RegPair: defined at src/isa/s390x/inst.isle line 1253. #[derive(Clone, Debug)] pub enum RegPair { RegPair { hi: Reg, lo: Reg }, } -/// Internal type ProducesBool: defined at src/isa/s390x/inst.isle line 2158. +/// Internal type ProducesBool: defined at src/isa/s390x/inst.isle line 2153. #[derive(Clone, Debug)] pub enum ProducesBool { ProducesBool { producer: ProducesFlags, cond: Cond }, @@ -829,7 +829,7 @@ pub enum ProducesBool { // Generated as internal constructor for term temp_reg. pub fn constructor_temp_reg(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; - // Rule at src/prelude.isle line 70. + // Rule at src/prelude.isle line 73. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); return Some(expr1_0); @@ -838,7 +838,7 @@ pub fn constructor_temp_reg(ctx: &mut C, arg0: Type) -> Option // Generated as internal constructor for term lo_reg. pub fn constructor_lo_reg(ctx: &mut C, arg0: Value) -> Option { let pattern0_0 = arg0; - // Rule at src/prelude.isle line 105. + // Rule at src/prelude.isle line 108. let expr0_0 = C::put_in_regs(ctx, pattern0_0); let expr1_0: usize = 0; let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); @@ -855,7 +855,7 @@ pub fn constructor_value_regs_none( inst: ref pattern1_0, } = pattern0_0 { - // Rule at src/prelude.isle line 287. + // Rule at src/prelude.isle line 299. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::value_regs_invalid(ctx); return Some(expr1_0); @@ -881,7 +881,7 @@ pub fn constructor_with_flags( result: pattern3_1, } = pattern2_0 { - // Rule at src/prelude.isle line 308. + // Rule at src/prelude.isle line 320. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern3_0); let expr2_0 = C::value_regs(ctx, pattern1_1, pattern3_1); @@ -909,7 +909,7 @@ pub fn constructor_with_flags_1( result: pattern3_1, } = pattern2_0 { - // Rule at src/prelude.isle line 316. + // Rule at src/prelude.isle line 328. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern3_0); return Some(pattern3_1); @@ -943,7 +943,7 @@ pub fn constructor_with_flags_2( result: pattern5_1, } = pattern4_0 { - // Rule at src/prelude.isle line 326. + // Rule at src/prelude.isle line 338. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern5_0); let expr2_0 = C::emit(ctx, &pattern3_0); @@ -960,7 +960,7 @@ pub fn constructor_mask_amt_reg(ctx: &mut C, arg0: Type, arg1: Reg) let pattern0_0 = arg0; if let Some(pattern1_0) = C::gpr32_ty(ctx, pattern0_0) { let pattern2_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1014. + // Rule at src/isa/s390x/inst.isle line 1009. let expr0_0: i64 = -1; let expr1_0 = C::mask_amt_imm(ctx, pattern1_0, expr0_0); let expr2_0 = C::u8_as_u16(ctx, expr1_0); @@ -971,7 +971,7 @@ pub fn constructor_mask_amt_reg(ctx: &mut C, arg0: Type, arg1: Reg) } if let Some(pattern1_0) = C::gpr64_ty(ctx, pattern0_0) { let pattern2_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1017. + // Rule at src/isa/s390x/inst.isle line 1012. return Some(pattern2_0); } return None; @@ -998,7 +998,7 @@ pub fn constructor_lower_address( let pattern7_0 = arg2; let pattern8_0 = C::i64_from_offset(ctx, pattern7_0); if pattern8_0 == 0 { - // Rule at src/isa/s390x/inst.isle line 1100. + // Rule at src/isa/s390x/inst.isle line 1095. let expr0_0 = C::put_in_reg(ctx, pattern6_0); let expr1_0 = C::put_in_reg(ctx, pattern6_1); let expr2_0 = C::memarg_reg_plus_reg(ctx, expr0_0, expr1_0, pattern0_0); @@ -1017,7 +1017,7 @@ pub fn constructor_lower_address( if let Some(pattern8_0) = C::memarg_symbol_offset_sum(ctx, pattern6_0, pattern7_0) { - // Rule at src/isa/s390x/inst.isle line 1103. + // Rule at src/isa/s390x/inst.isle line 1098. let expr0_0 = C::memarg_symbol(ctx, pattern3_0, pattern8_0, pattern0_0); return Some(expr0_0); } @@ -1027,7 +1027,7 @@ pub fn constructor_lower_address( } let pattern2_0 = arg2; let pattern3_0 = C::i64_from_offset(ctx, pattern2_0); - // Rule at src/isa/s390x/inst.isle line 1097. + // Rule at src/isa/s390x/inst.isle line 1092. let expr0_0 = C::put_in_reg(ctx, pattern1_0); let expr1_0 = C::memarg_reg_plus_off(ctx, expr0_0, pattern3_0, pattern0_0); return Some(expr1_0); @@ -1043,7 +1043,7 @@ pub fn constructor_stack_addr_impl( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1130. + // Rule at src/isa/s390x/inst.isle line 1125. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::abi_stackslot_addr(ctx, expr0_0, pattern1_0, pattern2_0); let expr2_0 = C::emit(ctx, &expr1_0); @@ -1063,7 +1063,7 @@ pub fn constructor_sink_load(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C) -> Option { - // Rule at src/isa/s390x/inst.isle line 1241. + // Rule at src/isa/s390x/inst.isle line 1236. let expr0_0: u8 = 0; let expr1_0 = C::writable_gpr(ctx, expr0_0); let expr2_0: u8 = 1; @@ -1176,7 +1176,7 @@ pub fn constructor_copy_writable_regpair( arg0: &RegPair, ) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/s390x/inst.isle line 1247. + // Rule at src/isa/s390x/inst.isle line 1242. let expr0_0 = constructor_temp_writable_regpair(ctx)?; return Some(expr0_0); } @@ -1192,7 +1192,7 @@ pub fn constructor_writable_regpair_hi( lo: pattern1_1, } = pattern0_0 { - // Rule at src/isa/s390x/inst.isle line 1251. + // Rule at src/isa/s390x/inst.isle line 1246. return Some(pattern1_0); } return None; @@ -1209,7 +1209,7 @@ pub fn constructor_writable_regpair_lo( lo: pattern1_1, } = pattern0_0 { - // Rule at src/isa/s390x/inst.isle line 1255. + // Rule at src/isa/s390x/inst.isle line 1250. return Some(pattern1_1); } return None; @@ -1226,7 +1226,7 @@ pub fn constructor_writable_regpair_to_regpair( lo: pattern1_1, } = pattern0_0 { - // Rule at src/isa/s390x/inst.isle line 1262. + // Rule at src/isa/s390x/inst.isle line 1257. let expr0_0 = C::writable_reg_to_reg(ctx, pattern1_0); let expr1_0 = C::writable_reg_to_reg(ctx, pattern1_1); let expr2_0 = RegPair::RegPair { @@ -1240,7 +1240,7 @@ pub fn constructor_writable_regpair_to_regpair( // Generated as internal constructor for term uninitialized_regpair. pub fn constructor_uninitialized_regpair(ctx: &mut C) -> Option { - // Rule at src/isa/s390x/inst.isle line 1267. + // Rule at src/isa/s390x/inst.isle line 1262. let expr0_0 = constructor_temp_writable_regpair(ctx)?; let expr1_0 = constructor_writable_regpair_to_regpair(ctx, &expr0_0)?; return Some(expr1_0); @@ -1254,7 +1254,7 @@ pub fn constructor_regpair_hi(ctx: &mut C, arg0: &RegPair) -> Option lo: pattern1_1, } = pattern0_0 { - // Rule at src/isa/s390x/inst.isle line 1272. + // Rule at src/isa/s390x/inst.isle line 1267. return Some(pattern1_0); } return None; @@ -1268,7 +1268,7 @@ pub fn constructor_regpair_lo(ctx: &mut C, arg0: &RegPair) -> Option lo: pattern1_1, } = pattern0_0 { - // Rule at src/isa/s390x/inst.isle line 1276. + // Rule at src/isa/s390x/inst.isle line 1271. return Some(pattern1_1); } return None; @@ -1286,7 +1286,7 @@ pub fn constructor_alu_rrr( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 1290. + // Rule at src/isa/s390x/inst.isle line 1285. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::AluRRR { alu_op: pattern1_0.clone(), @@ -1311,7 +1311,7 @@ pub fn constructor_alu_rrsimm16( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 1297. + // Rule at src/isa/s390x/inst.isle line 1292. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::AluRRSImm16 { alu_op: pattern1_0.clone(), @@ -1336,7 +1336,7 @@ pub fn constructor_alu_rr( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 1304. + // Rule at src/isa/s390x/inst.isle line 1299. let expr0_0 = constructor_copy_writable_reg(ctx, pattern0_0, pattern2_0)?; let expr1_0 = MInst::AluRR { alu_op: pattern1_0.clone(), @@ -1360,7 +1360,7 @@ pub fn constructor_alu_rx( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 1311. + // Rule at src/isa/s390x/inst.isle line 1306. let expr0_0 = constructor_copy_writable_reg(ctx, pattern0_0, pattern2_0)?; let expr1_0 = MInst::AluRX { alu_op: pattern1_0.clone(), @@ -1384,7 +1384,7 @@ pub fn constructor_alu_rsimm16( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 1318. + // Rule at src/isa/s390x/inst.isle line 1313. let expr0_0 = constructor_copy_writable_reg(ctx, pattern0_0, pattern2_0)?; let expr1_0 = MInst::AluRSImm16 { alu_op: pattern1_0.clone(), @@ -1408,7 +1408,7 @@ pub fn constructor_alu_rsimm32( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 1325. + // Rule at src/isa/s390x/inst.isle line 1320. let expr0_0 = constructor_copy_writable_reg(ctx, pattern0_0, pattern2_0)?; let expr1_0 = MInst::AluRSImm32 { alu_op: pattern1_0.clone(), @@ -1432,7 +1432,7 @@ pub fn constructor_alu_ruimm32( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 1332. + // Rule at src/isa/s390x/inst.isle line 1327. let expr0_0 = constructor_copy_writable_reg(ctx, pattern0_0, pattern2_0)?; let expr1_0 = MInst::AluRUImm32 { alu_op: pattern1_0.clone(), @@ -1456,7 +1456,7 @@ pub fn constructor_alu_ruimm16shifted( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 1339. + // Rule at src/isa/s390x/inst.isle line 1334. let expr0_0 = constructor_copy_writable_reg(ctx, pattern0_0, pattern2_0)?; let expr1_0 = MInst::AluRUImm16Shifted { alu_op: pattern1_0.clone(), @@ -1480,7 +1480,7 @@ pub fn constructor_alu_ruimm32shifted( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 1346. + // Rule at src/isa/s390x/inst.isle line 1341. let expr0_0 = constructor_copy_writable_reg(ctx, pattern0_0, pattern2_0)?; let expr1_0 = MInst::AluRUImm32Shifted { alu_op: pattern1_0.clone(), @@ -1496,7 +1496,7 @@ pub fn constructor_alu_ruimm32shifted( pub fn constructor_smul_wide(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1353. + // Rule at src/isa/s390x/inst.isle line 1348. let expr0_0 = constructor_temp_writable_regpair(ctx)?; let expr1_0 = MInst::SMulWide { rn: pattern0_0, @@ -1511,7 +1511,7 @@ pub fn constructor_smul_wide(ctx: &mut C, arg0: Reg, arg1: Reg) -> O pub fn constructor_umul_wide(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1360. + // Rule at src/isa/s390x/inst.isle line 1355. let expr0_0 = constructor_temp_writable_regpair(ctx)?; let expr1_0 = constructor_writable_regpair_lo(ctx, &expr0_0)?; let expr2_0 = MInst::Mov64 { @@ -1533,7 +1533,7 @@ pub fn constructor_sdivmod32( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1368. + // Rule at src/isa/s390x/inst.isle line 1363. let expr0_0 = constructor_copy_writable_regpair(ctx, pattern0_0)?; let expr1_0 = MInst::SDivMod32 { rn: pattern1_0 }; let expr2_0 = C::emit(ctx, &expr1_0); @@ -1549,7 +1549,7 @@ pub fn constructor_sdivmod64( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1375. + // Rule at src/isa/s390x/inst.isle line 1370. let expr0_0 = constructor_copy_writable_regpair(ctx, pattern0_0)?; let expr1_0 = MInst::SDivMod64 { rn: pattern1_0 }; let expr2_0 = C::emit(ctx, &expr1_0); @@ -1565,7 +1565,7 @@ pub fn constructor_udivmod32( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1382. + // Rule at src/isa/s390x/inst.isle line 1377. let expr0_0 = constructor_copy_writable_regpair(ctx, pattern0_0)?; let expr1_0 = MInst::UDivMod32 { rn: pattern1_0 }; let expr2_0 = C::emit(ctx, &expr1_0); @@ -1581,7 +1581,7 @@ pub fn constructor_udivmod64( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1389. + // Rule at src/isa/s390x/inst.isle line 1384. let expr0_0 = constructor_copy_writable_regpair(ctx, pattern0_0)?; let expr1_0 = MInst::UDivMod64 { rn: pattern1_0 }; let expr2_0 = C::emit(ctx, &expr1_0); @@ -1603,7 +1603,7 @@ pub fn constructor_shift_rr( let pattern2_0 = arg2; let pattern3_0 = arg3; let pattern4_0 = arg4; - // Rule at src/isa/s390x/inst.isle line 1396. + // Rule at src/isa/s390x/inst.isle line 1391. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::ShiftRR { shift_op: pattern1_0.clone(), @@ -1627,7 +1627,7 @@ pub fn constructor_unary_rr( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1403. + // Rule at src/isa/s390x/inst.isle line 1398. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::UnaryRR { op: pattern1_0.clone(), @@ -1649,7 +1649,7 @@ pub fn constructor_cmp_rr( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1410. + // Rule at src/isa/s390x/inst.isle line 1405. let expr0_0 = MInst::CmpRR { op: pattern0_0.clone(), rn: pattern1_0, @@ -1673,7 +1673,7 @@ pub fn constructor_cmp_rx( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1416. + // Rule at src/isa/s390x/inst.isle line 1411. let expr0_0 = MInst::CmpRX { op: pattern0_0.clone(), rn: pattern1_0, @@ -1697,7 +1697,7 @@ pub fn constructor_cmp_rsimm16( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1422. + // Rule at src/isa/s390x/inst.isle line 1417. let expr0_0 = MInst::CmpRSImm16 { op: pattern0_0.clone(), rn: pattern1_0, @@ -1721,7 +1721,7 @@ pub fn constructor_cmp_rsimm32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1428. + // Rule at src/isa/s390x/inst.isle line 1423. let expr0_0 = MInst::CmpRSImm32 { op: pattern0_0.clone(), rn: pattern1_0, @@ -1745,7 +1745,7 @@ pub fn constructor_cmp_ruimm32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1434. + // Rule at src/isa/s390x/inst.isle line 1429. let expr0_0 = MInst::CmpRUImm32 { op: pattern0_0.clone(), rn: pattern1_0, @@ -1771,7 +1771,7 @@ pub fn constructor_atomic_rmw_impl( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 1440. + // Rule at src/isa/s390x/inst.isle line 1435. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::AtomicRmw { alu_op: pattern1_0.clone(), @@ -1794,7 +1794,7 @@ pub fn constructor_atomic_cas32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1447. + // Rule at src/isa/s390x/inst.isle line 1442. let expr0_0: Type = I32; let expr1_0 = constructor_copy_writable_reg(ctx, expr0_0, pattern0_0)?; let expr2_0 = MInst::AtomicCas32 { @@ -1817,7 +1817,7 @@ pub fn constructor_atomic_cas64( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1454. + // Rule at src/isa/s390x/inst.isle line 1449. let expr0_0: Type = I64; let expr1_0 = constructor_copy_writable_reg(ctx, expr0_0, pattern0_0)?; let expr2_0 = MInst::AtomicCas64 { @@ -1832,7 +1832,7 @@ pub fn constructor_atomic_cas64( // Generated as internal constructor for term fence_impl. pub fn constructor_fence_impl(ctx: &mut C) -> Option { - // Rule at src/isa/s390x/inst.isle line 1461. + // Rule at src/isa/s390x/inst.isle line 1456. let expr0_0 = MInst::Fence; let expr1_0 = SideEffectNoResult::Inst { inst: expr0_0 }; return Some(expr1_0); @@ -1841,7 +1841,7 @@ pub fn constructor_fence_impl(ctx: &mut C) -> Option(ctx: &mut C, arg0: &MemArg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/s390x/inst.isle line 1466. + // Rule at src/isa/s390x/inst.isle line 1461. let expr0_0: Type = I32; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::Load32 { @@ -1856,7 +1856,7 @@ pub fn constructor_load32(ctx: &mut C, arg0: &MemArg) -> Option // Generated as internal constructor for term load64. pub fn constructor_load64(ctx: &mut C, arg0: &MemArg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/s390x/inst.isle line 1473. + // Rule at src/isa/s390x/inst.isle line 1468. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::Load64 { @@ -1871,7 +1871,7 @@ pub fn constructor_load64(ctx: &mut C, arg0: &MemArg) -> Option // Generated as internal constructor for term loadrev16. pub fn constructor_loadrev16(ctx: &mut C, arg0: &MemArg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/s390x/inst.isle line 1480. + // Rule at src/isa/s390x/inst.isle line 1475. let expr0_0: Type = I32; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::LoadRev16 { @@ -1886,7 +1886,7 @@ pub fn constructor_loadrev16(ctx: &mut C, arg0: &MemArg) -> Option(ctx: &mut C, arg0: &MemArg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/s390x/inst.isle line 1487. + // Rule at src/isa/s390x/inst.isle line 1482. let expr0_0: Type = I32; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::LoadRev32 { @@ -1901,7 +1901,7 @@ pub fn constructor_loadrev32(ctx: &mut C, arg0: &MemArg) -> Option(ctx: &mut C, arg0: &MemArg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/s390x/inst.isle line 1494. + // Rule at src/isa/s390x/inst.isle line 1489. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::LoadRev64 { @@ -1921,7 +1921,7 @@ pub fn constructor_store8( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1501. + // Rule at src/isa/s390x/inst.isle line 1496. let expr0_0 = MInst::Store8 { rd: pattern0_0, mem: pattern1_0.clone(), @@ -1938,7 +1938,7 @@ pub fn constructor_store16( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1506. + // Rule at src/isa/s390x/inst.isle line 1501. let expr0_0 = MInst::Store16 { rd: pattern0_0, mem: pattern1_0.clone(), @@ -1955,7 +1955,7 @@ pub fn constructor_store32( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1511. + // Rule at src/isa/s390x/inst.isle line 1506. let expr0_0 = MInst::Store32 { rd: pattern0_0, mem: pattern1_0.clone(), @@ -1972,7 +1972,7 @@ pub fn constructor_store64( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1516. + // Rule at src/isa/s390x/inst.isle line 1511. let expr0_0 = MInst::Store64 { rd: pattern0_0, mem: pattern1_0.clone(), @@ -1989,7 +1989,7 @@ pub fn constructor_store8_imm( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1521. + // Rule at src/isa/s390x/inst.isle line 1516. let expr0_0 = MInst::StoreImm8 { imm: pattern0_0, mem: pattern1_0.clone(), @@ -2006,7 +2006,7 @@ pub fn constructor_store16_imm( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1526. + // Rule at src/isa/s390x/inst.isle line 1521. let expr0_0 = MInst::StoreImm16 { imm: pattern0_0, mem: pattern1_0.clone(), @@ -2023,7 +2023,7 @@ pub fn constructor_store32_simm16( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1531. + // Rule at src/isa/s390x/inst.isle line 1526. let expr0_0 = MInst::StoreImm32SExt16 { imm: pattern0_0, mem: pattern1_0.clone(), @@ -2040,7 +2040,7 @@ pub fn constructor_store64_simm16( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1536. + // Rule at src/isa/s390x/inst.isle line 1531. let expr0_0 = MInst::StoreImm64SExt16 { imm: pattern0_0, mem: pattern1_0.clone(), @@ -2057,7 +2057,7 @@ pub fn constructor_storerev16( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1541. + // Rule at src/isa/s390x/inst.isle line 1536. let expr0_0 = MInst::StoreRev16 { rd: pattern0_0, mem: pattern1_0.clone(), @@ -2074,7 +2074,7 @@ pub fn constructor_storerev32( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1546. + // Rule at src/isa/s390x/inst.isle line 1541. let expr0_0 = MInst::StoreRev32 { rd: pattern0_0, mem: pattern1_0.clone(), @@ -2091,7 +2091,7 @@ pub fn constructor_storerev64( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1551. + // Rule at src/isa/s390x/inst.isle line 1546. let expr0_0 = MInst::StoreRev64 { rd: pattern0_0, mem: pattern1_0.clone(), @@ -2110,7 +2110,7 @@ pub fn constructor_fpu_rr( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1556. + // Rule at src/isa/s390x/inst.isle line 1551. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::FpuRR { fpu_op: pattern1_0.clone(), @@ -2134,7 +2134,7 @@ pub fn constructor_fpu_rrr( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 1563. + // Rule at src/isa/s390x/inst.isle line 1558. let expr0_0 = constructor_copy_writable_reg(ctx, pattern0_0, pattern2_0)?; let expr1_0 = MInst::FpuRRR { fpu_op: pattern1_0.clone(), @@ -2160,7 +2160,7 @@ pub fn constructor_fpu_rrrr( let pattern2_0 = arg2; let pattern3_0 = arg3; let pattern4_0 = arg4; - // Rule at src/isa/s390x/inst.isle line 1570. + // Rule at src/isa/s390x/inst.isle line 1565. let expr0_0 = constructor_copy_writable_reg(ctx, pattern0_0, pattern2_0)?; let expr1_0 = MInst::FpuRRRR { fpu_op: pattern1_0.clone(), @@ -2183,7 +2183,7 @@ pub fn constructor_fpu_copysign( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1577. + // Rule at src/isa/s390x/inst.isle line 1572. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::FpuCopysign { rd: expr0_0, @@ -2203,7 +2203,7 @@ pub fn constructor_fpu_cmp32( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1584. + // Rule at src/isa/s390x/inst.isle line 1579. let expr0_0 = MInst::FpuCmp32 { rn: pattern0_0, rm: pattern1_0, @@ -2224,7 +2224,7 @@ pub fn constructor_fpu_cmp64( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1590. + // Rule at src/isa/s390x/inst.isle line 1585. let expr0_0 = MInst::FpuCmp64 { rn: pattern0_0, rm: pattern1_0, @@ -2247,7 +2247,7 @@ pub fn constructor_fpu_to_int( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1596. + // Rule at src/isa/s390x/inst.isle line 1591. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::FpuToInt { op: pattern1_0.clone(), @@ -2272,7 +2272,7 @@ pub fn constructor_int_to_fpu( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1603. + // Rule at src/isa/s390x/inst.isle line 1598. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::IntToFpu { op: pattern1_0.clone(), @@ -2294,7 +2294,7 @@ pub fn constructor_fpu_round( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1610. + // Rule at src/isa/s390x/inst.isle line 1605. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::FpuRound { op: pattern1_0.clone(), @@ -2318,7 +2318,7 @@ pub fn constructor_fpuvec_rrr( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 1617. + // Rule at src/isa/s390x/inst.isle line 1612. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::FpuVecRRR { fpu_op: pattern1_0.clone(), @@ -2334,7 +2334,7 @@ pub fn constructor_fpuvec_rrr( // Generated as internal constructor for term mov_to_fpr. pub fn constructor_mov_to_fpr(ctx: &mut C, arg0: Reg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/s390x/inst.isle line 1624. + // Rule at src/isa/s390x/inst.isle line 1619. let expr0_0: Type = F64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::MovToFpr { @@ -2349,7 +2349,7 @@ pub fn constructor_mov_to_fpr(ctx: &mut C, arg0: Reg) -> Option // Generated as internal constructor for term mov_from_fpr. pub fn constructor_mov_from_fpr(ctx: &mut C, arg0: Reg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/s390x/inst.isle line 1631. + // Rule at src/isa/s390x/inst.isle line 1626. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::MovFromFpr { @@ -2364,7 +2364,7 @@ pub fn constructor_mov_from_fpr(ctx: &mut C, arg0: Reg) -> Option(ctx: &mut C, arg0: &MemArg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/s390x/inst.isle line 1638. + // Rule at src/isa/s390x/inst.isle line 1633. let expr0_0: Type = F32; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::FpuLoad32 { @@ -2379,7 +2379,7 @@ pub fn constructor_fpu_load32(ctx: &mut C, arg0: &MemArg) -> Option< // Generated as internal constructor for term fpu_load64. pub fn constructor_fpu_load64(ctx: &mut C, arg0: &MemArg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/s390x/inst.isle line 1645. + // Rule at src/isa/s390x/inst.isle line 1640. let expr0_0: Type = F64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::FpuLoad64 { @@ -2394,7 +2394,7 @@ pub fn constructor_fpu_load64(ctx: &mut C, arg0: &MemArg) -> Option< // Generated as internal constructor for term fpu_loadrev32. pub fn constructor_fpu_loadrev32(ctx: &mut C, arg0: &MemArg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/s390x/inst.isle line 1652. + // Rule at src/isa/s390x/inst.isle line 1647. let expr0_0: Type = F32; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::FpuLoadRev32 { @@ -2409,7 +2409,7 @@ pub fn constructor_fpu_loadrev32(ctx: &mut C, arg0: &MemArg) -> Opti // Generated as internal constructor for term fpu_loadrev64. pub fn constructor_fpu_loadrev64(ctx: &mut C, arg0: &MemArg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/s390x/inst.isle line 1659. + // Rule at src/isa/s390x/inst.isle line 1654. let expr0_0: Type = F64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::FpuLoadRev64 { @@ -2429,7 +2429,7 @@ pub fn constructor_fpu_store32( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1666. + // Rule at src/isa/s390x/inst.isle line 1661. let expr0_0 = MInst::FpuStore32 { rd: pattern0_0, mem: pattern1_0.clone(), @@ -2446,7 +2446,7 @@ pub fn constructor_fpu_store64( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1671. + // Rule at src/isa/s390x/inst.isle line 1666. let expr0_0 = MInst::FpuStore64 { rd: pattern0_0, mem: pattern1_0.clone(), @@ -2463,7 +2463,7 @@ pub fn constructor_fpu_storerev32( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1676. + // Rule at src/isa/s390x/inst.isle line 1671. let expr0_0 = MInst::FpuStoreRev32 { rd: pattern0_0, mem: pattern1_0.clone(), @@ -2480,7 +2480,7 @@ pub fn constructor_fpu_storerev64( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1681. + // Rule at src/isa/s390x/inst.isle line 1676. let expr0_0 = MInst::FpuStoreRev64 { rd: pattern0_0, mem: pattern1_0.clone(), @@ -2497,7 +2497,7 @@ pub fn constructor_load_ext_name_far( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1686. + // Rule at src/isa/s390x/inst.isle line 1681. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::LoadExtNameFar { @@ -2513,7 +2513,7 @@ pub fn constructor_load_ext_name_far( // Generated as internal constructor for term load_addr. pub fn constructor_load_addr(ctx: &mut C, arg0: &MemArg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/s390x/inst.isle line 1693. + // Rule at src/isa/s390x/inst.isle line 1688. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::LoadAddr { @@ -2533,7 +2533,7 @@ pub fn constructor_drop_flags(ctx: &mut C, arg0: &ProducesFlags) -> result: pattern1_1, } = pattern0_0 { - // Rule at src/isa/s390x/inst.isle line 1700. + // Rule at src/isa/s390x/inst.isle line 1695. let expr0_0 = C::emit(ctx, &pattern1_0); return Some(pattern1_1); } @@ -2551,7 +2551,7 @@ pub fn constructor_emit_mov( if pattern0_0 == F32 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1716. + // Rule at src/isa/s390x/inst.isle line 1711. let expr0_0 = MInst::FpuMove32 { rd: pattern2_0, rn: pattern3_0, @@ -2562,7 +2562,7 @@ pub fn constructor_emit_mov( if pattern0_0 == F64 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1719. + // Rule at src/isa/s390x/inst.isle line 1714. let expr0_0 = MInst::FpuMove64 { rd: pattern2_0, rn: pattern3_0, @@ -2573,7 +2573,7 @@ pub fn constructor_emit_mov( if let Some(pattern1_0) = C::gpr32_ty(ctx, pattern0_0) { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1710. + // Rule at src/isa/s390x/inst.isle line 1705. let expr0_0 = MInst::Mov32 { rd: pattern2_0, rm: pattern3_0, @@ -2584,7 +2584,7 @@ pub fn constructor_emit_mov( if let Some(pattern1_0) = C::gpr64_ty(ctx, pattern0_0) { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1713. + // Rule at src/isa/s390x/inst.isle line 1708. let expr0_0 = MInst::Mov64 { rd: pattern2_0, rm: pattern3_0, @@ -2603,7 +2603,7 @@ pub fn constructor_copy_writable_reg( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1724. + // Rule at src/isa/s390x/inst.isle line 1719. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = constructor_emit_mov(ctx, pattern0_0, expr0_0, pattern1_0)?; return Some(expr0_0); @@ -2613,7 +2613,7 @@ pub fn constructor_copy_writable_reg( pub fn constructor_copy_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1731. + // Rule at src/isa/s390x/inst.isle line 1726. let expr0_0 = constructor_copy_writable_reg(ctx, pattern0_0, pattern1_0)?; let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); return Some(expr1_0); @@ -2630,7 +2630,7 @@ pub fn constructor_emit_imm( if pattern0_0 == F32 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1786. + // Rule at src/isa/s390x/inst.isle line 1781. let expr0_0 = C::u64_as_u32(ctx, pattern3_0); let expr1_0 = MInst::LoadFpuConst32 { rd: pattern2_0, @@ -2642,7 +2642,7 @@ pub fn constructor_emit_imm( if pattern0_0 == F64 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1791. + // Rule at src/isa/s390x/inst.isle line 1786. let expr0_0 = MInst::LoadFpuConst64 { rd: pattern2_0, const_data: pattern3_0, @@ -2653,7 +2653,7 @@ pub fn constructor_emit_imm( if let Some(pattern1_0) = C::fits_in_16(ctx, pattern0_0) { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1740. + // Rule at src/isa/s390x/inst.isle line 1735. let expr0_0 = C::u64_as_i16(ctx, pattern3_0); let expr1_0 = MInst::Mov32SImm16 { rd: pattern2_0, @@ -2666,7 +2666,7 @@ pub fn constructor_emit_imm( let pattern2_0 = arg1; let pattern3_0 = arg2; if let Some(pattern4_0) = C::i16_from_u64(ctx, pattern3_0) { - // Rule at src/isa/s390x/inst.isle line 1744. + // Rule at src/isa/s390x/inst.isle line 1739. let expr0_0 = MInst::Mov32SImm16 { rd: pattern2_0, imm: pattern4_0, @@ -2674,7 +2674,7 @@ pub fn constructor_emit_imm( let expr1_0 = C::emit(ctx, &expr0_0); return Some(expr1_0); } - // Rule at src/isa/s390x/inst.isle line 1748. + // Rule at src/isa/s390x/inst.isle line 1743. let expr0_0 = C::u64_as_u32(ctx, pattern3_0); let expr1_0 = MInst::Mov32Imm { rd: pattern2_0, @@ -2688,14 +2688,14 @@ pub fn constructor_emit_imm( let pattern3_0 = arg2; if let Some(pattern4_0) = C::u64_nonzero_hipart(ctx, pattern3_0) { if let Some(pattern5_0) = C::u64_nonzero_lopart(ctx, pattern3_0) { - // Rule at src/isa/s390x/inst.isle line 1768. + // Rule at src/isa/s390x/inst.isle line 1763. let expr0_0 = constructor_emit_imm(ctx, pattern1_0, pattern2_0, pattern4_0)?; let expr1_0 = constructor_emit_insert_imm(ctx, pattern2_0, pattern5_0)?; return Some(expr1_0); } } if let Some(pattern4_0) = C::i16_from_u64(ctx, pattern3_0) { - // Rule at src/isa/s390x/inst.isle line 1752. + // Rule at src/isa/s390x/inst.isle line 1747. let expr0_0 = MInst::Mov64SImm16 { rd: pattern2_0, imm: pattern4_0, @@ -2704,7 +2704,7 @@ pub fn constructor_emit_imm( return Some(expr1_0); } if let Some(pattern4_0) = C::i32_from_u64(ctx, pattern3_0) { - // Rule at src/isa/s390x/inst.isle line 1756. + // Rule at src/isa/s390x/inst.isle line 1751. let expr0_0 = MInst::Mov64SImm32 { rd: pattern2_0, imm: pattern4_0, @@ -2713,7 +2713,7 @@ pub fn constructor_emit_imm( return Some(expr1_0); } if let Some(pattern4_0) = C::uimm32shifted_from_u64(ctx, pattern3_0) { - // Rule at src/isa/s390x/inst.isle line 1764. + // Rule at src/isa/s390x/inst.isle line 1759. let expr0_0 = MInst::Mov64UImm32Shifted { rd: pattern2_0, imm: pattern4_0, @@ -2722,7 +2722,7 @@ pub fn constructor_emit_imm( return Some(expr1_0); } if let Some(pattern4_0) = C::uimm16shifted_from_u64(ctx, pattern3_0) { - // Rule at src/isa/s390x/inst.isle line 1760. + // Rule at src/isa/s390x/inst.isle line 1755. let expr0_0 = MInst::Mov64UImm16Shifted { rd: pattern2_0, imm: pattern4_0, @@ -2743,7 +2743,7 @@ pub fn constructor_emit_insert_imm( let pattern0_0 = arg0; let pattern1_0 = arg1; if let Some(pattern2_0) = C::uimm32shifted_from_u64(ctx, pattern1_0) { - // Rule at src/isa/s390x/inst.isle line 1781. + // Rule at src/isa/s390x/inst.isle line 1776. let expr0_0 = MInst::Insert64UImm32Shifted { rd: pattern0_0, imm: pattern2_0, @@ -2752,7 +2752,7 @@ pub fn constructor_emit_insert_imm( return Some(expr1_0); } if let Some(pattern2_0) = C::uimm16shifted_from_u64(ctx, pattern1_0) { - // Rule at src/isa/s390x/inst.isle line 1777. + // Rule at src/isa/s390x/inst.isle line 1772. let expr0_0 = MInst::Insert64UImm16Shifted { rd: pattern0_0, imm: pattern2_0, @@ -2767,7 +2767,7 @@ pub fn constructor_emit_insert_imm( pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1796. + // Rule at src/isa/s390x/inst.isle line 1791. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = constructor_emit_imm(ctx, pattern0_0, expr0_0, pattern1_0)?; let expr2_0 = C::writable_reg_to_reg(ctx, expr0_0); @@ -2784,7 +2784,7 @@ pub fn constructor_imm_regpair_lo( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1804. + // Rule at src/isa/s390x/inst.isle line 1799. let expr0_0 = constructor_copy_writable_regpair(ctx, pattern2_0)?; let expr1_0 = constructor_writable_regpair_lo(ctx, &expr0_0)?; let expr2_0 = constructor_emit_imm(ctx, pattern0_0, expr1_0, pattern1_0)?; @@ -2802,7 +2802,7 @@ pub fn constructor_imm_regpair_hi( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1812. + // Rule at src/isa/s390x/inst.isle line 1807. let expr0_0 = constructor_copy_writable_regpair(ctx, pattern2_0)?; let expr1_0 = constructor_writable_regpair_hi(ctx, &expr0_0)?; let expr2_0 = constructor_emit_imm(ctx, pattern0_0, expr1_0, pattern1_0)?; @@ -2814,22 +2814,22 @@ pub fn constructor_imm_regpair_hi( pub fn constructor_ty_ext32(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I8 { - // Rule at src/isa/s390x/inst.isle line 1822. + // Rule at src/isa/s390x/inst.isle line 1817. let expr0_0: Type = I32; return Some(expr0_0); } if pattern0_0 == I16 { - // Rule at src/isa/s390x/inst.isle line 1823. + // Rule at src/isa/s390x/inst.isle line 1818. let expr0_0: Type = I32; return Some(expr0_0); } if pattern0_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 1824. + // Rule at src/isa/s390x/inst.isle line 1819. let expr0_0: Type = I32; return Some(expr0_0); } if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 1825. + // Rule at src/isa/s390x/inst.isle line 1820. let expr0_0: Type = I64; return Some(expr0_0); } @@ -2840,22 +2840,22 @@ pub fn constructor_ty_ext32(ctx: &mut C, arg0: Type) -> Option pub fn constructor_ty_ext64(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I8 { - // Rule at src/isa/s390x/inst.isle line 1829. + // Rule at src/isa/s390x/inst.isle line 1824. let expr0_0: Type = I64; return Some(expr0_0); } if pattern0_0 == I16 { - // Rule at src/isa/s390x/inst.isle line 1830. + // Rule at src/isa/s390x/inst.isle line 1825. let expr0_0: Type = I64; return Some(expr0_0); } if pattern0_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 1831. + // Rule at src/isa/s390x/inst.isle line 1826. let expr0_0: Type = I64; return Some(expr0_0); } if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 1832. + // Rule at src/isa/s390x/inst.isle line 1827. let expr0_0: Type = I64; return Some(expr0_0); } @@ -2872,7 +2872,7 @@ pub fn constructor_emit_zext32_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1837. + // Rule at src/isa/s390x/inst.isle line 1832. let expr0_0: bool = false; let expr1_0 = C::ty_bits(ctx, pattern1_0); let expr2_0: u8 = 32; @@ -2897,7 +2897,7 @@ pub fn constructor_emit_sext32_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1843. + // Rule at src/isa/s390x/inst.isle line 1838. let expr0_0: bool = true; let expr1_0 = C::ty_bits(ctx, pattern1_0); let expr2_0: u8 = 32; @@ -2922,7 +2922,7 @@ pub fn constructor_emit_zext64_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1849. + // Rule at src/isa/s390x/inst.isle line 1844. let expr0_0: bool = false; let expr1_0 = C::ty_bits(ctx, pattern1_0); let expr2_0: u8 = 64; @@ -2947,7 +2947,7 @@ pub fn constructor_emit_sext64_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1855. + // Rule at src/isa/s390x/inst.isle line 1850. let expr0_0: bool = true; let expr1_0 = C::ty_bits(ctx, pattern1_0); let expr2_0: u8 = 64; @@ -2966,7 +2966,7 @@ pub fn constructor_emit_sext64_reg( pub fn constructor_zext32_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1861. + // Rule at src/isa/s390x/inst.isle line 1856. let expr0_0: Type = I32; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = constructor_emit_zext32_reg(ctx, expr1_0, pattern0_0, pattern1_0)?; @@ -2978,7 +2978,7 @@ pub fn constructor_zext32_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> pub fn constructor_sext32_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1869. + // Rule at src/isa/s390x/inst.isle line 1864. let expr0_0: Type = I32; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = constructor_emit_sext32_reg(ctx, expr1_0, pattern0_0, pattern1_0)?; @@ -2990,7 +2990,7 @@ pub fn constructor_sext32_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> pub fn constructor_zext64_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1877. + // Rule at src/isa/s390x/inst.isle line 1872. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = constructor_emit_zext64_reg(ctx, expr1_0, pattern0_0, pattern1_0)?; @@ -3002,7 +3002,7 @@ pub fn constructor_zext64_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> pub fn constructor_sext64_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1885. + // Rule at src/isa/s390x/inst.isle line 1880. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = constructor_emit_sext64_reg(ctx, expr1_0, pattern0_0, pattern1_0)?; @@ -3021,7 +3021,7 @@ pub fn constructor_emit_zext32_mem( let pattern1_0 = arg1; if pattern1_0 == I8 { let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1893. + // Rule at src/isa/s390x/inst.isle line 1888. let expr0_0 = MInst::Load32ZExt8 { rd: pattern0_0, mem: pattern3_0.clone(), @@ -3031,7 +3031,7 @@ pub fn constructor_emit_zext32_mem( } if pattern1_0 == I16 { let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1894. + // Rule at src/isa/s390x/inst.isle line 1889. let expr0_0 = MInst::Load32ZExt16 { rd: pattern0_0, mem: pattern3_0.clone(), @@ -3053,7 +3053,7 @@ pub fn constructor_emit_sext32_mem( let pattern1_0 = arg1; if pattern1_0 == I8 { let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1898. + // Rule at src/isa/s390x/inst.isle line 1893. let expr0_0 = MInst::Load32SExt8 { rd: pattern0_0, mem: pattern3_0.clone(), @@ -3063,7 +3063,7 @@ pub fn constructor_emit_sext32_mem( } if pattern1_0 == I16 { let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1899. + // Rule at src/isa/s390x/inst.isle line 1894. let expr0_0 = MInst::Load32SExt16 { rd: pattern0_0, mem: pattern3_0.clone(), @@ -3085,7 +3085,7 @@ pub fn constructor_emit_zext64_mem( let pattern1_0 = arg1; if pattern1_0 == I8 { let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1903. + // Rule at src/isa/s390x/inst.isle line 1898. let expr0_0 = MInst::Load64ZExt8 { rd: pattern0_0, mem: pattern3_0.clone(), @@ -3095,7 +3095,7 @@ pub fn constructor_emit_zext64_mem( } if pattern1_0 == I16 { let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1904. + // Rule at src/isa/s390x/inst.isle line 1899. let expr0_0 = MInst::Load64ZExt16 { rd: pattern0_0, mem: pattern3_0.clone(), @@ -3105,7 +3105,7 @@ pub fn constructor_emit_zext64_mem( } if pattern1_0 == I32 { let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1905. + // Rule at src/isa/s390x/inst.isle line 1900. let expr0_0 = MInst::Load64ZExt32 { rd: pattern0_0, mem: pattern3_0.clone(), @@ -3127,7 +3127,7 @@ pub fn constructor_emit_sext64_mem( let pattern1_0 = arg1; if pattern1_0 == I8 { let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1909. + // Rule at src/isa/s390x/inst.isle line 1904. let expr0_0 = MInst::Load64SExt8 { rd: pattern0_0, mem: pattern3_0.clone(), @@ -3137,7 +3137,7 @@ pub fn constructor_emit_sext64_mem( } if pattern1_0 == I16 { let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1910. + // Rule at src/isa/s390x/inst.isle line 1905. let expr0_0 = MInst::Load64SExt16 { rd: pattern0_0, mem: pattern3_0.clone(), @@ -3147,7 +3147,7 @@ pub fn constructor_emit_sext64_mem( } if pattern1_0 == I32 { let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1911. + // Rule at src/isa/s390x/inst.isle line 1906. let expr0_0 = MInst::Load64SExt32 { rd: pattern0_0, mem: pattern3_0.clone(), @@ -3162,7 +3162,7 @@ pub fn constructor_emit_sext64_mem( pub fn constructor_zext32_mem(ctx: &mut C, arg0: Type, arg1: &MemArg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1915. + // Rule at src/isa/s390x/inst.isle line 1910. let expr0_0: Type = I32; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = constructor_emit_zext32_mem(ctx, expr1_0, pattern0_0, pattern1_0)?; @@ -3174,7 +3174,7 @@ pub fn constructor_zext32_mem(ctx: &mut C, arg0: Type, arg1: &MemArg pub fn constructor_sext32_mem(ctx: &mut C, arg0: Type, arg1: &MemArg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1922. + // Rule at src/isa/s390x/inst.isle line 1917. let expr0_0: Type = I32; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = constructor_emit_sext32_mem(ctx, expr1_0, pattern0_0, pattern1_0)?; @@ -3186,7 +3186,7 @@ pub fn constructor_sext32_mem(ctx: &mut C, arg0: Type, arg1: &MemArg pub fn constructor_zext64_mem(ctx: &mut C, arg0: Type, arg1: &MemArg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1929. + // Rule at src/isa/s390x/inst.isle line 1924. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = constructor_emit_zext64_mem(ctx, expr1_0, pattern0_0, pattern1_0)?; @@ -3198,7 +3198,7 @@ pub fn constructor_zext64_mem(ctx: &mut C, arg0: Type, arg1: &MemArg pub fn constructor_sext64_mem(ctx: &mut C, arg0: Type, arg1: &MemArg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1936. + // Rule at src/isa/s390x/inst.isle line 1931. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = constructor_emit_sext64_mem(ctx, expr1_0, pattern0_0, pattern1_0)?; @@ -3216,7 +3216,7 @@ pub fn constructor_emit_put_in_reg_zext32( let pattern1_0 = arg1; let pattern2_0 = C::value_type(ctx, pattern1_0); if let Some(pattern3_0) = C::u64_from_value(ctx, pattern1_0) { - // Rule at src/isa/s390x/inst.isle line 1944. + // Rule at src/isa/s390x/inst.isle line 1939. let expr0_0 = constructor_ty_ext32(ctx, pattern2_0)?; let expr1_0 = constructor_emit_imm(ctx, expr0_0, pattern0_0, pattern3_0)?; return Some(expr1_0); @@ -3233,7 +3233,7 @@ pub fn constructor_emit_put_in_reg_zext32( { if let &Opcode::Load = &pattern6_0 { if let Some(()) = C::bigendian(ctx, pattern6_2) { - // Rule at src/isa/s390x/inst.isle line 1946. + // Rule at src/isa/s390x/inst.isle line 1941. let expr0_0 = constructor_sink_load(ctx, pattern4_0)?; let expr1_0 = constructor_emit_zext32_mem(ctx, pattern0_0, pattern3_0, &expr0_0)?; @@ -3242,13 +3242,13 @@ pub fn constructor_emit_put_in_reg_zext32( } } } - // Rule at src/isa/s390x/inst.isle line 1948. + // Rule at src/isa/s390x/inst.isle line 1943. let expr0_0 = C::put_in_reg(ctx, pattern1_0); let expr1_0 = constructor_emit_zext32_reg(ctx, pattern0_0, pattern3_0, expr0_0)?; return Some(expr1_0); } if let Some(pattern3_0) = C::ty_32_or_64(ctx, pattern2_0) { - // Rule at src/isa/s390x/inst.isle line 1950. + // Rule at src/isa/s390x/inst.isle line 1945. let expr0_0 = C::put_in_reg(ctx, pattern1_0); let expr1_0 = constructor_emit_mov(ctx, pattern3_0, pattern0_0, expr0_0)?; return Some(expr1_0); @@ -3266,7 +3266,7 @@ pub fn constructor_emit_put_in_reg_sext32( let pattern1_0 = arg1; let pattern2_0 = C::value_type(ctx, pattern1_0); if let Some(pattern3_0) = C::u64_from_signed_value(ctx, pattern1_0) { - // Rule at src/isa/s390x/inst.isle line 1955. + // Rule at src/isa/s390x/inst.isle line 1950. let expr0_0 = constructor_ty_ext32(ctx, pattern2_0)?; let expr1_0 = constructor_emit_imm(ctx, expr0_0, pattern0_0, pattern3_0)?; return Some(expr1_0); @@ -3283,7 +3283,7 @@ pub fn constructor_emit_put_in_reg_sext32( { if let &Opcode::Load = &pattern6_0 { if let Some(()) = C::bigendian(ctx, pattern6_2) { - // Rule at src/isa/s390x/inst.isle line 1957. + // Rule at src/isa/s390x/inst.isle line 1952. let expr0_0 = constructor_sink_load(ctx, pattern4_0)?; let expr1_0 = constructor_emit_sext32_mem(ctx, pattern0_0, pattern3_0, &expr0_0)?; @@ -3292,13 +3292,13 @@ pub fn constructor_emit_put_in_reg_sext32( } } } - // Rule at src/isa/s390x/inst.isle line 1959. + // Rule at src/isa/s390x/inst.isle line 1954. let expr0_0 = C::put_in_reg(ctx, pattern1_0); let expr1_0 = constructor_emit_sext32_reg(ctx, pattern0_0, pattern3_0, expr0_0)?; return Some(expr1_0); } if let Some(pattern3_0) = C::ty_32_or_64(ctx, pattern2_0) { - // Rule at src/isa/s390x/inst.isle line 1961. + // Rule at src/isa/s390x/inst.isle line 1956. let expr0_0 = C::put_in_reg(ctx, pattern1_0); let expr1_0 = constructor_emit_mov(ctx, pattern3_0, pattern0_0, expr0_0)?; return Some(expr1_0); @@ -3316,7 +3316,7 @@ pub fn constructor_emit_put_in_reg_zext64( let pattern1_0 = arg1; let pattern2_0 = C::value_type(ctx, pattern1_0); if let Some(pattern3_0) = C::u64_from_value(ctx, pattern1_0) { - // Rule at src/isa/s390x/inst.isle line 1966. + // Rule at src/isa/s390x/inst.isle line 1961. let expr0_0 = constructor_ty_ext64(ctx, pattern2_0)?; let expr1_0 = constructor_emit_imm(ctx, expr0_0, pattern0_0, pattern3_0)?; return Some(expr1_0); @@ -3333,7 +3333,7 @@ pub fn constructor_emit_put_in_reg_zext64( { if let &Opcode::Load = &pattern6_0 { if let Some(()) = C::bigendian(ctx, pattern6_2) { - // Rule at src/isa/s390x/inst.isle line 1968. + // Rule at src/isa/s390x/inst.isle line 1963. let expr0_0 = constructor_sink_load(ctx, pattern4_0)?; let expr1_0 = constructor_emit_zext64_mem(ctx, pattern0_0, pattern3_0, &expr0_0)?; @@ -3342,13 +3342,13 @@ pub fn constructor_emit_put_in_reg_zext64( } } } - // Rule at src/isa/s390x/inst.isle line 1970. + // Rule at src/isa/s390x/inst.isle line 1965. let expr0_0 = C::put_in_reg(ctx, pattern1_0); let expr1_0 = constructor_emit_zext64_reg(ctx, pattern0_0, pattern3_0, expr0_0)?; return Some(expr1_0); } if let Some(pattern3_0) = C::gpr64_ty(ctx, pattern2_0) { - // Rule at src/isa/s390x/inst.isle line 1972. + // Rule at src/isa/s390x/inst.isle line 1967. let expr0_0 = C::put_in_reg(ctx, pattern1_0); let expr1_0 = constructor_emit_mov(ctx, pattern3_0, pattern0_0, expr0_0)?; return Some(expr1_0); @@ -3366,7 +3366,7 @@ pub fn constructor_emit_put_in_reg_sext64( let pattern1_0 = arg1; let pattern2_0 = C::value_type(ctx, pattern1_0); if let Some(pattern3_0) = C::u64_from_signed_value(ctx, pattern1_0) { - // Rule at src/isa/s390x/inst.isle line 1977. + // Rule at src/isa/s390x/inst.isle line 1972. let expr0_0 = constructor_ty_ext64(ctx, pattern2_0)?; let expr1_0 = constructor_emit_imm(ctx, expr0_0, pattern0_0, pattern3_0)?; return Some(expr1_0); @@ -3383,7 +3383,7 @@ pub fn constructor_emit_put_in_reg_sext64( { if let &Opcode::Load = &pattern6_0 { if let Some(()) = C::bigendian(ctx, pattern6_2) { - // Rule at src/isa/s390x/inst.isle line 1979. + // Rule at src/isa/s390x/inst.isle line 1974. let expr0_0 = constructor_sink_load(ctx, pattern4_0)?; let expr1_0 = constructor_emit_sext64_mem(ctx, pattern0_0, pattern3_0, &expr0_0)?; @@ -3392,13 +3392,13 @@ pub fn constructor_emit_put_in_reg_sext64( } } } - // Rule at src/isa/s390x/inst.isle line 1981. + // Rule at src/isa/s390x/inst.isle line 1976. let expr0_0 = C::put_in_reg(ctx, pattern1_0); let expr1_0 = constructor_emit_sext64_reg(ctx, pattern0_0, pattern3_0, expr0_0)?; return Some(expr1_0); } if let Some(pattern3_0) = C::gpr64_ty(ctx, pattern2_0) { - // Rule at src/isa/s390x/inst.isle line 1983. + // Rule at src/isa/s390x/inst.isle line 1978. let expr0_0 = C::put_in_reg(ctx, pattern1_0); let expr1_0 = constructor_emit_mov(ctx, pattern3_0, pattern0_0, expr0_0)?; return Some(expr1_0); @@ -3411,7 +3411,7 @@ pub fn constructor_put_in_reg_zext32(ctx: &mut C, arg0: Value) -> Op let pattern0_0 = arg0; let pattern1_0 = C::value_type(ctx, pattern0_0); if let Some(pattern2_0) = C::u64_from_value(ctx, pattern0_0) { - // Rule at src/isa/s390x/inst.isle line 1988. + // Rule at src/isa/s390x/inst.isle line 1983. let expr0_0 = constructor_ty_ext32(ctx, pattern1_0)?; let expr1_0 = constructor_imm(ctx, expr0_0, pattern2_0)?; return Some(expr1_0); @@ -3428,7 +3428,7 @@ pub fn constructor_put_in_reg_zext32(ctx: &mut C, arg0: Value) -> Op { if let &Opcode::Load = &pattern5_0 { if let Some(()) = C::bigendian(ctx, pattern5_2) { - // Rule at src/isa/s390x/inst.isle line 1990. + // Rule at src/isa/s390x/inst.isle line 1985. let expr0_0 = constructor_sink_load(ctx, pattern3_0)?; let expr1_0 = constructor_zext32_mem(ctx, pattern2_0, &expr0_0)?; return Some(expr1_0); @@ -3436,13 +3436,13 @@ pub fn constructor_put_in_reg_zext32(ctx: &mut C, arg0: Value) -> Op } } } - // Rule at src/isa/s390x/inst.isle line 1992. + // Rule at src/isa/s390x/inst.isle line 1987. let expr0_0 = C::put_in_reg(ctx, pattern0_0); let expr1_0 = constructor_zext32_reg(ctx, pattern2_0, expr0_0)?; return Some(expr1_0); } if let Some(pattern2_0) = C::ty_32_or_64(ctx, pattern1_0) { - // Rule at src/isa/s390x/inst.isle line 1994. + // Rule at src/isa/s390x/inst.isle line 1989. let expr0_0 = C::put_in_reg(ctx, pattern0_0); return Some(expr0_0); } @@ -3454,7 +3454,7 @@ pub fn constructor_put_in_reg_sext32(ctx: &mut C, arg0: Value) -> Op let pattern0_0 = arg0; let pattern1_0 = C::value_type(ctx, pattern0_0); if let Some(pattern2_0) = C::u64_from_signed_value(ctx, pattern0_0) { - // Rule at src/isa/s390x/inst.isle line 1999. + // Rule at src/isa/s390x/inst.isle line 1994. let expr0_0 = constructor_ty_ext32(ctx, pattern1_0)?; let expr1_0 = constructor_imm(ctx, expr0_0, pattern2_0)?; return Some(expr1_0); @@ -3471,7 +3471,7 @@ pub fn constructor_put_in_reg_sext32(ctx: &mut C, arg0: Value) -> Op { if let &Opcode::Load = &pattern5_0 { if let Some(()) = C::bigendian(ctx, pattern5_2) { - // Rule at src/isa/s390x/inst.isle line 2001. + // Rule at src/isa/s390x/inst.isle line 1996. let expr0_0 = constructor_sink_load(ctx, pattern3_0)?; let expr1_0 = constructor_sext32_mem(ctx, pattern2_0, &expr0_0)?; return Some(expr1_0); @@ -3479,13 +3479,13 @@ pub fn constructor_put_in_reg_sext32(ctx: &mut C, arg0: Value) -> Op } } } - // Rule at src/isa/s390x/inst.isle line 2003. + // Rule at src/isa/s390x/inst.isle line 1998. let expr0_0 = C::put_in_reg(ctx, pattern0_0); let expr1_0 = constructor_sext32_reg(ctx, pattern2_0, expr0_0)?; return Some(expr1_0); } if let Some(pattern2_0) = C::ty_32_or_64(ctx, pattern1_0) { - // Rule at src/isa/s390x/inst.isle line 2005. + // Rule at src/isa/s390x/inst.isle line 2000. let expr0_0 = C::put_in_reg(ctx, pattern0_0); return Some(expr0_0); } @@ -3497,7 +3497,7 @@ pub fn constructor_put_in_reg_zext64(ctx: &mut C, arg0: Value) -> Op let pattern0_0 = arg0; let pattern1_0 = C::value_type(ctx, pattern0_0); if let Some(pattern2_0) = C::u64_from_value(ctx, pattern0_0) { - // Rule at src/isa/s390x/inst.isle line 2010. + // Rule at src/isa/s390x/inst.isle line 2005. let expr0_0 = constructor_ty_ext64(ctx, pattern1_0)?; let expr1_0 = constructor_imm(ctx, expr0_0, pattern2_0)?; return Some(expr1_0); @@ -3514,7 +3514,7 @@ pub fn constructor_put_in_reg_zext64(ctx: &mut C, arg0: Value) -> Op { if let &Opcode::Load = &pattern5_0 { if let Some(()) = C::bigendian(ctx, pattern5_2) { - // Rule at src/isa/s390x/inst.isle line 2012. + // Rule at src/isa/s390x/inst.isle line 2007. let expr0_0 = constructor_sink_load(ctx, pattern3_0)?; let expr1_0 = constructor_zext64_mem(ctx, pattern2_0, &expr0_0)?; return Some(expr1_0); @@ -3522,13 +3522,13 @@ pub fn constructor_put_in_reg_zext64(ctx: &mut C, arg0: Value) -> Op } } } - // Rule at src/isa/s390x/inst.isle line 2014. + // Rule at src/isa/s390x/inst.isle line 2009. let expr0_0 = C::put_in_reg(ctx, pattern0_0); let expr1_0 = constructor_zext64_reg(ctx, pattern2_0, expr0_0)?; return Some(expr1_0); } if let Some(pattern2_0) = C::gpr64_ty(ctx, pattern1_0) { - // Rule at src/isa/s390x/inst.isle line 2016. + // Rule at src/isa/s390x/inst.isle line 2011. let expr0_0 = C::put_in_reg(ctx, pattern0_0); return Some(expr0_0); } @@ -3540,7 +3540,7 @@ pub fn constructor_put_in_reg_sext64(ctx: &mut C, arg0: Value) -> Op let pattern0_0 = arg0; let pattern1_0 = C::value_type(ctx, pattern0_0); if let Some(pattern2_0) = C::u64_from_signed_value(ctx, pattern0_0) { - // Rule at src/isa/s390x/inst.isle line 2021. + // Rule at src/isa/s390x/inst.isle line 2016. let expr0_0 = constructor_ty_ext64(ctx, pattern1_0)?; let expr1_0 = constructor_imm(ctx, expr0_0, pattern2_0)?; return Some(expr1_0); @@ -3557,7 +3557,7 @@ pub fn constructor_put_in_reg_sext64(ctx: &mut C, arg0: Value) -> Op { if let &Opcode::Load = &pattern5_0 { if let Some(()) = C::bigendian(ctx, pattern5_2) { - // Rule at src/isa/s390x/inst.isle line 2023. + // Rule at src/isa/s390x/inst.isle line 2018. let expr0_0 = constructor_sink_load(ctx, pattern3_0)?; let expr1_0 = constructor_sext64_mem(ctx, pattern2_0, &expr0_0)?; return Some(expr1_0); @@ -3565,13 +3565,13 @@ pub fn constructor_put_in_reg_sext64(ctx: &mut C, arg0: Value) -> Op } } } - // Rule at src/isa/s390x/inst.isle line 2025. + // Rule at src/isa/s390x/inst.isle line 2020. let expr0_0 = C::put_in_reg(ctx, pattern0_0); let expr1_0 = constructor_sext64_reg(ctx, pattern2_0, expr0_0)?; return Some(expr1_0); } if let Some(pattern2_0) = C::gpr64_ty(ctx, pattern1_0) { - // Rule at src/isa/s390x/inst.isle line 2027. + // Rule at src/isa/s390x/inst.isle line 2022. let expr0_0 = C::put_in_reg(ctx, pattern0_0); return Some(expr0_0); } @@ -3586,7 +3586,7 @@ pub fn constructor_put_in_regpair_lo_zext32( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2033. + // Rule at src/isa/s390x/inst.isle line 2028. let expr0_0 = constructor_copy_writable_regpair(ctx, pattern1_0)?; let expr1_0 = constructor_writable_regpair_lo(ctx, &expr0_0)?; let expr2_0 = constructor_emit_put_in_reg_zext32(ctx, expr1_0, pattern0_0)?; @@ -3602,7 +3602,7 @@ pub fn constructor_put_in_regpair_lo_sext32( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2041. + // Rule at src/isa/s390x/inst.isle line 2036. let expr0_0 = constructor_copy_writable_regpair(ctx, pattern1_0)?; let expr1_0 = constructor_writable_regpair_lo(ctx, &expr0_0)?; let expr2_0 = constructor_emit_put_in_reg_sext32(ctx, expr1_0, pattern0_0)?; @@ -3618,7 +3618,7 @@ pub fn constructor_put_in_regpair_lo_zext64( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2049. + // Rule at src/isa/s390x/inst.isle line 2044. let expr0_0 = constructor_copy_writable_regpair(ctx, pattern1_0)?; let expr1_0 = constructor_writable_regpair_lo(ctx, &expr0_0)?; let expr2_0 = constructor_emit_put_in_reg_zext64(ctx, expr1_0, pattern0_0)?; @@ -3634,7 +3634,7 @@ pub fn constructor_put_in_regpair_lo_sext64( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2057. + // Rule at src/isa/s390x/inst.isle line 2052. let expr0_0 = constructor_copy_writable_regpair(ctx, pattern1_0)?; let expr1_0 = constructor_writable_regpair_lo(ctx, &expr0_0)?; let expr2_0 = constructor_emit_put_in_reg_sext64(ctx, expr1_0, pattern0_0)?; @@ -3655,7 +3655,7 @@ pub fn constructor_emit_cmov_imm( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 2067. + // Rule at src/isa/s390x/inst.isle line 2062. let expr0_0 = MInst::CMov32SImm16 { rd: pattern2_0, cond: pattern3_0.clone(), @@ -3672,7 +3672,7 @@ pub fn constructor_emit_cmov_imm( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 2070. + // Rule at src/isa/s390x/inst.isle line 2065. let expr0_0 = MInst::CMov64SImm16 { rd: pattern2_0, cond: pattern3_0.clone(), @@ -3700,7 +3700,7 @@ pub fn constructor_cmov_imm( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 2076. + // Rule at src/isa/s390x/inst.isle line 2071. let expr0_0 = constructor_copy_writable_reg(ctx, pattern0_0, pattern3_0)?; let expr1_0 = constructor_emit_cmov_imm(ctx, pattern0_0, expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -3720,7 +3720,7 @@ pub fn constructor_cmov_imm_regpair_lo( let pattern2_0 = arg2; let pattern3_0 = arg3; let pattern4_0 = arg4; - // Rule at src/isa/s390x/inst.isle line 2083. + // Rule at src/isa/s390x/inst.isle line 2078. let expr0_0 = constructor_copy_writable_regpair(ctx, pattern4_0)?; let expr1_0 = constructor_writable_regpair_lo(ctx, &expr0_0)?; let expr2_0 = constructor_emit_cmov_imm(ctx, pattern0_0, expr1_0, pattern2_0, pattern3_0)?; @@ -3743,7 +3743,7 @@ pub fn constructor_cmov_imm_regpair_hi( let pattern2_0 = arg2; let pattern3_0 = arg3; let pattern4_0 = arg4; - // Rule at src/isa/s390x/inst.isle line 2092. + // Rule at src/isa/s390x/inst.isle line 2087. let expr0_0 = constructor_copy_writable_regpair(ctx, pattern4_0)?; let expr1_0 = constructor_writable_regpair_hi(ctx, &expr0_0)?; let expr2_0 = constructor_emit_cmov_imm(ctx, pattern0_0, expr1_0, pattern2_0, pattern3_0)?; @@ -3765,7 +3765,7 @@ pub fn constructor_emit_cmov_reg( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 2106. + // Rule at src/isa/s390x/inst.isle line 2101. let expr0_0 = MInst::FpuCMov32 { rd: pattern2_0, cond: pattern3_0.clone(), @@ -3782,7 +3782,7 @@ pub fn constructor_emit_cmov_reg( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 2109. + // Rule at src/isa/s390x/inst.isle line 2104. let expr0_0 = MInst::FpuCMov64 { rd: pattern2_0, cond: pattern3_0.clone(), @@ -3799,7 +3799,7 @@ pub fn constructor_emit_cmov_reg( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 2100. + // Rule at src/isa/s390x/inst.isle line 2095. let expr0_0 = MInst::CMov32 { rd: pattern2_0, cond: pattern3_0.clone(), @@ -3816,7 +3816,7 @@ pub fn constructor_emit_cmov_reg( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 2103. + // Rule at src/isa/s390x/inst.isle line 2098. let expr0_0 = MInst::CMov64 { rd: pattern2_0, cond: pattern3_0.clone(), @@ -3844,7 +3844,7 @@ pub fn constructor_cmov_reg( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 2115. + // Rule at src/isa/s390x/inst.isle line 2110. let expr0_0 = constructor_copy_writable_reg(ctx, pattern0_0, pattern3_0)?; let expr1_0 = constructor_emit_cmov_reg(ctx, pattern0_0, expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -3865,7 +3865,7 @@ pub fn constructor_trap_if( { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2123. + // Rule at src/isa/s390x/inst.isle line 2118. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = MInst::TrapIf { cond: pattern2_0.clone(), @@ -3891,7 +3891,7 @@ pub fn constructor_icmps_reg_and_trap( let pattern2_0 = arg2; let pattern3_0 = arg3; let pattern4_0 = arg4; - // Rule at src/isa/s390x/inst.isle line 2129. + // Rule at src/isa/s390x/inst.isle line 2124. let expr0_0 = constructor_cmpop_cmps(ctx, pattern0_0)?; let expr1_0 = MInst::CmpTrapRR { op: expr0_0, @@ -3919,7 +3919,7 @@ pub fn constructor_icmps_simm16_and_trap( let pattern2_0 = arg2; let pattern3_0 = arg3; let pattern4_0 = arg4; - // Rule at src/isa/s390x/inst.isle line 2135. + // Rule at src/isa/s390x/inst.isle line 2130. let expr0_0 = constructor_cmpop_cmps(ctx, pattern0_0)?; let expr1_0 = MInst::CmpTrapRSImm16 { op: expr0_0, @@ -3947,7 +3947,7 @@ pub fn constructor_icmpu_reg_and_trap( let pattern2_0 = arg2; let pattern3_0 = arg3; let pattern4_0 = arg4; - // Rule at src/isa/s390x/inst.isle line 2141. + // Rule at src/isa/s390x/inst.isle line 2136. let expr0_0 = constructor_cmpop_cmpu(ctx, pattern0_0)?; let expr1_0 = MInst::CmpTrapRR { op: expr0_0, @@ -3975,7 +3975,7 @@ pub fn constructor_icmpu_uimm16_and_trap( let pattern2_0 = arg2; let pattern3_0 = arg3; let pattern4_0 = arg4; - // Rule at src/isa/s390x/inst.isle line 2147. + // Rule at src/isa/s390x/inst.isle line 2142. let expr0_0 = constructor_cmpop_cmpu(ctx, pattern0_0)?; let expr1_0 = MInst::CmpTrapRUImm16 { op: expr0_0, @@ -3997,7 +3997,7 @@ pub fn constructor_bool( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2160. + // Rule at src/isa/s390x/inst.isle line 2155. let expr0_0 = ProducesBool::ProducesBool { producer: pattern0_0.clone(), cond: pattern1_0.clone(), @@ -4016,7 +4016,7 @@ pub fn constructor_invert_bool( cond: ref pattern1_1, } = pattern0_0 { - // Rule at src/isa/s390x/inst.isle line 2164. + // Rule at src/isa/s390x/inst.isle line 2159. let expr0_0 = C::invert_cond(ctx, &pattern1_1); let expr1_0 = constructor_bool(ctx, &pattern1_0, &expr0_0)?; return Some(expr1_0); @@ -4032,7 +4032,7 @@ pub fn constructor_emit_producer(ctx: &mut C, arg0: &ProducesFlags) result: pattern1_1, } = pattern0_0 { - // Rule at src/isa/s390x/inst.isle line 2173. + // Rule at src/isa/s390x/inst.isle line 2168. let expr0_0 = C::emit(ctx, &pattern1_0); return Some(expr0_0); } @@ -4047,7 +4047,7 @@ pub fn constructor_emit_consumer(ctx: &mut C, arg0: &ConsumesFlags) result: pattern1_1, } = pattern0_0 { - // Rule at src/isa/s390x/inst.isle line 2175. + // Rule at src/isa/s390x/inst.isle line 2170. let expr0_0 = C::emit(ctx, &pattern1_0); return Some(expr0_0); } @@ -4071,7 +4071,7 @@ pub fn constructor_select_bool_reg( { let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 2179. + // Rule at src/isa/s390x/inst.isle line 2174. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = constructor_emit_producer(ctx, &pattern2_0)?; let expr2_0 = constructor_emit_mov(ctx, pattern0_0, expr0_0, pattern4_0)?; @@ -4100,7 +4100,7 @@ pub fn constructor_select_bool_imm( { let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 2188. + // Rule at src/isa/s390x/inst.isle line 2183. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = constructor_emit_producer(ctx, &pattern2_0)?; let expr2_0 = constructor_emit_imm(ctx, pattern0_0, expr0_0, pattern4_0)?; @@ -4121,7 +4121,7 @@ pub fn constructor_lower_bool( let pattern0_0 = arg0; if pattern0_0 == B1 { let pattern2_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2198. + // Rule at src/isa/s390x/inst.isle line 2193. let expr0_0: Type = B1; let expr1_0: i16 = 1; let expr2_0: u64 = 0; @@ -4130,7 +4130,7 @@ pub fn constructor_lower_bool( } if pattern0_0 == B8 { let pattern2_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2199. + // Rule at src/isa/s390x/inst.isle line 2194. let expr0_0: Type = B8; let expr1_0: i16 = -1; let expr2_0: u64 = 0; @@ -4139,7 +4139,7 @@ pub fn constructor_lower_bool( } if pattern0_0 == B16 { let pattern2_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2200. + // Rule at src/isa/s390x/inst.isle line 2195. let expr0_0: Type = B16; let expr1_0: i16 = -1; let expr2_0: u64 = 0; @@ -4148,7 +4148,7 @@ pub fn constructor_lower_bool( } if pattern0_0 == B32 { let pattern2_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2201. + // Rule at src/isa/s390x/inst.isle line 2196. let expr0_0: Type = B32; let expr1_0: i16 = -1; let expr2_0: u64 = 0; @@ -4157,7 +4157,7 @@ pub fn constructor_lower_bool( } if pattern0_0 == B64 { let pattern2_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2202. + // Rule at src/isa/s390x/inst.isle line 2197. let expr0_0: Type = B64; let expr1_0: i16 = -1; let expr2_0: u64 = 0; @@ -4172,7 +4172,7 @@ pub fn constructor_clz_reg(ctx: &mut C, arg0: i16, arg1: Reg) -> Opt let pattern0_0 = arg0; if pattern0_0 == 64 { let pattern2_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2211. + // Rule at src/isa/s390x/inst.isle line 2206. let expr0_0 = constructor_temp_writable_regpair(ctx)?; let expr1_0 = MInst::Flogr { rn: pattern2_0 }; let expr2_0 = C::emit(ctx, &expr1_0); @@ -4180,7 +4180,7 @@ pub fn constructor_clz_reg(ctx: &mut C, arg0: i16, arg1: Reg) -> Opt return Some(expr3_0); } let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2220. + // Rule at src/isa/s390x/inst.isle line 2215. let expr0_0 = constructor_temp_writable_regpair(ctx)?; let expr1_0 = MInst::Flogr { rn: pattern1_0 }; let expr2_0 = C::emit(ctx, &expr1_0); @@ -4201,22 +4201,22 @@ pub fn constructor_clz_reg(ctx: &mut C, arg0: i16, arg1: Reg) -> Opt pub fn constructor_aluop_add(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I8 { - // Rule at src/isa/s390x/inst.isle line 2231. + // Rule at src/isa/s390x/inst.isle line 2226. let expr0_0 = ALUOp::Add32; return Some(expr0_0); } if pattern0_0 == I16 { - // Rule at src/isa/s390x/inst.isle line 2232. + // Rule at src/isa/s390x/inst.isle line 2227. let expr0_0 = ALUOp::Add32; return Some(expr0_0); } if pattern0_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 2233. + // Rule at src/isa/s390x/inst.isle line 2228. let expr0_0 = ALUOp::Add32; return Some(expr0_0); } if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2234. + // Rule at src/isa/s390x/inst.isle line 2229. let expr0_0 = ALUOp::Add64; return Some(expr0_0); } @@ -4227,17 +4227,17 @@ pub fn constructor_aluop_add(ctx: &mut C, arg0: Type) -> Option(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I16 { - // Rule at src/isa/s390x/inst.isle line 2237. + // Rule at src/isa/s390x/inst.isle line 2232. let expr0_0 = ALUOp::Add32Ext16; return Some(expr0_0); } if pattern0_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 2238. + // Rule at src/isa/s390x/inst.isle line 2233. let expr0_0 = ALUOp::Add32Ext16; return Some(expr0_0); } if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2239. + // Rule at src/isa/s390x/inst.isle line 2234. let expr0_0 = ALUOp::Add64Ext16; return Some(expr0_0); } @@ -4248,7 +4248,7 @@ pub fn constructor_aluop_add_sext16(ctx: &mut C, arg0: Type) -> Opti pub fn constructor_aluop_add_sext32(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2242. + // Rule at src/isa/s390x/inst.isle line 2237. let expr0_0 = ALUOp::Add64Ext32; return Some(expr0_0); } @@ -4265,7 +4265,7 @@ pub fn constructor_add_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2245. + // Rule at src/isa/s390x/inst.isle line 2240. let expr0_0 = constructor_aluop_add(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rrr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4281,7 +4281,7 @@ pub fn constructor_add_reg_sext32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2248. + // Rule at src/isa/s390x/inst.isle line 2243. let expr0_0 = constructor_aluop_add_sext32(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4297,7 +4297,7 @@ pub fn constructor_add_simm16( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2251. + // Rule at src/isa/s390x/inst.isle line 2246. let expr0_0 = constructor_aluop_add(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rrsimm16(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4313,7 +4313,7 @@ pub fn constructor_add_simm32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2254. + // Rule at src/isa/s390x/inst.isle line 2249. let expr0_0 = constructor_aluop_add(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rsimm32(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4329,7 +4329,7 @@ pub fn constructor_add_mem( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2257. + // Rule at src/isa/s390x/inst.isle line 2252. let expr0_0 = constructor_aluop_add(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rx(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4345,7 +4345,7 @@ pub fn constructor_add_mem_sext16( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2260. + // Rule at src/isa/s390x/inst.isle line 2255. let expr0_0 = constructor_aluop_add_sext16(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rx(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4361,7 +4361,7 @@ pub fn constructor_add_mem_sext32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2263. + // Rule at src/isa/s390x/inst.isle line 2258. let expr0_0 = constructor_aluop_add_sext32(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rx(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4371,12 +4371,12 @@ pub fn constructor_add_mem_sext32( pub fn constructor_aluop_add_logical(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 2269. + // Rule at src/isa/s390x/inst.isle line 2264. let expr0_0 = ALUOp::AddLogical32; return Some(expr0_0); } if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2270. + // Rule at src/isa/s390x/inst.isle line 2265. let expr0_0 = ALUOp::AddLogical64; return Some(expr0_0); } @@ -4387,7 +4387,7 @@ pub fn constructor_aluop_add_logical(ctx: &mut C, arg0: Type) -> Opt pub fn constructor_aluop_add_logical_zext32(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2273. + // Rule at src/isa/s390x/inst.isle line 2268. let expr0_0 = ALUOp::AddLogical64Ext32; return Some(expr0_0); } @@ -4404,7 +4404,7 @@ pub fn constructor_add_logical_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2276. + // Rule at src/isa/s390x/inst.isle line 2271. let expr0_0 = constructor_aluop_add_logical(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rrr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4420,7 +4420,7 @@ pub fn constructor_add_logical_reg_zext32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2279. + // Rule at src/isa/s390x/inst.isle line 2274. let expr0_0 = constructor_aluop_add_logical_zext32(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4436,7 +4436,7 @@ pub fn constructor_add_logical_zimm32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2282. + // Rule at src/isa/s390x/inst.isle line 2277. let expr0_0 = constructor_aluop_add_logical(ctx, pattern0_0)?; let expr1_0 = constructor_alu_ruimm32(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4452,7 +4452,7 @@ pub fn constructor_add_logical_mem( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2285. + // Rule at src/isa/s390x/inst.isle line 2280. let expr0_0 = constructor_aluop_add_logical(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rx(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4468,7 +4468,7 @@ pub fn constructor_add_logical_mem_zext32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2288. + // Rule at src/isa/s390x/inst.isle line 2283. let expr0_0 = constructor_aluop_add_logical_zext32(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rx(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4478,22 +4478,22 @@ pub fn constructor_add_logical_mem_zext32( pub fn constructor_aluop_sub(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I8 { - // Rule at src/isa/s390x/inst.isle line 2294. + // Rule at src/isa/s390x/inst.isle line 2289. let expr0_0 = ALUOp::Sub32; return Some(expr0_0); } if pattern0_0 == I16 { - // Rule at src/isa/s390x/inst.isle line 2295. + // Rule at src/isa/s390x/inst.isle line 2290. let expr0_0 = ALUOp::Sub32; return Some(expr0_0); } if pattern0_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 2296. + // Rule at src/isa/s390x/inst.isle line 2291. let expr0_0 = ALUOp::Sub32; return Some(expr0_0); } if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2297. + // Rule at src/isa/s390x/inst.isle line 2292. let expr0_0 = ALUOp::Sub64; return Some(expr0_0); } @@ -4504,17 +4504,17 @@ pub fn constructor_aluop_sub(ctx: &mut C, arg0: Type) -> Option(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I16 { - // Rule at src/isa/s390x/inst.isle line 2300. + // Rule at src/isa/s390x/inst.isle line 2295. let expr0_0 = ALUOp::Sub32Ext16; return Some(expr0_0); } if pattern0_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 2301. + // Rule at src/isa/s390x/inst.isle line 2296. let expr0_0 = ALUOp::Sub32Ext16; return Some(expr0_0); } if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2302. + // Rule at src/isa/s390x/inst.isle line 2297. let expr0_0 = ALUOp::Sub64Ext16; return Some(expr0_0); } @@ -4525,7 +4525,7 @@ pub fn constructor_aluop_sub_sext16(ctx: &mut C, arg0: Type) -> Opti pub fn constructor_aluop_sub_sext32(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2305. + // Rule at src/isa/s390x/inst.isle line 2300. let expr0_0 = ALUOp::Sub64Ext32; return Some(expr0_0); } @@ -4542,7 +4542,7 @@ pub fn constructor_sub_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2308. + // Rule at src/isa/s390x/inst.isle line 2303. let expr0_0 = constructor_aluop_sub(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rrr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4558,7 +4558,7 @@ pub fn constructor_sub_reg_sext32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2311. + // Rule at src/isa/s390x/inst.isle line 2306. let expr0_0 = constructor_aluop_sub_sext32(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4574,7 +4574,7 @@ pub fn constructor_sub_mem( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2314. + // Rule at src/isa/s390x/inst.isle line 2309. let expr0_0 = constructor_aluop_sub(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rx(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4590,7 +4590,7 @@ pub fn constructor_sub_mem_sext16( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2317. + // Rule at src/isa/s390x/inst.isle line 2312. let expr0_0 = constructor_aluop_sub_sext16(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rx(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4606,7 +4606,7 @@ pub fn constructor_sub_mem_sext32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2320. + // Rule at src/isa/s390x/inst.isle line 2315. let expr0_0 = constructor_aluop_sub_sext32(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rx(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4616,12 +4616,12 @@ pub fn constructor_sub_mem_sext32( pub fn constructor_aluop_sub_logical(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 2326. + // Rule at src/isa/s390x/inst.isle line 2321. let expr0_0 = ALUOp::SubLogical32; return Some(expr0_0); } if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2327. + // Rule at src/isa/s390x/inst.isle line 2322. let expr0_0 = ALUOp::SubLogical64; return Some(expr0_0); } @@ -4632,7 +4632,7 @@ pub fn constructor_aluop_sub_logical(ctx: &mut C, arg0: Type) -> Opt pub fn constructor_aluop_sub_logical_zext32(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2330. + // Rule at src/isa/s390x/inst.isle line 2325. let expr0_0 = ALUOp::SubLogical64Ext32; return Some(expr0_0); } @@ -4649,7 +4649,7 @@ pub fn constructor_sub_logical_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2333. + // Rule at src/isa/s390x/inst.isle line 2328. let expr0_0 = constructor_aluop_sub_logical(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rrr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4665,7 +4665,7 @@ pub fn constructor_sub_logical_reg_zext32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2336. + // Rule at src/isa/s390x/inst.isle line 2331. let expr0_0 = constructor_aluop_sub_logical_zext32(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4681,7 +4681,7 @@ pub fn constructor_sub_logical_zimm32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2339. + // Rule at src/isa/s390x/inst.isle line 2334. let expr0_0 = constructor_aluop_sub_logical(ctx, pattern0_0)?; let expr1_0 = constructor_alu_ruimm32(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4697,7 +4697,7 @@ pub fn constructor_sub_logical_mem( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2342. + // Rule at src/isa/s390x/inst.isle line 2337. let expr0_0 = constructor_aluop_sub_logical(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rx(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4713,7 +4713,7 @@ pub fn constructor_sub_logical_mem_zext32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2345. + // Rule at src/isa/s390x/inst.isle line 2340. let expr0_0 = constructor_aluop_sub_logical(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rx(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4723,22 +4723,22 @@ pub fn constructor_sub_logical_mem_zext32( pub fn constructor_aluop_mul(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I8 { - // Rule at src/isa/s390x/inst.isle line 2351. + // Rule at src/isa/s390x/inst.isle line 2346. let expr0_0 = ALUOp::Mul32; return Some(expr0_0); } if pattern0_0 == I16 { - // Rule at src/isa/s390x/inst.isle line 2352. + // Rule at src/isa/s390x/inst.isle line 2347. let expr0_0 = ALUOp::Mul32; return Some(expr0_0); } if pattern0_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 2353. + // Rule at src/isa/s390x/inst.isle line 2348. let expr0_0 = ALUOp::Mul32; return Some(expr0_0); } if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2354. + // Rule at src/isa/s390x/inst.isle line 2349. let expr0_0 = ALUOp::Mul64; return Some(expr0_0); } @@ -4749,17 +4749,17 @@ pub fn constructor_aluop_mul(ctx: &mut C, arg0: Type) -> Option(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I16 { - // Rule at src/isa/s390x/inst.isle line 2357. + // Rule at src/isa/s390x/inst.isle line 2352. let expr0_0 = ALUOp::Mul32Ext16; return Some(expr0_0); } if pattern0_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 2358. + // Rule at src/isa/s390x/inst.isle line 2353. let expr0_0 = ALUOp::Mul32Ext16; return Some(expr0_0); } if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2359. + // Rule at src/isa/s390x/inst.isle line 2354. let expr0_0 = ALUOp::Mul64Ext16; return Some(expr0_0); } @@ -4770,7 +4770,7 @@ pub fn constructor_aluop_mul_sext16(ctx: &mut C, arg0: Type) -> Opti pub fn constructor_aluop_mul_sext32(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2362. + // Rule at src/isa/s390x/inst.isle line 2357. let expr0_0 = ALUOp::Mul64Ext32; return Some(expr0_0); } @@ -4787,7 +4787,7 @@ pub fn constructor_mul_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2365. + // Rule at src/isa/s390x/inst.isle line 2360. let expr0_0 = constructor_aluop_mul(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rrr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4803,7 +4803,7 @@ pub fn constructor_mul_reg_sext32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2368. + // Rule at src/isa/s390x/inst.isle line 2363. let expr0_0 = constructor_aluop_mul_sext32(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4819,7 +4819,7 @@ pub fn constructor_mul_simm16( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2371. + // Rule at src/isa/s390x/inst.isle line 2366. let expr0_0 = constructor_aluop_mul(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rsimm16(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4835,7 +4835,7 @@ pub fn constructor_mul_simm32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2374. + // Rule at src/isa/s390x/inst.isle line 2369. let expr0_0 = constructor_aluop_mul(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rsimm32(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4851,7 +4851,7 @@ pub fn constructor_mul_mem( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2377. + // Rule at src/isa/s390x/inst.isle line 2372. let expr0_0 = constructor_aluop_mul(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rx(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4867,7 +4867,7 @@ pub fn constructor_mul_mem_sext16( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2380. + // Rule at src/isa/s390x/inst.isle line 2375. let expr0_0 = constructor_aluop_mul_sext16(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rx(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4883,7 +4883,7 @@ pub fn constructor_mul_mem_sext32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2383. + // Rule at src/isa/s390x/inst.isle line 2378. let expr0_0 = constructor_aluop_mul_sext32(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rx(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4900,14 +4900,14 @@ pub fn constructor_udivmod( if pattern0_0 == I32 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2389. + // Rule at src/isa/s390x/inst.isle line 2384. let expr0_0 = constructor_udivmod32(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if pattern0_0 == I64 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2390. + // Rule at src/isa/s390x/inst.isle line 2385. let expr0_0 = constructor_udivmod64(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } @@ -4925,14 +4925,14 @@ pub fn constructor_sdivmod( if pattern0_0 == I32 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2396. + // Rule at src/isa/s390x/inst.isle line 2391. let expr0_0 = constructor_sdivmod32(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if pattern0_0 == I64 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2397. + // Rule at src/isa/s390x/inst.isle line 2392. let expr0_0 = constructor_sdivmod64(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } @@ -4943,12 +4943,12 @@ pub fn constructor_sdivmod( pub fn constructor_aluop_and(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if let Some(pattern1_0) = C::gpr32_ty(ctx, pattern0_0) { - // Rule at src/isa/s390x/inst.isle line 2403. + // Rule at src/isa/s390x/inst.isle line 2398. let expr0_0 = ALUOp::And32; return Some(expr0_0); } if let Some(pattern1_0) = C::gpr64_ty(ctx, pattern0_0) { - // Rule at src/isa/s390x/inst.isle line 2404. + // Rule at src/isa/s390x/inst.isle line 2399. let expr0_0 = ALUOp::And64; return Some(expr0_0); } @@ -4965,7 +4965,7 @@ pub fn constructor_and_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2407. + // Rule at src/isa/s390x/inst.isle line 2402. let expr0_0 = constructor_aluop_and(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rrr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4981,7 +4981,7 @@ pub fn constructor_and_uimm16shifted( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2410. + // Rule at src/isa/s390x/inst.isle line 2405. let expr0_0 = constructor_aluop_and(ctx, pattern0_0)?; let expr1_0 = constructor_alu_ruimm16shifted(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; @@ -4998,7 +4998,7 @@ pub fn constructor_and_uimm32shifted( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2413. + // Rule at src/isa/s390x/inst.isle line 2408. let expr0_0 = constructor_aluop_and(ctx, pattern0_0)?; let expr1_0 = constructor_alu_ruimm32shifted(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; @@ -5015,7 +5015,7 @@ pub fn constructor_and_mem( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2416. + // Rule at src/isa/s390x/inst.isle line 2411. let expr0_0 = constructor_aluop_and(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rx(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -5025,12 +5025,12 @@ pub fn constructor_and_mem( pub fn constructor_aluop_or(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if let Some(pattern1_0) = C::gpr32_ty(ctx, pattern0_0) { - // Rule at src/isa/s390x/inst.isle line 2422. + // Rule at src/isa/s390x/inst.isle line 2417. let expr0_0 = ALUOp::Orr32; return Some(expr0_0); } if let Some(pattern1_0) = C::gpr64_ty(ctx, pattern0_0) { - // Rule at src/isa/s390x/inst.isle line 2423. + // Rule at src/isa/s390x/inst.isle line 2418. let expr0_0 = ALUOp::Orr64; return Some(expr0_0); } @@ -5047,7 +5047,7 @@ pub fn constructor_or_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2426. + // Rule at src/isa/s390x/inst.isle line 2421. let expr0_0 = constructor_aluop_or(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rrr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -5063,7 +5063,7 @@ pub fn constructor_or_uimm16shifted( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2429. + // Rule at src/isa/s390x/inst.isle line 2424. let expr0_0 = constructor_aluop_or(ctx, pattern0_0)?; let expr1_0 = constructor_alu_ruimm16shifted(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; @@ -5080,7 +5080,7 @@ pub fn constructor_or_uimm32shifted( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2432. + // Rule at src/isa/s390x/inst.isle line 2427. let expr0_0 = constructor_aluop_or(ctx, pattern0_0)?; let expr1_0 = constructor_alu_ruimm32shifted(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; @@ -5097,7 +5097,7 @@ pub fn constructor_or_mem( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2435. + // Rule at src/isa/s390x/inst.isle line 2430. let expr0_0 = constructor_aluop_or(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rx(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -5107,12 +5107,12 @@ pub fn constructor_or_mem( pub fn constructor_aluop_xor(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if let Some(pattern1_0) = C::gpr32_ty(ctx, pattern0_0) { - // Rule at src/isa/s390x/inst.isle line 2441. + // Rule at src/isa/s390x/inst.isle line 2436. let expr0_0 = ALUOp::Xor32; return Some(expr0_0); } if let Some(pattern1_0) = C::gpr64_ty(ctx, pattern0_0) { - // Rule at src/isa/s390x/inst.isle line 2442. + // Rule at src/isa/s390x/inst.isle line 2437. let expr0_0 = ALUOp::Xor64; return Some(expr0_0); } @@ -5129,7 +5129,7 @@ pub fn constructor_xor_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2445. + // Rule at src/isa/s390x/inst.isle line 2440. let expr0_0 = constructor_aluop_xor(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rrr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -5145,7 +5145,7 @@ pub fn constructor_xor_uimm32shifted( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2448. + // Rule at src/isa/s390x/inst.isle line 2443. let expr0_0 = constructor_aluop_xor(ctx, pattern0_0)?; let expr1_0 = constructor_alu_ruimm32shifted(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; @@ -5162,7 +5162,7 @@ pub fn constructor_xor_mem( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2451. + // Rule at src/isa/s390x/inst.isle line 2446. let expr0_0 = constructor_aluop_xor(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rx(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -5173,7 +5173,7 @@ pub fn constructor_not_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> Op let pattern0_0 = arg0; if let Some(pattern1_0) = C::gpr32_ty(ctx, pattern0_0) { let pattern2_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2457. + // Rule at src/isa/s390x/inst.isle line 2452. let expr0_0: u32 = 4294967295; let expr1_0: u8 = 0; let expr2_0 = C::uimm32shifted(ctx, expr0_0, expr1_0); @@ -5182,7 +5182,7 @@ pub fn constructor_not_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> Op } if let Some(pattern1_0) = C::gpr64_ty(ctx, pattern0_0) { let pattern2_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2459. + // Rule at src/isa/s390x/inst.isle line 2454. let expr0_0: u32 = 4294967295; let expr1_0: u8 = 0; let expr2_0 = C::uimm32shifted(ctx, expr0_0, expr1_0); @@ -5200,12 +5200,12 @@ pub fn constructor_not_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> Op pub fn constructor_aluop_and_not(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if let Some(pattern1_0) = C::gpr32_ty(ctx, pattern0_0) { - // Rule at src/isa/s390x/inst.isle line 2468. + // Rule at src/isa/s390x/inst.isle line 2463. let expr0_0 = ALUOp::AndNot32; return Some(expr0_0); } if let Some(pattern1_0) = C::gpr64_ty(ctx, pattern0_0) { - // Rule at src/isa/s390x/inst.isle line 2469. + // Rule at src/isa/s390x/inst.isle line 2464. let expr0_0 = ALUOp::AndNot64; return Some(expr0_0); } @@ -5222,7 +5222,7 @@ pub fn constructor_and_not_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2472. + // Rule at src/isa/s390x/inst.isle line 2467. let expr0_0 = constructor_aluop_and_not(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rrr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -5232,12 +5232,12 @@ pub fn constructor_and_not_reg( pub fn constructor_aluop_or_not(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if let Some(pattern1_0) = C::gpr32_ty(ctx, pattern0_0) { - // Rule at src/isa/s390x/inst.isle line 2478. + // Rule at src/isa/s390x/inst.isle line 2473. let expr0_0 = ALUOp::OrrNot32; return Some(expr0_0); } if let Some(pattern1_0) = C::gpr64_ty(ctx, pattern0_0) { - // Rule at src/isa/s390x/inst.isle line 2479. + // Rule at src/isa/s390x/inst.isle line 2474. let expr0_0 = ALUOp::OrrNot64; return Some(expr0_0); } @@ -5254,7 +5254,7 @@ pub fn constructor_or_not_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2482. + // Rule at src/isa/s390x/inst.isle line 2477. let expr0_0 = constructor_aluop_or_not(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rrr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -5264,12 +5264,12 @@ pub fn constructor_or_not_reg( pub fn constructor_aluop_xor_not(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if let Some(pattern1_0) = C::gpr32_ty(ctx, pattern0_0) { - // Rule at src/isa/s390x/inst.isle line 2488. + // Rule at src/isa/s390x/inst.isle line 2483. let expr0_0 = ALUOp::XorNot32; return Some(expr0_0); } if let Some(pattern1_0) = C::gpr64_ty(ctx, pattern0_0) { - // Rule at src/isa/s390x/inst.isle line 2489. + // Rule at src/isa/s390x/inst.isle line 2484. let expr0_0 = ALUOp::XorNot64; return Some(expr0_0); } @@ -5286,7 +5286,7 @@ pub fn constructor_xor_not_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2492. + // Rule at src/isa/s390x/inst.isle line 2487. let expr0_0 = constructor_aluop_xor_not(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rrr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -5296,12 +5296,12 @@ pub fn constructor_xor_not_reg( pub fn constructor_unaryop_abs(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 2498. + // Rule at src/isa/s390x/inst.isle line 2493. let expr0_0 = UnaryOp::Abs32; return Some(expr0_0); } if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2499. + // Rule at src/isa/s390x/inst.isle line 2494. let expr0_0 = UnaryOp::Abs64; return Some(expr0_0); } @@ -5312,7 +5312,7 @@ pub fn constructor_unaryop_abs(ctx: &mut C, arg0: Type) -> Option(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2502. + // Rule at src/isa/s390x/inst.isle line 2497. let expr0_0 = UnaryOp::Abs64Ext32; return Some(expr0_0); } @@ -5323,7 +5323,7 @@ pub fn constructor_unaryop_abs_sext32(ctx: &mut C, arg0: Type) -> Op pub fn constructor_abs_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2505. + // Rule at src/isa/s390x/inst.isle line 2500. let expr0_0 = constructor_unaryop_abs(ctx, pattern0_0)?; let expr1_0 = constructor_unary_rr(ctx, pattern0_0, &expr0_0, pattern1_0)?; return Some(expr1_0); @@ -5333,7 +5333,7 @@ pub fn constructor_abs_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> Op pub fn constructor_abs_reg_sext32(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2508. + // Rule at src/isa/s390x/inst.isle line 2503. let expr0_0 = constructor_unaryop_abs_sext32(ctx, pattern0_0)?; let expr1_0 = constructor_unary_rr(ctx, pattern0_0, &expr0_0, pattern1_0)?; return Some(expr1_0); @@ -5343,22 +5343,22 @@ pub fn constructor_abs_reg_sext32(ctx: &mut C, arg0: Type, arg1: Reg pub fn constructor_unaryop_neg(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I8 { - // Rule at src/isa/s390x/inst.isle line 2514. + // Rule at src/isa/s390x/inst.isle line 2509. let expr0_0 = UnaryOp::Neg32; return Some(expr0_0); } if pattern0_0 == I16 { - // Rule at src/isa/s390x/inst.isle line 2515. + // Rule at src/isa/s390x/inst.isle line 2510. let expr0_0 = UnaryOp::Neg32; return Some(expr0_0); } if pattern0_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 2516. + // Rule at src/isa/s390x/inst.isle line 2511. let expr0_0 = UnaryOp::Neg32; return Some(expr0_0); } if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2517. + // Rule at src/isa/s390x/inst.isle line 2512. let expr0_0 = UnaryOp::Neg64; return Some(expr0_0); } @@ -5369,7 +5369,7 @@ pub fn constructor_unaryop_neg(ctx: &mut C, arg0: Type) -> Option(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2520. + // Rule at src/isa/s390x/inst.isle line 2515. let expr0_0 = UnaryOp::Neg64Ext32; return Some(expr0_0); } @@ -5380,7 +5380,7 @@ pub fn constructor_unaryop_neg_sext32(ctx: &mut C, arg0: Type) -> Op pub fn constructor_neg_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2523. + // Rule at src/isa/s390x/inst.isle line 2518. let expr0_0 = constructor_unaryop_neg(ctx, pattern0_0)?; let expr1_0 = constructor_unary_rr(ctx, pattern0_0, &expr0_0, pattern1_0)?; return Some(expr1_0); @@ -5390,7 +5390,7 @@ pub fn constructor_neg_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> Op pub fn constructor_neg_reg_sext32(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2526. + // Rule at src/isa/s390x/inst.isle line 2521. let expr0_0 = constructor_unaryop_neg_sext32(ctx, pattern0_0)?; let expr1_0 = constructor_unary_rr(ctx, pattern0_0, &expr0_0, pattern1_0)?; return Some(expr1_0); @@ -5400,12 +5400,12 @@ pub fn constructor_neg_reg_sext32(ctx: &mut C, arg0: Type, arg1: Reg pub fn constructor_shiftop_rot(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 2532. + // Rule at src/isa/s390x/inst.isle line 2527. let expr0_0 = ShiftOp::RotL32; return Some(expr0_0); } if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2533. + // Rule at src/isa/s390x/inst.isle line 2528. let expr0_0 = ShiftOp::RotL64; return Some(expr0_0); } @@ -5422,7 +5422,7 @@ pub fn constructor_rot_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2536. + // Rule at src/isa/s390x/inst.isle line 2531. let expr0_0 = constructor_shiftop_rot(ctx, pattern0_0)?; let expr1_0: u8 = 0; let expr2_0 = constructor_shift_rr(ctx, pattern0_0, &expr0_0, pattern1_0, expr1_0, pattern2_0)?; @@ -5439,7 +5439,7 @@ pub fn constructor_rot_imm( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2540. + // Rule at src/isa/s390x/inst.isle line 2535. let expr0_0 = constructor_shiftop_rot(ctx, pattern0_0)?; let expr1_0 = C::zero_reg(ctx); let expr2_0 = constructor_shift_rr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0, expr1_0)?; @@ -5450,22 +5450,22 @@ pub fn constructor_rot_imm( pub fn constructor_shiftop_lshl(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I8 { - // Rule at src/isa/s390x/inst.isle line 2547. + // Rule at src/isa/s390x/inst.isle line 2542. let expr0_0 = ShiftOp::LShL32; return Some(expr0_0); } if pattern0_0 == I16 { - // Rule at src/isa/s390x/inst.isle line 2548. + // Rule at src/isa/s390x/inst.isle line 2543. let expr0_0 = ShiftOp::LShL32; return Some(expr0_0); } if pattern0_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 2549. + // Rule at src/isa/s390x/inst.isle line 2544. let expr0_0 = ShiftOp::LShL32; return Some(expr0_0); } if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2550. + // Rule at src/isa/s390x/inst.isle line 2545. let expr0_0 = ShiftOp::LShL64; return Some(expr0_0); } @@ -5482,7 +5482,7 @@ pub fn constructor_lshl_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2553. + // Rule at src/isa/s390x/inst.isle line 2548. let expr0_0 = constructor_shiftop_lshl(ctx, pattern0_0)?; let expr1_0: u8 = 0; let expr2_0 = constructor_shift_rr(ctx, pattern0_0, &expr0_0, pattern1_0, expr1_0, pattern2_0)?; @@ -5499,7 +5499,7 @@ pub fn constructor_lshl_imm( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2557. + // Rule at src/isa/s390x/inst.isle line 2552. let expr0_0 = constructor_shiftop_lshl(ctx, pattern0_0)?; let expr1_0 = C::zero_reg(ctx); let expr2_0 = constructor_shift_rr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0, expr1_0)?; @@ -5510,12 +5510,12 @@ pub fn constructor_lshl_imm( pub fn constructor_shiftop_lshr(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 2564. + // Rule at src/isa/s390x/inst.isle line 2559. let expr0_0 = ShiftOp::LShR32; return Some(expr0_0); } if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2565. + // Rule at src/isa/s390x/inst.isle line 2560. let expr0_0 = ShiftOp::LShR64; return Some(expr0_0); } @@ -5532,7 +5532,7 @@ pub fn constructor_lshr_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2568. + // Rule at src/isa/s390x/inst.isle line 2563. let expr0_0 = constructor_shiftop_lshr(ctx, pattern0_0)?; let expr1_0: u8 = 0; let expr2_0 = constructor_shift_rr(ctx, pattern0_0, &expr0_0, pattern1_0, expr1_0, pattern2_0)?; @@ -5549,7 +5549,7 @@ pub fn constructor_lshr_imm( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2572. + // Rule at src/isa/s390x/inst.isle line 2567. let expr0_0 = constructor_shiftop_lshr(ctx, pattern0_0)?; let expr1_0 = C::zero_reg(ctx); let expr2_0 = constructor_shift_rr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0, expr1_0)?; @@ -5560,12 +5560,12 @@ pub fn constructor_lshr_imm( pub fn constructor_shiftop_ashr(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 2579. + // Rule at src/isa/s390x/inst.isle line 2574. let expr0_0 = ShiftOp::AShR32; return Some(expr0_0); } if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2580. + // Rule at src/isa/s390x/inst.isle line 2575. let expr0_0 = ShiftOp::AShR64; return Some(expr0_0); } @@ -5582,7 +5582,7 @@ pub fn constructor_ashr_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2583. + // Rule at src/isa/s390x/inst.isle line 2578. let expr0_0 = constructor_shiftop_ashr(ctx, pattern0_0)?; let expr1_0: u8 = 0; let expr2_0 = constructor_shift_rr(ctx, pattern0_0, &expr0_0, pattern1_0, expr1_0, pattern2_0)?; @@ -5599,7 +5599,7 @@ pub fn constructor_ashr_imm( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2587. + // Rule at src/isa/s390x/inst.isle line 2582. let expr0_0 = constructor_shiftop_ashr(ctx, pattern0_0)?; let expr1_0 = C::zero_reg(ctx); let expr2_0 = constructor_shift_rr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0, expr1_0)?; @@ -5609,7 +5609,7 @@ pub fn constructor_ashr_imm( // Generated as internal constructor for term popcnt_byte. pub fn constructor_popcnt_byte(ctx: &mut C, arg0: Reg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/s390x/inst.isle line 2594. + // Rule at src/isa/s390x/inst.isle line 2589. let expr0_0: Type = I64; let expr1_0 = UnaryOp::PopcntByte; let expr2_0 = constructor_unary_rr(ctx, expr0_0, &expr1_0, pattern0_0)?; @@ -5619,7 +5619,7 @@ pub fn constructor_popcnt_byte(ctx: &mut C, arg0: Reg) -> Option(ctx: &mut C, arg0: Reg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/s390x/inst.isle line 2597. + // Rule at src/isa/s390x/inst.isle line 2592. let expr0_0: Type = I64; let expr1_0 = UnaryOp::PopcntReg; let expr2_0 = constructor_unary_rr(ctx, expr0_0, &expr1_0, pattern0_0)?; @@ -5637,7 +5637,7 @@ pub fn constructor_atomic_rmw_and( if pattern0_0 == I32 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2603. + // Rule at src/isa/s390x/inst.isle line 2598. let expr0_0: Type = I32; let expr1_0 = ALUOp::And32; let expr2_0 = constructor_atomic_rmw_impl(ctx, expr0_0, &expr1_0, pattern2_0, pattern3_0)?; @@ -5646,7 +5646,7 @@ pub fn constructor_atomic_rmw_and( if pattern0_0 == I64 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2604. + // Rule at src/isa/s390x/inst.isle line 2599. let expr0_0: Type = I64; let expr1_0 = ALUOp::And64; let expr2_0 = constructor_atomic_rmw_impl(ctx, expr0_0, &expr1_0, pattern2_0, pattern3_0)?; @@ -5666,7 +5666,7 @@ pub fn constructor_atomic_rmw_or( if pattern0_0 == I32 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2607. + // Rule at src/isa/s390x/inst.isle line 2602. let expr0_0: Type = I32; let expr1_0 = ALUOp::Orr32; let expr2_0 = constructor_atomic_rmw_impl(ctx, expr0_0, &expr1_0, pattern2_0, pattern3_0)?; @@ -5675,7 +5675,7 @@ pub fn constructor_atomic_rmw_or( if pattern0_0 == I64 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2608. + // Rule at src/isa/s390x/inst.isle line 2603. let expr0_0: Type = I64; let expr1_0 = ALUOp::Orr64; let expr2_0 = constructor_atomic_rmw_impl(ctx, expr0_0, &expr1_0, pattern2_0, pattern3_0)?; @@ -5695,7 +5695,7 @@ pub fn constructor_atomic_rmw_xor( if pattern0_0 == I32 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2611. + // Rule at src/isa/s390x/inst.isle line 2606. let expr0_0: Type = I32; let expr1_0 = ALUOp::Xor32; let expr2_0 = constructor_atomic_rmw_impl(ctx, expr0_0, &expr1_0, pattern2_0, pattern3_0)?; @@ -5704,7 +5704,7 @@ pub fn constructor_atomic_rmw_xor( if pattern0_0 == I64 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2612. + // Rule at src/isa/s390x/inst.isle line 2607. let expr0_0: Type = I64; let expr1_0 = ALUOp::Xor64; let expr2_0 = constructor_atomic_rmw_impl(ctx, expr0_0, &expr1_0, pattern2_0, pattern3_0)?; @@ -5724,7 +5724,7 @@ pub fn constructor_atomic_rmw_add( if pattern0_0 == I32 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2615. + // Rule at src/isa/s390x/inst.isle line 2610. let expr0_0: Type = I32; let expr1_0 = ALUOp::Add32; let expr2_0 = constructor_atomic_rmw_impl(ctx, expr0_0, &expr1_0, pattern2_0, pattern3_0)?; @@ -5733,7 +5733,7 @@ pub fn constructor_atomic_rmw_add( if pattern0_0 == I64 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2616. + // Rule at src/isa/s390x/inst.isle line 2611. let expr0_0: Type = I64; let expr1_0 = ALUOp::Add64; let expr2_0 = constructor_atomic_rmw_impl(ctx, expr0_0, &expr1_0, pattern2_0, pattern3_0)?; @@ -5746,12 +5746,12 @@ pub fn constructor_atomic_rmw_add( pub fn constructor_fpuop2_add(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32 { - // Rule at src/isa/s390x/inst.isle line 2622. + // Rule at src/isa/s390x/inst.isle line 2617. let expr0_0 = FPUOp2::Add32; return Some(expr0_0); } if pattern0_0 == F64 { - // Rule at src/isa/s390x/inst.isle line 2623. + // Rule at src/isa/s390x/inst.isle line 2618. let expr0_0 = FPUOp2::Add64; return Some(expr0_0); } @@ -5768,7 +5768,7 @@ pub fn constructor_fadd_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2626. + // Rule at src/isa/s390x/inst.isle line 2621. let expr0_0 = constructor_fpuop2_add(ctx, pattern0_0)?; let expr1_0 = constructor_fpu_rrr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -5778,12 +5778,12 @@ pub fn constructor_fadd_reg( pub fn constructor_fpuop2_sub(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32 { - // Rule at src/isa/s390x/inst.isle line 2632. + // Rule at src/isa/s390x/inst.isle line 2627. let expr0_0 = FPUOp2::Sub32; return Some(expr0_0); } if pattern0_0 == F64 { - // Rule at src/isa/s390x/inst.isle line 2633. + // Rule at src/isa/s390x/inst.isle line 2628. let expr0_0 = FPUOp2::Sub64; return Some(expr0_0); } @@ -5800,7 +5800,7 @@ pub fn constructor_fsub_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2636. + // Rule at src/isa/s390x/inst.isle line 2631. let expr0_0 = constructor_fpuop2_sub(ctx, pattern0_0)?; let expr1_0 = constructor_fpu_rrr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -5810,12 +5810,12 @@ pub fn constructor_fsub_reg( pub fn constructor_fpuop2_mul(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32 { - // Rule at src/isa/s390x/inst.isle line 2642. + // Rule at src/isa/s390x/inst.isle line 2637. let expr0_0 = FPUOp2::Mul32; return Some(expr0_0); } if pattern0_0 == F64 { - // Rule at src/isa/s390x/inst.isle line 2643. + // Rule at src/isa/s390x/inst.isle line 2638. let expr0_0 = FPUOp2::Mul64; return Some(expr0_0); } @@ -5832,7 +5832,7 @@ pub fn constructor_fmul_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2646. + // Rule at src/isa/s390x/inst.isle line 2641. let expr0_0 = constructor_fpuop2_mul(ctx, pattern0_0)?; let expr1_0 = constructor_fpu_rrr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -5842,12 +5842,12 @@ pub fn constructor_fmul_reg( pub fn constructor_fpuop2_div(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32 { - // Rule at src/isa/s390x/inst.isle line 2652. + // Rule at src/isa/s390x/inst.isle line 2647. let expr0_0 = FPUOp2::Div32; return Some(expr0_0); } if pattern0_0 == F64 { - // Rule at src/isa/s390x/inst.isle line 2653. + // Rule at src/isa/s390x/inst.isle line 2648. let expr0_0 = FPUOp2::Div64; return Some(expr0_0); } @@ -5864,7 +5864,7 @@ pub fn constructor_fdiv_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2656. + // Rule at src/isa/s390x/inst.isle line 2651. let expr0_0 = constructor_fpuop2_div(ctx, pattern0_0)?; let expr1_0 = constructor_fpu_rrr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -5874,12 +5874,12 @@ pub fn constructor_fdiv_reg( pub fn constructor_fpuop2_min(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32 { - // Rule at src/isa/s390x/inst.isle line 2662. + // Rule at src/isa/s390x/inst.isle line 2657. let expr0_0 = FPUOp2::Min32; return Some(expr0_0); } if pattern0_0 == F64 { - // Rule at src/isa/s390x/inst.isle line 2663. + // Rule at src/isa/s390x/inst.isle line 2658. let expr0_0 = FPUOp2::Min64; return Some(expr0_0); } @@ -5896,7 +5896,7 @@ pub fn constructor_fmin_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2666. + // Rule at src/isa/s390x/inst.isle line 2661. let expr0_0 = constructor_fpuop2_min(ctx, pattern0_0)?; let expr1_0 = constructor_fpuvec_rrr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -5906,12 +5906,12 @@ pub fn constructor_fmin_reg( pub fn constructor_fpuop2_max(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32 { - // Rule at src/isa/s390x/inst.isle line 2672. + // Rule at src/isa/s390x/inst.isle line 2667. let expr0_0 = FPUOp2::Max32; return Some(expr0_0); } if pattern0_0 == F64 { - // Rule at src/isa/s390x/inst.isle line 2673. + // Rule at src/isa/s390x/inst.isle line 2668. let expr0_0 = FPUOp2::Max64; return Some(expr0_0); } @@ -5928,7 +5928,7 @@ pub fn constructor_fmax_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2676. + // Rule at src/isa/s390x/inst.isle line 2671. let expr0_0 = constructor_fpuop2_max(ctx, pattern0_0)?; let expr1_0 = constructor_fpuvec_rrr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -5938,12 +5938,12 @@ pub fn constructor_fmax_reg( pub fn constructor_fpuop3_fma(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32 { - // Rule at src/isa/s390x/inst.isle line 2682. + // Rule at src/isa/s390x/inst.isle line 2677. let expr0_0 = FPUOp3::MAdd32; return Some(expr0_0); } if pattern0_0 == F64 { - // Rule at src/isa/s390x/inst.isle line 2683. + // Rule at src/isa/s390x/inst.isle line 2678. let expr0_0 = FPUOp3::MAdd64; return Some(expr0_0); } @@ -5962,7 +5962,7 @@ pub fn constructor_fma_reg( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 2686. + // Rule at src/isa/s390x/inst.isle line 2681. let expr0_0 = constructor_fpuop3_fma(ctx, pattern0_0)?; let expr1_0 = constructor_fpu_rrrr( ctx, pattern0_0, &expr0_0, pattern3_0, pattern1_0, pattern2_0, @@ -5974,12 +5974,12 @@ pub fn constructor_fma_reg( pub fn constructor_fpuop1_sqrt(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32 { - // Rule at src/isa/s390x/inst.isle line 2692. + // Rule at src/isa/s390x/inst.isle line 2687. let expr0_0 = FPUOp1::Sqrt32; return Some(expr0_0); } if pattern0_0 == F64 { - // Rule at src/isa/s390x/inst.isle line 2693. + // Rule at src/isa/s390x/inst.isle line 2688. let expr0_0 = FPUOp1::Sqrt64; return Some(expr0_0); } @@ -5990,7 +5990,7 @@ pub fn constructor_fpuop1_sqrt(ctx: &mut C, arg0: Type) -> Option(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2696. + // Rule at src/isa/s390x/inst.isle line 2691. let expr0_0 = constructor_fpuop1_sqrt(ctx, pattern0_0)?; let expr1_0 = constructor_fpu_rr(ctx, pattern0_0, &expr0_0, pattern1_0)?; return Some(expr1_0); @@ -6000,12 +6000,12 @@ pub fn constructor_sqrt_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> O pub fn constructor_fpuop1_neg(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32 { - // Rule at src/isa/s390x/inst.isle line 2702. + // Rule at src/isa/s390x/inst.isle line 2697. let expr0_0 = FPUOp1::Neg32; return Some(expr0_0); } if pattern0_0 == F64 { - // Rule at src/isa/s390x/inst.isle line 2703. + // Rule at src/isa/s390x/inst.isle line 2698. let expr0_0 = FPUOp1::Neg64; return Some(expr0_0); } @@ -6016,7 +6016,7 @@ pub fn constructor_fpuop1_neg(ctx: &mut C, arg0: Type) -> Option(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2706. + // Rule at src/isa/s390x/inst.isle line 2701. let expr0_0 = constructor_fpuop1_neg(ctx, pattern0_0)?; let expr1_0 = constructor_fpu_rr(ctx, pattern0_0, &expr0_0, pattern1_0)?; return Some(expr1_0); @@ -6026,12 +6026,12 @@ pub fn constructor_fneg_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> O pub fn constructor_fpuop1_abs(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32 { - // Rule at src/isa/s390x/inst.isle line 2712. + // Rule at src/isa/s390x/inst.isle line 2707. let expr0_0 = FPUOp1::Abs32; return Some(expr0_0); } if pattern0_0 == F64 { - // Rule at src/isa/s390x/inst.isle line 2713. + // Rule at src/isa/s390x/inst.isle line 2708. let expr0_0 = FPUOp1::Abs64; return Some(expr0_0); } @@ -6042,7 +6042,7 @@ pub fn constructor_fpuop1_abs(ctx: &mut C, arg0: Type) -> Option(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2716. + // Rule at src/isa/s390x/inst.isle line 2711. let expr0_0 = constructor_fpuop1_abs(ctx, pattern0_0)?; let expr1_0 = constructor_fpu_rr(ctx, pattern0_0, &expr0_0, pattern1_0)?; return Some(expr1_0); @@ -6052,12 +6052,12 @@ pub fn constructor_fabs_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> O pub fn constructor_fpuroundmode_ceil(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32 { - // Rule at src/isa/s390x/inst.isle line 2722. + // Rule at src/isa/s390x/inst.isle line 2717. let expr0_0 = FpuRoundMode::Plus32; return Some(expr0_0); } if pattern0_0 == F64 { - // Rule at src/isa/s390x/inst.isle line 2723. + // Rule at src/isa/s390x/inst.isle line 2718. let expr0_0 = FpuRoundMode::Plus64; return Some(expr0_0); } @@ -6068,7 +6068,7 @@ pub fn constructor_fpuroundmode_ceil(ctx: &mut C, arg0: Type) -> Opt pub fn constructor_ceil_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2726. + // Rule at src/isa/s390x/inst.isle line 2721. let expr0_0 = constructor_fpuroundmode_ceil(ctx, pattern0_0)?; let expr1_0 = constructor_fpu_round(ctx, pattern0_0, &expr0_0, pattern1_0)?; return Some(expr1_0); @@ -6078,12 +6078,12 @@ pub fn constructor_ceil_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> O pub fn constructor_fpuroundmode_floor(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32 { - // Rule at src/isa/s390x/inst.isle line 2732. + // Rule at src/isa/s390x/inst.isle line 2727. let expr0_0 = FpuRoundMode::Minus32; return Some(expr0_0); } if pattern0_0 == F64 { - // Rule at src/isa/s390x/inst.isle line 2733. + // Rule at src/isa/s390x/inst.isle line 2728. let expr0_0 = FpuRoundMode::Minus64; return Some(expr0_0); } @@ -6094,7 +6094,7 @@ pub fn constructor_fpuroundmode_floor(ctx: &mut C, arg0: Type) -> Op pub fn constructor_floor_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2736. + // Rule at src/isa/s390x/inst.isle line 2731. let expr0_0 = constructor_fpuroundmode_floor(ctx, pattern0_0)?; let expr1_0 = constructor_fpu_round(ctx, pattern0_0, &expr0_0, pattern1_0)?; return Some(expr1_0); @@ -6104,12 +6104,12 @@ pub fn constructor_floor_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> pub fn constructor_fpuroundmode_trunc(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32 { - // Rule at src/isa/s390x/inst.isle line 2742. + // Rule at src/isa/s390x/inst.isle line 2737. let expr0_0 = FpuRoundMode::Zero32; return Some(expr0_0); } if pattern0_0 == F64 { - // Rule at src/isa/s390x/inst.isle line 2743. + // Rule at src/isa/s390x/inst.isle line 2738. let expr0_0 = FpuRoundMode::Zero64; return Some(expr0_0); } @@ -6120,7 +6120,7 @@ pub fn constructor_fpuroundmode_trunc(ctx: &mut C, arg0: Type) -> Op pub fn constructor_trunc_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2746. + // Rule at src/isa/s390x/inst.isle line 2741. let expr0_0 = constructor_fpuroundmode_trunc(ctx, pattern0_0)?; let expr1_0 = constructor_fpu_round(ctx, pattern0_0, &expr0_0, pattern1_0)?; return Some(expr1_0); @@ -6133,12 +6133,12 @@ pub fn constructor_fpuroundmode_nearest( ) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32 { - // Rule at src/isa/s390x/inst.isle line 2752. + // Rule at src/isa/s390x/inst.isle line 2747. let expr0_0 = FpuRoundMode::Nearest32; return Some(expr0_0); } if pattern0_0 == F64 { - // Rule at src/isa/s390x/inst.isle line 2753. + // Rule at src/isa/s390x/inst.isle line 2748. let expr0_0 = FpuRoundMode::Nearest64; return Some(expr0_0); } @@ -6149,7 +6149,7 @@ pub fn constructor_fpuroundmode_nearest( pub fn constructor_nearest_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2756. + // Rule at src/isa/s390x/inst.isle line 2751. let expr0_0 = constructor_fpuroundmode_nearest(ctx, pattern0_0)?; let expr1_0 = constructor_fpu_round(ctx, pattern0_0, &expr0_0, pattern1_0)?; return Some(expr1_0); @@ -6165,7 +6165,7 @@ pub fn constructor_fpuop1_promote( if pattern0_0 == F64 { let pattern2_0 = arg1; if pattern2_0 == F32 { - // Rule at src/isa/s390x/inst.isle line 2762. + // Rule at src/isa/s390x/inst.isle line 2757. let expr0_0 = FPUOp1::Cvt32To64; return Some(expr0_0); } @@ -6183,7 +6183,7 @@ pub fn constructor_fpromote_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2765. + // Rule at src/isa/s390x/inst.isle line 2760. let expr0_0 = constructor_fpuop1_promote(ctx, pattern0_0, pattern1_0)?; let expr1_0 = constructor_fpu_rr(ctx, pattern0_0, &expr0_0, pattern2_0)?; return Some(expr1_0); @@ -6199,7 +6199,7 @@ pub fn constructor_fpuop1_demote( if pattern0_0 == F32 { let pattern2_0 = arg1; if pattern2_0 == F64 { - // Rule at src/isa/s390x/inst.isle line 2772. + // Rule at src/isa/s390x/inst.isle line 2767. let expr0_0 = FPUOp1::Cvt64To32; return Some(expr0_0); } @@ -6217,7 +6217,7 @@ pub fn constructor_fdemote_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2775. + // Rule at src/isa/s390x/inst.isle line 2770. let expr0_0 = constructor_fpuop1_demote(ctx, pattern0_0, pattern1_0)?; let expr1_0 = constructor_fpu_rr(ctx, pattern0_0, &expr0_0, pattern2_0)?; return Some(expr1_0); @@ -6233,12 +6233,12 @@ pub fn constructor_uint_to_fpu_op( if pattern0_0 == F32 { let pattern2_0 = arg1; if pattern2_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 2782. + // Rule at src/isa/s390x/inst.isle line 2777. let expr0_0 = IntToFpuOp::U32ToF32; return Some(expr0_0); } if pattern2_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2784. + // Rule at src/isa/s390x/inst.isle line 2779. let expr0_0 = IntToFpuOp::U64ToF32; return Some(expr0_0); } @@ -6246,12 +6246,12 @@ pub fn constructor_uint_to_fpu_op( if pattern0_0 == F64 { let pattern2_0 = arg1; if pattern2_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 2783. + // Rule at src/isa/s390x/inst.isle line 2778. let expr0_0 = IntToFpuOp::U32ToF64; return Some(expr0_0); } if pattern2_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2785. + // Rule at src/isa/s390x/inst.isle line 2780. let expr0_0 = IntToFpuOp::U64ToF64; return Some(expr0_0); } @@ -6269,7 +6269,7 @@ pub fn constructor_fcvt_from_uint_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2788. + // Rule at src/isa/s390x/inst.isle line 2783. let expr0_0 = constructor_uint_to_fpu_op(ctx, pattern0_0, pattern1_0)?; let expr1_0 = constructor_int_to_fpu(ctx, pattern0_0, &expr0_0, pattern2_0)?; return Some(expr1_0); @@ -6285,12 +6285,12 @@ pub fn constructor_sint_to_fpu_op( if pattern0_0 == F32 { let pattern2_0 = arg1; if pattern2_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 2795. + // Rule at src/isa/s390x/inst.isle line 2790. let expr0_0 = IntToFpuOp::I32ToF32; return Some(expr0_0); } if pattern2_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2797. + // Rule at src/isa/s390x/inst.isle line 2792. let expr0_0 = IntToFpuOp::I64ToF32; return Some(expr0_0); } @@ -6298,12 +6298,12 @@ pub fn constructor_sint_to_fpu_op( if pattern0_0 == F64 { let pattern2_0 = arg1; if pattern2_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 2796. + // Rule at src/isa/s390x/inst.isle line 2791. let expr0_0 = IntToFpuOp::I32ToF64; return Some(expr0_0); } if pattern2_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2798. + // Rule at src/isa/s390x/inst.isle line 2793. let expr0_0 = IntToFpuOp::I64ToF64; return Some(expr0_0); } @@ -6321,7 +6321,7 @@ pub fn constructor_fcvt_from_sint_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2801. + // Rule at src/isa/s390x/inst.isle line 2796. let expr0_0 = constructor_sint_to_fpu_op(ctx, pattern0_0, pattern1_0)?; let expr1_0 = constructor_int_to_fpu(ctx, pattern0_0, &expr0_0, pattern2_0)?; return Some(expr1_0); @@ -6337,12 +6337,12 @@ pub fn constructor_fpu_to_uint_op( if pattern0_0 == I32 { let pattern2_0 = arg1; if pattern2_0 == F32 { - // Rule at src/isa/s390x/inst.isle line 2808. + // Rule at src/isa/s390x/inst.isle line 2803. let expr0_0 = FpuToIntOp::F32ToU32; return Some(expr0_0); } if pattern2_0 == F64 { - // Rule at src/isa/s390x/inst.isle line 2809. + // Rule at src/isa/s390x/inst.isle line 2804. let expr0_0 = FpuToIntOp::F64ToU32; return Some(expr0_0); } @@ -6350,12 +6350,12 @@ pub fn constructor_fpu_to_uint_op( if pattern0_0 == I64 { let pattern2_0 = arg1; if pattern2_0 == F32 { - // Rule at src/isa/s390x/inst.isle line 2810. + // Rule at src/isa/s390x/inst.isle line 2805. let expr0_0 = FpuToIntOp::F32ToU64; return Some(expr0_0); } if pattern2_0 == F64 { - // Rule at src/isa/s390x/inst.isle line 2811. + // Rule at src/isa/s390x/inst.isle line 2806. let expr0_0 = FpuToIntOp::F64ToU64; return Some(expr0_0); } @@ -6373,7 +6373,7 @@ pub fn constructor_fcvt_to_uint_reg_with_flags( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2814. + // Rule at src/isa/s390x/inst.isle line 2809. let expr0_0 = constructor_fpu_to_uint_op(ctx, pattern0_0, pattern1_0)?; let expr1_0 = constructor_fpu_to_int(ctx, pattern0_0, &expr0_0, pattern2_0)?; return Some(expr1_0); @@ -6389,7 +6389,7 @@ pub fn constructor_fcvt_to_uint_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2818. + // Rule at src/isa/s390x/inst.isle line 2813. let expr0_0 = constructor_fcvt_to_uint_reg_with_flags(ctx, pattern0_0, pattern1_0, pattern2_0)?; let expr1_0 = constructor_drop_flags(ctx, &expr0_0)?; return Some(expr1_0); @@ -6405,12 +6405,12 @@ pub fn constructor_fpu_to_sint_op( if pattern0_0 == I32 { let pattern2_0 = arg1; if pattern2_0 == F32 { - // Rule at src/isa/s390x/inst.isle line 2825. + // Rule at src/isa/s390x/inst.isle line 2820. let expr0_0 = FpuToIntOp::F32ToI32; return Some(expr0_0); } if pattern2_0 == F64 { - // Rule at src/isa/s390x/inst.isle line 2826. + // Rule at src/isa/s390x/inst.isle line 2821. let expr0_0 = FpuToIntOp::F64ToI32; return Some(expr0_0); } @@ -6418,12 +6418,12 @@ pub fn constructor_fpu_to_sint_op( if pattern0_0 == I64 { let pattern2_0 = arg1; if pattern2_0 == F32 { - // Rule at src/isa/s390x/inst.isle line 2827. + // Rule at src/isa/s390x/inst.isle line 2822. let expr0_0 = FpuToIntOp::F32ToI64; return Some(expr0_0); } if pattern2_0 == F64 { - // Rule at src/isa/s390x/inst.isle line 2828. + // Rule at src/isa/s390x/inst.isle line 2823. let expr0_0 = FpuToIntOp::F64ToI64; return Some(expr0_0); } @@ -6441,7 +6441,7 @@ pub fn constructor_fcvt_to_sint_reg_with_flags( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2831. + // Rule at src/isa/s390x/inst.isle line 2826. let expr0_0 = constructor_fpu_to_sint_op(ctx, pattern0_0, pattern1_0)?; let expr1_0 = constructor_fpu_to_int(ctx, pattern0_0, &expr0_0, pattern2_0)?; return Some(expr1_0); @@ -6457,7 +6457,7 @@ pub fn constructor_fcvt_to_sint_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2835. + // Rule at src/isa/s390x/inst.isle line 2830. let expr0_0 = constructor_fcvt_to_sint_reg_with_flags(ctx, pattern0_0, pattern1_0, pattern2_0)?; let expr1_0 = constructor_drop_flags(ctx, &expr0_0)?; return Some(expr1_0); @@ -6467,12 +6467,12 @@ pub fn constructor_fcvt_to_sint_reg( pub fn constructor_cmpop_cmps(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 2842. + // Rule at src/isa/s390x/inst.isle line 2837. let expr0_0 = CmpOp::CmpS32; return Some(expr0_0); } if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2843. + // Rule at src/isa/s390x/inst.isle line 2838. let expr0_0 = CmpOp::CmpS64; return Some(expr0_0); } @@ -6483,12 +6483,12 @@ pub fn constructor_cmpop_cmps(ctx: &mut C, arg0: Type) -> Option(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 2846. + // Rule at src/isa/s390x/inst.isle line 2841. let expr0_0 = CmpOp::CmpS32Ext16; return Some(expr0_0); } if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2847. + // Rule at src/isa/s390x/inst.isle line 2842. let expr0_0 = CmpOp::CmpS64Ext16; return Some(expr0_0); } @@ -6499,7 +6499,7 @@ pub fn constructor_cmpop_cmps_sext16(ctx: &mut C, arg0: Type) -> Opt pub fn constructor_cmpop_cmps_sext32(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2850. + // Rule at src/isa/s390x/inst.isle line 2845. let expr0_0 = CmpOp::CmpS64Ext32; return Some(expr0_0); } @@ -6516,7 +6516,7 @@ pub fn constructor_icmps_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2853. + // Rule at src/isa/s390x/inst.isle line 2848. let expr0_0 = constructor_cmpop_cmps(ctx, pattern0_0)?; let expr1_0 = constructor_cmp_rr(ctx, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -6532,7 +6532,7 @@ pub fn constructor_icmps_reg_sext32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2856. + // Rule at src/isa/s390x/inst.isle line 2851. let expr0_0 = constructor_cmpop_cmps_sext32(ctx, pattern0_0)?; let expr1_0 = constructor_cmp_rr(ctx, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -6548,7 +6548,7 @@ pub fn constructor_icmps_simm16( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2859. + // Rule at src/isa/s390x/inst.isle line 2854. let expr0_0 = constructor_cmpop_cmps(ctx, pattern0_0)?; let expr1_0 = constructor_cmp_rsimm16(ctx, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -6564,7 +6564,7 @@ pub fn constructor_icmps_simm32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2862. + // Rule at src/isa/s390x/inst.isle line 2857. let expr0_0 = constructor_cmpop_cmps(ctx, pattern0_0)?; let expr1_0 = constructor_cmp_rsimm32(ctx, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -6580,7 +6580,7 @@ pub fn constructor_icmps_mem( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2865. + // Rule at src/isa/s390x/inst.isle line 2860. let expr0_0 = constructor_cmpop_cmps(ctx, pattern0_0)?; let expr1_0 = constructor_cmp_rx(ctx, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -6596,7 +6596,7 @@ pub fn constructor_icmps_mem_sext16( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2868. + // Rule at src/isa/s390x/inst.isle line 2863. let expr0_0 = constructor_cmpop_cmps_sext16(ctx, pattern0_0)?; let expr1_0 = constructor_cmp_rx(ctx, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -6612,7 +6612,7 @@ pub fn constructor_icmps_mem_sext32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2871. + // Rule at src/isa/s390x/inst.isle line 2866. let expr0_0 = constructor_cmpop_cmps_sext32(ctx, pattern0_0)?; let expr1_0 = constructor_cmp_rx(ctx, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -6622,12 +6622,12 @@ pub fn constructor_icmps_mem_sext32( pub fn constructor_cmpop_cmpu(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 2877. + // Rule at src/isa/s390x/inst.isle line 2872. let expr0_0 = CmpOp::CmpL32; return Some(expr0_0); } if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2878. + // Rule at src/isa/s390x/inst.isle line 2873. let expr0_0 = CmpOp::CmpL64; return Some(expr0_0); } @@ -6638,12 +6638,12 @@ pub fn constructor_cmpop_cmpu(ctx: &mut C, arg0: Type) -> Option(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 2881. + // Rule at src/isa/s390x/inst.isle line 2876. let expr0_0 = CmpOp::CmpL32Ext16; return Some(expr0_0); } if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2882. + // Rule at src/isa/s390x/inst.isle line 2877. let expr0_0 = CmpOp::CmpL64Ext16; return Some(expr0_0); } @@ -6654,7 +6654,7 @@ pub fn constructor_cmpop_cmpu_zext16(ctx: &mut C, arg0: Type) -> Opt pub fn constructor_cmpop_cmpu_zext32(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2885. + // Rule at src/isa/s390x/inst.isle line 2880. let expr0_0 = CmpOp::CmpL64Ext32; return Some(expr0_0); } @@ -6671,7 +6671,7 @@ pub fn constructor_icmpu_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2888. + // Rule at src/isa/s390x/inst.isle line 2883. let expr0_0 = constructor_cmpop_cmpu(ctx, pattern0_0)?; let expr1_0 = constructor_cmp_rr(ctx, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -6687,7 +6687,7 @@ pub fn constructor_icmpu_reg_zext32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2891. + // Rule at src/isa/s390x/inst.isle line 2886. let expr0_0 = constructor_cmpop_cmpu_zext32(ctx, pattern0_0)?; let expr1_0 = constructor_cmp_rr(ctx, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -6703,7 +6703,7 @@ pub fn constructor_icmpu_uimm32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2894. + // Rule at src/isa/s390x/inst.isle line 2889. let expr0_0 = constructor_cmpop_cmpu(ctx, pattern0_0)?; let expr1_0 = constructor_cmp_ruimm32(ctx, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -6719,7 +6719,7 @@ pub fn constructor_icmpu_mem( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2897. + // Rule at src/isa/s390x/inst.isle line 2892. let expr0_0 = constructor_cmpop_cmpu(ctx, pattern0_0)?; let expr1_0 = constructor_cmp_rx(ctx, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -6735,7 +6735,7 @@ pub fn constructor_icmpu_mem_zext16( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2900. + // Rule at src/isa/s390x/inst.isle line 2895. let expr0_0 = constructor_cmpop_cmpu_zext16(ctx, pattern0_0)?; let expr1_0 = constructor_cmp_rx(ctx, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -6751,7 +6751,7 @@ pub fn constructor_icmpu_mem_zext32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2903. + // Rule at src/isa/s390x/inst.isle line 2898. let expr0_0 = constructor_cmpop_cmpu_zext32(ctx, pattern0_0)?; let expr1_0 = constructor_cmp_rx(ctx, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -6768,14 +6768,14 @@ pub fn constructor_fcmp_reg( if pattern0_0 == F32 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2909. + // Rule at src/isa/s390x/inst.isle line 2904. let expr0_0 = constructor_fpu_cmp32(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if pattern0_0 == F64 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2910. + // Rule at src/isa/s390x/inst.isle line 2905. let expr0_0 = constructor_fpu_cmp64(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } diff --git a/cranelift/codegen/src/isa/x64/encoding/rex.rs b/cranelift/codegen/src/isa/x64/encoding/rex.rs index 17e5408cc1b9..509309205df5 100644 --- a/cranelift/codegen/src/isa/x64/encoding/rex.rs +++ b/cranelift/codegen/src/isa/x64/encoding/rex.rs @@ -47,7 +47,8 @@ pub(crate) fn encode_sib(shift: u8, enc_index: u8, enc_base: u8) -> u8 { /// Get the encoding number of a GPR. #[inline(always)] -pub(crate) fn int_reg_enc(reg: Reg) -> u8 { +pub(crate) fn int_reg_enc(reg: impl Into) -> u8 { + let reg = reg.into(); debug_assert!(reg.is_real()); debug_assert_eq!(reg.get_class(), RegClass::I64); reg.get_hw_encoding() @@ -55,7 +56,8 @@ pub(crate) fn int_reg_enc(reg: Reg) -> u8 { /// Get the encoding number of any register. #[inline(always)] -pub(crate) fn reg_enc(reg: Reg) -> u8 { +pub(crate) fn reg_enc(reg: impl Into) -> u8 { + let reg = reg.into(); debug_assert!(reg.is_real()); reg.get_hw_encoding() } diff --git a/cranelift/codegen/src/isa/x64/inst.isle b/cranelift/codegen/src/isa/x64/inst.isle index a126750872fa..4ba053caef37 100644 --- a/cranelift/codegen/src/isa/x64/inst.isle +++ b/cranelift/codegen/src/isa/x64/inst.isle @@ -2,82 +2,486 @@ ;;;; `MInst` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -(type MInst extern - (enum (Nop (len u8)) - (Ud2 (trap_code TrapCode)) - (AluRmiR (size OperandSize) - (op AluRmiROpcode) - (src1 Reg) - (src2 RegMemImm) - (dst WritableReg)) - (MulHi (size OperandSize) - (signed bool) - (src1 Reg) - (src2 RegMem) - (dst_lo WritableReg) - (dst_hi WritableReg)) - (XmmRmR (op SseOpcode) - (src1 Reg) - (src2 RegMem) - (dst WritableReg)) - (XmmUnaryRmR (op SseOpcode) - (src RegMem) - (dst WritableReg)) - (XmmUnaryRmREvex (op Avx512Opcode) - (src RegMem) - (dst WritableReg)) - (XmmRmiReg (opcode SseOpcode) - (src1 Reg) - (src2 RegMemImm) +;; Don't build `MInst` variants directly, in general. Instead, use the +;; instruction-emitting helpers defined further down. + +(type MInst nodebug + (enum + ;; Nops of various sizes, including zero. + (Nop (len u8)) + + ;; ========================================= + ;; Integer instructions. + + ;; Integer arithmetic/bit-twiddling. + (AluRmiR (size OperandSize) ;; 4 or 8 + (op AluRmiROpcode) + (src1 Reg) + (src2 RegMemImm) + (dst WritableReg)) + + ;; Instructions on general-purpose registers that only read src and + ;; defines dst (dst is not modified). `bsr`, etc. + (UnaryRmR (size OperandSize) ;; 2, 4, or 8 + (op UnaryRmROpcode) + (src GprMem) + (dst WritableGpr)) + + ;; Bitwise not. + (Not (size OperandSize) ;; 1, 2, 4, or 8 + (src Gpr) + (dst WritableGpr)) + + ;; Integer negation. + (Neg (size OperandSize) ;; 1, 2, 4, or 8 + (src Gpr) + (dst WritableGpr)) + + ;; Integer quotient and remainder: (div idiv) $rax $rdx (reg addr) + (Div (size OperandSize) ;; 1, 2, 4, or 8 + (signed bool) + (divisor RegMem) + (dividend Reg) + (dst_quotient WritableReg) + (dst_remainder WritableReg)) + + ;; The high (and low) bits of a (un)signed multiply: `RDX:RAX := RAX * + ;; rhs`. + (MulHi (size OperandSize) + (signed bool) + (src1 Reg) + (src2 RegMem) + (dst_lo WritableReg) + (dst_hi WritableReg)) + + ;; A synthetic sequence to implement the right inline checks for + ;; remainder and division, assuming the dividend is in %rax. + ;; + ;; Puts the result back into %rax if is_div, %rdx if !is_div, to mimic + ;; what the div instruction does. + ;; + ;; The generated code sequence is described in the emit's function match + ;; arm for this instruction. + ;; + ;; Note: %rdx is marked as modified by this instruction, to avoid an + ;; early clobber problem with the temporary and divisor registers. Make + ;; sure to zero %rdx right before this instruction, or you might run into + ;; regalloc failures where %rdx is live before its first def! + (CheckedDivOrRemSeq (kind DivOrRemKind) + (size OperandSize) + (dividend Reg) + ;; The divisor operand. Note it's marked as modified + ;; so that it gets assigned a register different from + ;; the temporary. + (divisor WritableReg) + (dst_quotient WritableReg) + (dst_remainder WritableReg) + (tmp OptionWritableReg)) + + ;; Do a sign-extend based on the sign of the value in rax into rdx: (cwd + ;; cdq cqo) or al into ah: (cbw) + (SignExtendData (size OperandSize) ;; 1, 2, 4, or 8 + (src Reg) (dst WritableReg)) - (XmmRmRImm (op SseOpcode) - (src1 Reg) - (src2 RegMem) - (dst WritableReg) - (imm u8) - (size OperandSize)) - (XmmUninitializedValue (dst WritableReg)) - (CmpRmiR (size OperandSize) - (opcode CmpOpcode) - (src RegMemImm) - (dst Reg)) - (Imm (dst_size OperandSize) - (simm64 u64) + + ;; Constant materialization: (imm32 imm64) reg. + ;; + ;; Either: movl $imm32, %reg32 or movabsq $imm64, %reg32. + (Imm (dst_size OperandSize) ;; 4 or 8 + (simm64 u64) + (dst WritableReg)) + + ;; GPR to GPR move: mov (64 32) reg reg. + (MovRR (size OperandSize) ;; 4 or 8 + (src Reg) + (dst WritableReg)) + + ;; Zero-extended loads, except for 64 bits: movz (bl bq wl wq lq) addr + ;; reg. + ;; + ;; Note that the lq variant doesn't really exist since the default + ;; zero-extend rule makes it unnecessary. For that case we emit the + ;; equivalent "movl AM, reg32". + (MovzxRmR (ext_mode ExtMode) + (src RegMem) (dst WritableReg)) - (ShiftR (size OperandSize) - (kind ShiftKind) - (src Reg) - (num_bits Imm8Reg) - (dst WritableReg)) - (MovzxRmR (ext_mode ExtMode) - (src RegMem) - (dst WritableReg)) - (MovsxRmR (ext_mode ExtMode) - (src RegMem) - (dst WritableReg)) - (Mov64MR (src SyntheticAmode) - (dst WritableReg)) - (Cmove (size OperandSize) - (cc CC) - (consequent RegMem) - (alternative Reg) - (dst WritableReg)) - (XmmRmREvex (op Avx512Opcode) - (src1 RegMem) - (src2 Reg) - (dst WritableReg)) - (GprToXmm (op SseOpcode) - (src RegMem) - (dst WritableReg) - (src_size OperandSize)) - (Not (size OperandSize) - (src Reg) + + ;; A plain 64-bit integer load, since MovZX_RM_R can't represent that. + (Mov64MR (src SyntheticAmode) + (dst WritableReg)) + + ;; Loads the memory address of addr into dst. + (LoadEffectiveAddress (addr SyntheticAmode) + (dst WritableGpr)) + + ;; Sign-extended loads and moves: movs (bl bq wl wq lq) addr reg. + (MovsxRmR (ext_mode ExtMode) + (src RegMem) (dst WritableReg)) - (Neg (size OperandSize) + + ;; Integer stores: mov (b w l q) reg addr. + (MovRM (size OperandSize) ;; 1, 2, 4, or 8 + (src Reg) + (dst SyntheticAmode)) + + ;; Arithmetic shifts: (shl shr sar) (b w l q) imm reg. + (ShiftR (size OperandSize) ;; 1, 2, 4, or 8 + (kind ShiftKind) + (src Reg) + ;; shift count: `Imm8Reg::Imm8(0 .. #bits-in-type - 1)` or + ;; `Imm8Reg::Reg(r)` where `r` get's move mitosis'd into `%cl`. + (num_bits Imm8Reg) + (dst WritableReg)) + + ;; Arithmetic SIMD shifts. + (XmmRmiReg (opcode SseOpcode) + (src1 Xmm) + (src2 XmmMemImm) + (dst WritableXmm)) + + ;; Integer comparisons/tests: cmp or test (b w l q) (reg addr imm) reg. + (CmpRmiR (size OperandSize) ;; 1, 2, 4, or 8 + (opcode CmpOpcode) + (src RegMemImm) + (dst Reg)) + + ;; Materializes the requested condition code in the destinaton reg. + (Setcc (cc CC) + (dst WritableReg)) + + ;; Integer conditional move. + ;; + ;; Overwrites the destination register. + (Cmove (size OperandSize) + (cc CC) + (consequent RegMem) + (alternative Reg) + (dst WritableReg)) + + ;; ========================================= + ;; Stack manipulation. + + ;; pushq (reg addr imm) + (Push64 (src RegMemImm)) + + ;; popq reg + (Pop64 (dst WritableReg)) + + ;; ========================================= + ;; Floating-point operations. + + ;; XMM (scalar or vector) binary op: (add sub and or xor mul adc? sbb?) + ;; (32 64) (reg addr) reg + (XmmRmR (op SseOpcode) + (src1 Xmm) + (src2 XmmMem) + (dst WritableXmm)) + + ;; XMM (scalar or vector) binary op that relies on the EVEX prefix. + (XmmRmREvex (op Avx512Opcode) + (src1 XmmMem) + (src2 Xmm) + (dst WritableXmm)) + + ;; XMM (scalar or vector) unary op: mov between XMM registers (32 64) + ;; (reg addr) reg, sqrt, etc. + ;; + ;; This differs from XMM_RM_R in that the dst register of XmmUnaryRmR is + ;; not used in the computation of the instruction dst value and so does + ;; not have to be a previously valid value. This is characteristic of mov + ;; instructions. + (XmmUnaryRmR (op SseOpcode) + (src XmmMem) + (dst WritableXmm)) + + ;; XMM (scalar or vector) unary op that relies on the EVEX prefix. + (XmmUnaryRmREvex (op Avx512Opcode) + (src XmmMem) + (dst WritableXmm)) + + ;; XMM (scalar or vector) unary op (from xmm to reg/mem): stores, movd, + ;; movq + (XmmMovRM (op SseOpcode) (src Reg) + (dst SyntheticAmode)) + + ;; XMM (vector) unary op (to move a constant value into an xmm register): + ;; movups + (XmmLoadConst (src VCodeConstant) + (dst WritableReg) + (ty Type)) + + ;; XMM (scalar) unary op (from xmm to integer reg): movd, movq, + ;; cvtts{s,d}2si + (XmmToGpr (op SseOpcode) + (src Xmm) + (dst WritableGpr) + (dst_size OperandSize)) + + ;; XMM (scalar) unary op (from integer to float reg): movd, movq, + ;; cvtsi2s{s,d} + (GprToXmm (op SseOpcode) + (src RegMem) + (dst WritableXmm) + (src_size OperandSize)) + + ;; Converts an unsigned int64 to a float32/float64. + (CvtUint64ToFloatSeq (dst_size OperandSize) ;; 4 or 8 + ;; A copy of the source register, fed by + ;; lowering. It is marked as modified during + ;; register allocation to make sure that the + ;; temporary registers differ from the src register, + ;; since both registers are live at the same time in + ;; the generated code sequence. + (src WritableGpr) + (dst WritableXmm) + (tmp_gpr1 WritableGpr) + (tmp_gpr2 WritableGpr)) + + ;; Converts a scalar xmm to a signed int32/int64. + (CvtFloatToSintSeq (dst_size OperandSize) + (src_size OperandSize) + (is_saturating bool) + ;; A copy of the source register, fed by + ;; lowering. It is marked as modified during + ;; register allocation to make sure that the + ;; temporary registers differ from the src register, + ;; since both registers are live at the same time in + ;; the generated code sequence. + (src WritableXmm) + (dst WritableGpr) + (tmp_gpr WritableGpr) + (tmp_xmm WritableXmm)) + + ;; Converts a scalar xmm to an unsigned int32/int64. + (CvtFloatToUintSeq (dst_size OperandSize) + (src_size OperandSize) + (is_saturating bool) + ;; A copy of the source register, fed by + ;; lowering. It is marked as modified during + ;; register allocation to make sure that the + ;; temporary registers differ from the src register, + ;; since both registers are live at the same time in + ;; the generated code sequence. + (src WritableXmm) + (dst WritableGpr) + (tmp_gpr WritableGpr) + (tmp_xmm WritableXmm)) + + ;; A sequence to compute min/max with the proper NaN semantics for xmm + ;; registers. + (XmmMinMaxSeq (size OperandSize) + (is_min bool) + (lhs Reg) + (rhs_dst WritableReg)) + + ;; XMM (scalar) conditional move. + ;; + ;; Overwrites the destination register if cc is set. + (XmmCmove (size OperandSize) + (cc CC) + (src RegMem) (dst WritableReg)) - (LoadEffectiveAddress (addr SyntheticAmode) - (dst WritableReg)))) + + ;; Float comparisons/tests: cmp (b w l q) (reg addr imm) reg. + (XmmCmpRmR (op SseOpcode) + (src RegMem) + (dst Reg)) + + ;; A binary XMM instruction with an 8-bit immediate: e.g. cmp (ps pd) imm + ;; (reg addr) reg + ;; + ;; Note: this has to use `Reg*`, not `Xmm*`, operands because it is used + ;; in various lane insertion and extraction instructions that move + ;; between XMMs and GPRs. + (XmmRmRImm (op SseOpcode) + (src1 Reg) + (src2 RegMem) + (dst WritableReg) + (imm u8) + (size OperandSize)) + + ;; ========================================= + ;; Control flow instructions. + + ;; Direct call: call simm32. + (CallKnown (dest ExternalName) + (uses VecReg) + (defs VecWritableReg) + (opcode Opcode)) + + ;; Indirect call: callq (reg mem) + (CallUnknown (dest RegMem) + (uses VecReg) + (defs VecWritableReg) + (opcode Opcode)) + + ;; Return. + (Ret) + + ;; A placeholder instruction, generating no code, meaning that a function + ;; epilogue must be inserted there. + (EpiloguePlaceholder) + + ;; Jump to a known target: jmp simm32. + (JmpKnown (dst MachLabel)) + + + ;; One-way conditional branch: jcond cond target. + ;; + ;; This instruction is useful when we have conditional jumps depending on + ;; more than two conditions, see for instance the lowering of Brz/brnz + ;; with Fcmp inputs. + ;; + ;; A note of caution: in contexts where the branch target is another + ;; block, this has to be the same successor as the one specified in the + ;; terminator branch of the current block. Otherwise, this might confuse + ;; register allocation by creating new invisible edges. + (JmpIf (cc CC) + (taken MachLabel)) + + ;; Two-way conditional branch: jcond cond target target. + ;; + ;; Emitted as a compound sequence; the MachBuffer will shrink it as + ;; appropriate. + (JmpCond (cc CC) + (taken MachLabel) + (not_taken MachLabel)) + + ;; Jump-table sequence, as one compound instruction (see note in lower.rs + ;; for rationale). + ;; + ;; The generated code sequence is described in the emit's function match + ;; arm for this instruction. + ;; + ;; See comment in lowering about the temporaries signedness. + (JmpTableSeq (idx Reg) + (tmp1 WritableReg) + (tmp2 WritableReg) + (default_target MachLabel) + (targets VecMachLabel) + (targets_for_term VecMachLabel)) + + ;; Indirect jump: jmpq (reg mem). + (JmpUnknown (target RegMem)) + + ;; Traps if the condition code is set. + (TrapIf (cc CC) + (trap_code TrapCode)) + + ;; A debug trap. + (Hlt) + + ;; An instruction that will always trigger the illegal instruction + ;; exception. + (Ud2 (trap_code TrapCode)) + + ;; Loads an external symbol in a register, with a relocation: + ;; + ;; movq $name@GOTPCREL(%rip), dst if PIC is enabled, or + ;; movabsq $name, dst otherwise. + (LoadExtName (dst WritableReg) + (name BoxExternalName) + (offset i64)) + + ;; ========================================= + ;; Instructions pertaining to atomic memory accesses. + + ;; A standard (native) `lock cmpxchg src, (amode)`, with register + ;; conventions: + ;; + ;; `mem` (read) address + ;; `replacement` (read) replacement value + ;; %rax (modified) in: expected value, out: value that was actually at `dst` + ;; %rflags is written. Do not assume anything about it after the instruction. + ;; + ;; The instruction "succeeded" iff the lowest `ty` bits of %rax + ;; afterwards are the same as they were before. + (LockCmpxchg (ty Type) ;; I8, I16, I32, or I64 + (replacement Reg) + (expected Reg) + (mem SyntheticAmode) + (dst_old WritableReg)) + + ;; A synthetic instruction, based on a loop around a native `lock + ;; cmpxchg` instruction. + ;; + ;; This atomically modifies a value in memory and returns the old value. + ;; The sequence consists of an initial "normal" load from `dst`, followed + ;; by a loop which computes the new value and tries to compare-and-swap + ;; ("CAS") it into `dst`, using the native instruction `lock + ;; cmpxchg{b,w,l,q}` . The loop iterates until the CAS is successful. + ;; If there is no contention, there will be only one pass through the + ;; loop body. The sequence does *not* perform any explicit memory fence + ;; instructions (mfence/sfence/lfence). + ;; + ;; Note that the transaction is atomic in the sense that, as observed by + ;; some other thread, `dst` either has the initial or final value, but no + ;; other. It isn't atomic in the sense of guaranteeing that no other + ;; thread writes to `dst` in between the initial load and the CAS -- but + ;; that would cause the CAS to fail unless the other thread's last write + ;; before the CAS wrote the same value that was already there. In other + ;; words, this implementation suffers (unavoidably) from the A-B-A + ;; problem. + ;; + ;; This instruction sequence has fixed register uses as follows: + ;; + ;; %r9 (read) address + ;; %r10 (read) second operand for `op` + ;; %r11 (written) scratch reg; value afterwards has no meaning + ;; %rax (written) the old value at %r9 + ;; %rflags is written. Do not assume anything about it after the instruction. + (AtomicRmwSeq (ty Type) ;; I8, I16, I32, or I64 + (op AtomicRmwOp) + (address Reg) + (operand Reg) + (temp WritableReg) + (dst_old WritableReg)) + + ;; A memory fence (mfence, lfence or sfence). + (Fence (kind FenceKind)) + + ;; ========================================= + ;; Meta-instructions generating no code. + + ;; Marker, no-op in generated code: SP "virtual offset" is adjusted. + ;; + ;; This controls how `MemArg::NominalSPOffset` args are lowered. + (VirtualSPOffsetAdj (offset i64)) + + ;; Provides a way to tell the register allocator that the upcoming + ;; sequence of instructions will overwrite `dst` so it should be + ;; considered as a `def`; use this with care. + ;; + ;; This is useful when we have a sequence of instructions whose register + ;; usages are nominally `mod`s, but such that the combination of + ;; operations creates a result that is independent of the initial + ;; register value. It's thus semantically a `def`, not a `mod`, when all + ;; the instructions are taken together, so we want to ensure the register + ;; is defined (its live-range starts) prior to the sequence to keep + ;; analyses happy. + ;; + ;; One alternative would be a compound instruction that somehow + ;; encapsulates the others and reports its own `def`s/`use`s/`mod`s; this + ;; adds complexity (the instruction list is no longer flat) and requires + ;; knowledge about semantics and initial-value independence anyway. + (XmmUninitializedValue (dst WritableXmm)) + + ;; A call to the `ElfTlsGetAddr` libcall. Returns address of TLS symbol + ;; in `rax`. + (ElfTlsGetAddr (symbol ExternalName)) + + ;; A Mach-O TLS symbol access. Returns address of the TLS symbol in + ;; `rax`. + (MachOTlsGetAddr (symbol ExternalName)) + + ;; A definition of a value label. + (ValueLabelMarker (reg Reg) + (label ValueLabel)) + + ;; An unwind pseudoinstruction describing the state of the machine at + ;; this program point. + (Unwind (inst UnwindInst)))) (type OperandSize extern (enum Size8 @@ -85,6 +489,13 @@ Size32 Size64)) +(type VCodeConstant (primitive VCodeConstant)) + +(type FenceKind extern + (enum MFence + LFence + SFence)) + ;; Get the `OperandSize` for a given `Type`, rounding smaller types up to 32 bits. (decl operand_size_of_type_32_64 (Type) OperandSize) (extern constructor operand_size_of_type_32_64 operand_size_of_type_32_64) @@ -112,6 +523,19 @@ And8 Or8)) +(type UnaryRmROpcode extern + (enum Bsr + Bsf + Lzcnt + Tzcnt + Popcnt)) + +(type DivOrRemKind extern + (enum SignedDiv + UnsignedDiv + SignedRem + UnsignedRem)) + (type SseOpcode extern (enum Addps Addpd @@ -330,7 +754,7 @@ (type Amode extern (enum)) -(decl amode_imm_reg_reg_shift (u32 Reg Reg u8) Amode) +(decl amode_imm_reg_reg_shift (u32 Gpr Gpr u8) Amode) (extern constructor amode_imm_reg_reg_shift amode_imm_reg_reg_shift) (decl amode_to_synthetic_amode (Amode) SyntheticAmode) @@ -397,13 +821,176 @@ (decl encode_fcmp_imm (FcmpImm) u8) (extern constructor encode_fcmp_imm encode_fcmp_imm) +;;;; Newtypes for Different Register Classes ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(type Gpr (primitive Gpr)) +(type WritableGpr (primitive WritableGpr)) +(type GprMem extern (enum)) +(type GprMemImm extern (enum)) + +(type Xmm (primitive Xmm)) +(type WritableXmm (primitive WritableXmm)) +(type XmmMem extern (enum)) +(type XmmMemImm extern (enum)) + +;; Convert a `WritableGpr` to a `WritableReg`. +(decl writable_gpr_to_reg (WritableGpr) WritableReg) +(extern constructor writable_gpr_to_reg writable_gpr_to_reg) + +;; Convert a `WritableXmm` to a `WritableReg`. +(decl writable_xmm_to_reg (WritableXmm) WritableReg) +(extern constructor writable_xmm_to_reg writable_xmm_to_reg) + +;; Convert a `WritableReg` to a `WritableXmm`. +(decl writable_reg_to_xmm (WritableReg) WritableXmm) +(extern constructor writable_reg_to_xmm writable_reg_to_xmm) + +;; Convert a `WritableXmm` to an `Xmm`. +(decl writable_xmm_to_xmm (WritableXmm) Xmm) +(extern constructor writable_xmm_to_xmm writable_xmm_to_xmm) + +;; Convert a `WritableGpr` to an `Gpr`. +(decl writable_gpr_to_gpr (WritableGpr) Gpr) +(extern constructor writable_gpr_to_gpr writable_gpr_to_gpr) + +;; Convert an `Gpr` to a `Reg`. +(decl gpr_to_reg (Gpr) Reg) +(extern constructor gpr_to_reg gpr_to_reg) + +;; Convert an `Xmm` to a `Reg`. +(decl xmm_to_reg (Xmm) Reg) +(extern constructor xmm_to_reg xmm_to_reg) + +;; Convert an `Xmm` into an `XmmMemImm`. +(decl xmm_to_xmm_mem_imm (Xmm) XmmMemImm) +(extern constructor xmm_to_xmm_mem_imm xmm_to_xmm_mem_imm) + +;; Allocate a new temporary GPR register. +(decl temp_writable_gpr () WritableGpr) +(extern constructor temp_writable_gpr temp_writable_gpr) + +;; Allocate a new temporary XMM register. +(decl temp_writable_xmm () WritableXmm) +(extern constructor temp_writable_xmm temp_writable_xmm) + +;; Construct a new `XmmMem` from the given `RegMem`. +;; +;; Asserts that the `RegMem`'s register, if any, is an XMM register. +(decl xmm_mem_new (RegMem) XmmMem) +(extern constructor xmm_mem_new xmm_mem_new) + +;; Construct a new `GprMemImm` from the given `RegMemImm`. +;; +;; Asserts that the `RegMemImm`'s register, if any, is an GPR register. +(decl gpr_mem_imm_new (RegMemImm) GprMemImm) +(extern constructor gpr_mem_imm_new gpr_mem_imm_new) + +;; Construct a new `XmmMemImm` from the given `RegMemImm`. +;; +;; Asserts that the `RegMemImm`'s register, if any, is an XMM register. +(decl xmm_mem_imm_new (RegMemImm) XmmMemImm) +(extern constructor xmm_mem_imm_new xmm_mem_imm_new) + +;; Construct a new `XmmMem` from an `Xmm`. +(decl xmm_to_xmm_mem (Xmm) XmmMem) +(extern constructor xmm_to_xmm_mem xmm_to_xmm_mem) + +;; Construct a new `XmmMem` from an `RegMem`. +(decl xmm_mem_to_reg_mem (XmmMem) RegMem) +(extern constructor xmm_mem_to_reg_mem xmm_mem_to_reg_mem) + +;; Convert a `GprMem` to a `RegMem`. +(decl gpr_mem_to_reg_mem (GprMem) RegMem) +(extern constructor gpr_mem_to_reg_mem gpr_mem_to_reg_mem) + +;; Construct a new `Xmm` from a `Reg`. +;; +;; Asserts that the register is a XMM. +(decl xmm_new (Reg) Xmm) +(extern constructor xmm_new xmm_new) + +;; Construct a new `Gpr` from a `Reg`. +;; +;; Asserts that the register is a GPR. +(decl gpr_new (Reg) Gpr) +(extern constructor gpr_new gpr_new) + +;; Construct a new `GprMem` from a `RegMem`. +;; +;; Asserts that the `RegMem`'s register, if any, is a GPR. +(decl gpr_mem_new (RegMem) GprMem) +(extern constructor gpr_mem_new gpr_mem_new) + +;; Construct a `GprMem` from a `Reg`. +;; +;; Asserts that the `Reg` is a GPR. +(decl reg_to_gpr_mem (Reg) GprMem) +(extern constructor reg_to_gpr_mem reg_to_gpr_mem) + +;; Put a value into a GPR. +;; +;; Asserts that the value goes into a GPR. +(decl put_in_gpr (Value) Gpr) +(rule (put_in_gpr val) + (gpr_new (put_in_reg val))) + +;; Put a value into a `GprMem`. +;; +;; Asserts that the value goes into a GPR. +(decl put_in_gpr_mem (Value) GprMem) +(rule (put_in_gpr_mem val) + (gpr_mem_new (put_in_reg_mem val))) + +;; Put a value into a `GprMemImm`. +;; +;; Asserts that the value goes into a GPR. +(decl put_in_gpr_mem_imm (Value) GprMemImm) +(rule (put_in_gpr_mem_imm val) + (gpr_mem_imm_new (put_in_reg_mem_imm val))) + +;; Put a value into a XMM. +;; +;; Asserts that the value goes into a XMM. +(decl put_in_xmm (Value) Xmm) +(rule (put_in_xmm val) + (xmm_new (put_in_reg val))) + +;; Put a value into a `XmmMem`. +;; +;; Asserts that the value goes into a XMM. +(decl put_in_xmm_mem (Value) XmmMem) +(rule (put_in_xmm_mem val) + (xmm_mem_new (put_in_reg_mem val))) + +;; Put a value into a `XmmMemImm`. +;; +;; Asserts that the value goes into a XMM. +(decl put_in_xmm_mem_imm (Value) XmmMemImm) +(rule (put_in_xmm_mem_imm val) + (xmm_mem_imm_new (put_in_reg_mem_imm val))) + +;; Construct a `ValueRegs` out of a single GPR register. +(decl value_gpr (Gpr) ValueRegs) +(rule (value_gpr x) + (value_reg (gpr_to_reg x))) + +;; Construct a `ValueRegs` out of two GPR registers. +(decl value_gprs (Gpr Gpr) ValueRegs) +(rule (value_gprs x y) + (value_regs (gpr_to_reg x) (gpr_to_reg y))) + +;; Construct a `ValueRegs` out of a single XMM register. +(decl value_xmm (Xmm) ValueRegs) +(rule (value_xmm x) + (value_reg (xmm_to_reg x))) + ;;;; Helpers for Getting Particular Physical Registers ;;;;;;;;;;;;;;;;;;;;;;;;; ;; ;; These should only be used for legalization purposes, when we can't otherwise ;; rely on something like `Inst::mov_mitosis` to put an operand into the ;; appropriate physical register for whatever reason. -(decl xmm0 () WritableReg) +(decl xmm0 () WritableXmm) (extern constructor xmm0 xmm0) ;;;; Helpers for Querying Enabled ISA Extensions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -507,8 +1094,8 @@ (rule (sse_xor_op $F64X2) (SseOpcode.Xorpd)) (rule (sse_xor_op (multi_lane _bits _lanes)) (SseOpcode.Pxor)) -;; Performs an xor operation of the two operands specified -(decl sse_xor (Type Reg RegMem) Reg) +;; Performs an xor operation of the two operands specified. +(decl sse_xor (Type Xmm XmmMem) Xmm) (rule (sse_xor ty x y) (xmm_rm_r ty (sse_xor_op ty) x y)) ;; Determine the appropriate operation to compare two vectors of the specified @@ -532,45 +1119,46 @@ ;; then it runs the risk of comparing NaN against NaN and not actually producing ;; an all-ones mask. By using integer comparision operations we're guaranteeed ;; that everything is equal to itself. -(decl vector_all_ones (Type) Reg) +(decl vector_all_ones (Type) Xmm) (rule (vector_all_ones ty) - (let ((wr WritableReg (temp_writable_reg ty)) - (r Reg (writable_reg_to_reg wr)) + (let ((wr WritableXmm (temp_writable_xmm)) + (r Xmm (writable_xmm_to_xmm wr)) (_ Unit (emit (MInst.XmmRmR (sse_cmp_op $I32X4) r - (RegMem.Reg r) + (xmm_to_xmm_mem r) wr)))) r)) ;; Helper for creating an SSE register holding an `i64x2` from two `i64` values. -(decl make_i64x2_from_lanes (RegMem RegMem) Reg) +(decl make_i64x2_from_lanes (GprMem GprMem) Xmm) (rule (make_i64x2_from_lanes lo hi) - (let ((dst_w WritableReg (temp_writable_reg $I64X2)) - (dst_r Reg (writable_reg_to_reg dst_w)) - (_0 Unit (emit (MInst.XmmUninitializedValue dst_w))) + (let ((dst_xmm_w WritableXmm (temp_writable_xmm)) + (dst_reg_w WritableReg (writable_xmm_to_reg dst_xmm_w)) + (dst_xmm_r Xmm (writable_xmm_to_xmm dst_xmm_w)) + (dst_reg_r Reg (xmm_to_reg dst_xmm_r)) + (_0 Unit (emit (MInst.XmmUninitializedValue dst_xmm_w))) (_1 Unit (emit (MInst.XmmRmRImm (SseOpcode.Pinsrd) - dst_r - lo - dst_w + dst_reg_r + (gpr_mem_to_reg_mem lo) + dst_reg_w 0 (OperandSize.Size64)))) (_2 Unit (emit (MInst.XmmRmRImm (SseOpcode.Pinsrd) - dst_r - hi - dst_w + dst_reg_r + (gpr_mem_to_reg_mem hi) + dst_reg_w 1 (OperandSize.Size64))))) - dst_r)) + dst_xmm_r)) ;; Move a `RegMemImm.Reg` operand to an XMM register, if necessary. -(decl reg_mem_imm_to_xmm (RegMemImm) RegMemImm) -(rule (reg_mem_imm_to_xmm rmi @ (RegMemImm.Mem _)) rmi) -(rule (reg_mem_imm_to_xmm rmi @ (RegMemImm.Imm _)) rmi) -(rule (reg_mem_imm_to_xmm (RegMemImm.Reg r)) - (RegMemImm.Reg (gpr_to_xmm $I8X16 - (SseOpcode.Movd) - (RegMem.Reg r) - (OperandSize.Size32)))) +(decl mov_rmi_to_xmm (RegMemImm) XmmMemImm) +(rule (mov_rmi_to_xmm rmi @ (RegMemImm.Mem _)) (xmm_mem_imm_new rmi)) +(rule (mov_rmi_to_xmm rmi @ (RegMemImm.Imm _)) (xmm_mem_imm_new rmi)) +(rule (mov_rmi_to_xmm (RegMemImm.Reg r)) + (xmm_to_xmm_mem_imm (gpr_to_xmm (SseOpcode.Movd) + (reg_to_gpr_mem r) + (OperandSize.Size32)))) ;;;; Helpers for Emitting Loads ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -588,24 +1176,24 @@ (writable_reg_to_reg dst))) (rule (x64_load $F32 addr _ext_kind) - (xmm_unary_rm_r (SseOpcode.Movss) - (synthetic_amode_to_reg_mem addr))) + (xmm_to_reg (xmm_unary_rm_r (SseOpcode.Movss) + (xmm_mem_new (synthetic_amode_to_reg_mem addr))))) (rule (x64_load $F64 addr _ext_kind) - (xmm_unary_rm_r (SseOpcode.Movsd) - (synthetic_amode_to_reg_mem addr))) + (xmm_to_reg (xmm_unary_rm_r (SseOpcode.Movsd) + (xmm_mem_new (synthetic_amode_to_reg_mem addr))))) (rule (x64_load $F32X4 addr _ext_kind) - (xmm_unary_rm_r (SseOpcode.Movups) - (synthetic_amode_to_reg_mem addr))) + (xmm_to_reg (xmm_unary_rm_r (SseOpcode.Movups) + (xmm_mem_new (synthetic_amode_to_reg_mem addr))))) (rule (x64_load $F64X2 addr _ext_kind) - (xmm_unary_rm_r (SseOpcode.Movupd) - (synthetic_amode_to_reg_mem addr))) + (xmm_to_reg (xmm_unary_rm_r (SseOpcode.Movupd) + (xmm_mem_new (synthetic_amode_to_reg_mem addr))))) (rule (x64_load (multi_lane _bits _lanes) addr _ext_kind) - (xmm_unary_rm_r (SseOpcode.Movdqu) - (synthetic_amode_to_reg_mem addr))) + (xmm_to_reg (xmm_unary_rm_r (SseOpcode.Movdqu) + (xmm_mem_new (synthetic_amode_to_reg_mem addr))))) ;;;; Instruction Constructors ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; @@ -735,11 +1323,15 @@ ;; `f32` immediates. (rule (imm $F32 bits) - (gpr_to_xmm $F32 (SseOpcode.Movd) (RegMem.Reg (imm $I32 bits)) (OperandSize.Size32))) + (xmm_to_reg (gpr_to_xmm (SseOpcode.Movd) + (gpr_mem_new (RegMem.Reg (imm $I32 bits))) + (OperandSize.Size32)))) ;; `f64` immediates. (rule (imm $F64 bits) - (gpr_to_xmm $F64 (SseOpcode.Movq) (RegMem.Reg (imm $I64 bits)) (OperandSize.Size64))) + (xmm_to_reg (gpr_to_xmm (SseOpcode.Movq) + (gpr_mem_new (RegMem.Reg (imm $I64 bits))) + (OperandSize.Size64)))) (decl nonzero_u64_fits_in_u32 (u64) u64) (extern extractor nonzero_u64_fits_in_u32 nonzero_u64_fits_in_u32) @@ -766,35 +1358,35 @@ ;; Special case for zero immediates with vector types, they turn into an xor ;; specific to the vector type. (rule (imm ty @ (multi_lane _bits _lanes) 0) - (let ((wr WritableReg (temp_writable_reg ty)) - (r Reg (writable_reg_to_reg wr)) + (let ((wr WritableXmm (temp_writable_xmm)) + (r Xmm (writable_xmm_to_xmm wr)) (_ Unit (emit (MInst.XmmRmR (sse_xor_op ty) r - (RegMem.Reg r) + (xmm_to_xmm_mem r) wr)))) - r)) + (xmm_to_reg r))) ;; Special case for `f32` zero immediates to use `xorps`. (rule (imm $F32 0) - (let ((wr WritableReg (temp_writable_reg $F32)) - (r Reg (writable_reg_to_reg wr)) + (let ((wr WritableXmm (temp_writable_xmm)) + (r Xmm (writable_xmm_to_xmm wr)) (_ Unit (emit (MInst.XmmRmR (SseOpcode.Xorps) r - (RegMem.Reg r) + (xmm_to_xmm_mem r) wr)))) - r)) + (xmm_to_reg r))) ;; TODO: use cmpeqps for all 1s ;; Special case for `f64` zero immediates to use `xorpd`. (rule (imm $F64 0) - (let ((wr WritableReg (temp_writable_reg $F64)) - (r Reg (writable_reg_to_reg wr)) + (let ((wr WritableXmm (temp_writable_xmm)) + (r Xmm (writable_xmm_to_xmm wr)) (_ Unit (emit (MInst.XmmRmR (SseOpcode.Xorpd) r - (RegMem.Reg r) + (xmm_to_xmm_mem r) wr)))) - r)) + (xmm_to_reg r))) ;; TODO: use cmpeqpd for all 1s @@ -877,199 +1469,199 @@ (writable_reg_to_reg dst))) ;; Helper for creating `MInst.XmmRmR` instructions. -(decl xmm_rm_r (Type SseOpcode Reg RegMem) Reg) +(decl xmm_rm_r (Type SseOpcode Xmm XmmMem) Xmm) (rule (xmm_rm_r ty op src1 src2) - (let ((dst WritableReg (temp_writable_reg ty)) + (let ((dst WritableXmm (temp_writable_xmm)) (_ Unit (emit (MInst.XmmRmR op src1 src2 dst)))) - (writable_reg_to_reg dst))) + (writable_xmm_to_xmm dst))) ;; Helper for creating `paddb` instructions. -(decl paddb (Reg RegMem) Reg) +(decl paddb (Xmm XmmMem) Xmm) (rule (paddb src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Paddb) src1 src2)) ;; Helper for creating `paddw` instructions. -(decl paddw (Reg RegMem) Reg) +(decl paddw (Xmm XmmMem) Xmm) (rule (paddw src1 src2) (xmm_rm_r $I16X8 (SseOpcode.Paddw) src1 src2)) ;; Helper for creating `paddd` instructions. -(decl paddd (Reg RegMem) Reg) +(decl paddd (Xmm XmmMem) Xmm) (rule (paddd src1 src2) (xmm_rm_r $I32X4 (SseOpcode.Paddd) src1 src2)) ;; Helper for creating `paddq` instructions. -(decl paddq (Reg RegMem) Reg) +(decl paddq (Xmm XmmMem) Xmm) (rule (paddq src1 src2) (xmm_rm_r $I64X2 (SseOpcode.Paddq) src1 src2)) ;; Helper for creating `paddsb` instructions. -(decl paddsb (Reg RegMem) Reg) +(decl paddsb (Xmm XmmMem) Xmm) (rule (paddsb src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Paddsb) src1 src2)) ;; Helper for creating `paddsw` instructions. -(decl paddsw (Reg RegMem) Reg) +(decl paddsw (Xmm XmmMem) Xmm) (rule (paddsw src1 src2) (xmm_rm_r $I16X8 (SseOpcode.Paddsw) src1 src2)) ;; Helper for creating `paddusb` instructions. -(decl paddusb (Reg RegMem) Reg) +(decl paddusb (Xmm XmmMem) Xmm) (rule (paddusb src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Paddusb) src1 src2)) ;; Helper for creating `paddusw` instructions. -(decl paddusw (Reg RegMem) Reg) +(decl paddusw (Xmm XmmMem) Xmm) (rule (paddusw src1 src2) (xmm_rm_r $I16X8 (SseOpcode.Paddusw) src1 src2)) ;; Helper for creating `psubb` instructions. -(decl psubb (Reg RegMem) Reg) +(decl psubb (Xmm XmmMem) Xmm) (rule (psubb src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Psubb) src1 src2)) ;; Helper for creating `psubw` instructions. -(decl psubw (Reg RegMem) Reg) +(decl psubw (Xmm XmmMem) Xmm) (rule (psubw src1 src2) (xmm_rm_r $I16X8 (SseOpcode.Psubw) src1 src2)) ;; Helper for creating `psubd` instructions. -(decl psubd (Reg RegMem) Reg) +(decl psubd (Xmm XmmMem) Xmm) (rule (psubd src1 src2) (xmm_rm_r $I32X4 (SseOpcode.Psubd) src1 src2)) ;; Helper for creating `psubq` instructions. -(decl psubq (Reg RegMem) Reg) +(decl psubq (Xmm XmmMem) Xmm) (rule (psubq src1 src2) (xmm_rm_r $I64X2 (SseOpcode.Psubq) src1 src2)) ;; Helper for creating `psubsb` instructions. -(decl psubsb (Reg RegMem) Reg) +(decl psubsb (Xmm XmmMem) Xmm) (rule (psubsb src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Psubsb) src1 src2)) ;; Helper for creating `psubsw` instructions. -(decl psubsw (Reg RegMem) Reg) +(decl psubsw (Xmm XmmMem) Xmm) (rule (psubsw src1 src2) (xmm_rm_r $I16X8 (SseOpcode.Psubsw) src1 src2)) ;; Helper for creating `psubusb` instructions. -(decl psubusb (Reg RegMem) Reg) +(decl psubusb (Xmm XmmMem) Xmm) (rule (psubusb src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Psubusb) src1 src2)) ;; Helper for creating `psubusw` instructions. -(decl psubusw (Reg RegMem) Reg) +(decl psubusw (Xmm XmmMem) Xmm) (rule (psubusw src1 src2) (xmm_rm_r $I16X8 (SseOpcode.Psubusw) src1 src2)) ;; Helper for creating `pavgb` instructions. -(decl pavgb (Reg RegMem) Reg) +(decl pavgb (Xmm XmmMem) Xmm) (rule (pavgb src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Pavgb) src1 src2)) ;; Helper for creating `pavgw` instructions. -(decl pavgw (Reg RegMem) Reg) +(decl pavgw (Xmm XmmMem) Xmm) (rule (pavgw src1 src2) (xmm_rm_r $I16X8 (SseOpcode.Pavgw) src1 src2)) ;; Helper for creating `pand` instructions. -(decl pand (Reg RegMem) Reg) +(decl pand (Xmm XmmMem) Xmm) (rule (pand src1 src2) (xmm_rm_r $F32X4 (SseOpcode.Pand) src1 src2)) ;; Helper for creating `andps` instructions. -(decl andps (Reg RegMem) Reg) +(decl andps (Xmm XmmMem) Xmm) (rule (andps src1 src2) (xmm_rm_r $F32X4 (SseOpcode.Andps) src1 src2)) ;; Helper for creating `andpd` instructions. -(decl andpd (Reg RegMem) Reg) +(decl andpd (Xmm XmmMem) Xmm) (rule (andpd src1 src2) (xmm_rm_r $F64X2 (SseOpcode.Andpd) src1 src2)) ;; Helper for creating `por` instructions. -(decl por (Reg RegMem) Reg) +(decl por (Xmm XmmMem) Xmm) (rule (por src1 src2) (xmm_rm_r $F32X4 (SseOpcode.Por) src1 src2)) ;; Helper for creating `orps` instructions. -(decl orps (Reg RegMem) Reg) +(decl orps (Xmm XmmMem) Xmm) (rule (orps src1 src2) (xmm_rm_r $F32X4 (SseOpcode.Orps) src1 src2)) ;; Helper for creating `orpd` instructions. -(decl orpd (Reg RegMem) Reg) +(decl orpd (Xmm XmmMem) Xmm) (rule (orpd src1 src2) (xmm_rm_r $F64X2 (SseOpcode.Orpd) src1 src2)) ;; Helper for creating `pxor` instructions. -(decl pxor (Reg RegMem) Reg) +(decl pxor (Xmm XmmMem) Xmm) (rule (pxor src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Pxor) src1 src2)) ;; Helper for creating `xorps` instructions. -(decl xorps (Reg RegMem) Reg) +(decl xorps (Xmm XmmMem) Xmm) (rule (xorps src1 src2) (xmm_rm_r $F32X4 (SseOpcode.Xorps) src1 src2)) ;; Helper for creating `xorpd` instructions. -(decl xorpd (Reg RegMem) Reg) +(decl xorpd (Xmm XmmMem) Xmm) (rule (xorpd src1 src2) (xmm_rm_r $F64X2 (SseOpcode.Xorpd) src1 src2)) ;; Helper for creating `pmullw` instructions. -(decl pmullw (Reg RegMem) Reg) +(decl pmullw (Xmm XmmMem) Xmm) (rule (pmullw src1 src2) (xmm_rm_r $I16X8 (SseOpcode.Pmullw) src1 src2)) ;; Helper for creating `pmulld` instructions. -(decl pmulld (Reg RegMem) Reg) +(decl pmulld (Xmm XmmMem) Xmm) (rule (pmulld src1 src2) (xmm_rm_r $I16X8 (SseOpcode.Pmulld) src1 src2)) ;; Helper for creating `pmulhw` instructions. -(decl pmulhw (Reg RegMem) Reg) +(decl pmulhw (Xmm XmmMem) Xmm) (rule (pmulhw src1 src2) (xmm_rm_r $I16X8 (SseOpcode.Pmulhw) src1 src2)) ;; Helper for creating `pmulhuw` instructions. -(decl pmulhuw (Reg RegMem) Reg) +(decl pmulhuw (Xmm XmmMem) Xmm) (rule (pmulhuw src1 src2) (xmm_rm_r $I16X8 (SseOpcode.Pmulhuw) src1 src2)) ;; Helper for creating `pmuldq` instructions. -(decl pmuldq (Reg RegMem) Reg) +(decl pmuldq (Xmm XmmMem) Xmm) (rule (pmuldq src1 src2) (xmm_rm_r $I16X8 (SseOpcode.Pmuldq) src1 src2)) ;; Helper for creating `pmuludq` instructions. -(decl pmuludq (Reg RegMem) Reg) +(decl pmuludq (Xmm XmmMem) Xmm) (rule (pmuludq src1 src2) (xmm_rm_r $I64X2 (SseOpcode.Pmuludq) src1 src2)) ;; Helper for creating `punpckhwd` instructions. -(decl punpckhwd (Reg RegMem) Reg) +(decl punpckhwd (Xmm XmmMem) Xmm) (rule (punpckhwd src1 src2) (xmm_rm_r $I16X8 (SseOpcode.Punpckhwd) src1 src2)) ;; Helper for creating `punpcklwd` instructions. -(decl punpcklwd (Reg RegMem) Reg) +(decl punpcklwd (Xmm XmmMem) Xmm) (rule (punpcklwd src1 src2) (xmm_rm_r $I16X8 (SseOpcode.Punpcklwd) src1 src2)) ;; Helper for creating `andnps` instructions. -(decl andnps (Reg RegMem) Reg) +(decl andnps (Xmm XmmMem) Xmm) (rule (andnps src1 src2) (xmm_rm_r $F32X4 (SseOpcode.Andnps) src1 src2)) ;; Helper for creating `andnpd` instructions. -(decl andnpd (Reg RegMem) Reg) +(decl andnpd (Xmm XmmMem) Xmm) (rule (andnpd src1 src2) (xmm_rm_r $F64X2 (SseOpcode.Andnpd) src1 src2)) ;; Helper for creating `pandn` instructions. -(decl pandn (Reg RegMem) Reg) +(decl pandn (Xmm XmmMem) Xmm) (rule (pandn src1 src2) (xmm_rm_r $F64X2 (SseOpcode.Pandn) src1 src2)) @@ -1084,367 +1676,387 @@ (rule (sse_mov_op (multi_lane _bits _lanes)) (SseOpcode.Movdqa)) ;; Helper for creating `blendvp{d,s}` and `pblendvb` instructions. -(decl sse_blend (Type RegMem RegMem Reg) Reg) +(decl sse_blend (Type XmmMem XmmMem Xmm) Xmm) (rule (sse_blend ty mask src1 src2) ;; Move the mask into `xmm0`, as blend instructions implicitly operate on ;; that register. (This kind of thing would normally happen inside of ;; `Inst::mov_mitosis`, but has to happen here, where we still have the ;; mask register, because the mask is implicit and doesn't appear in the ;; `Inst` itself.) - (let ((mask2 WritableReg (xmm0)) - (_ Unit (emit (MInst.XmmUnaryRmR (sse_mov_op ty) mask mask2)))) + (let ((mask2 WritableXmm (xmm0)) + (_ Unit (emit (MInst.XmmUnaryRmR (sse_mov_op ty) + mask + mask2)))) (xmm_rm_r ty (sse_blend_op ty) src2 src1))) ;; Helper for creating `blendvpd` instructions. -(decl blendvpd (Reg RegMem Reg) Reg) +(decl blendvpd (Xmm XmmMem Xmm) Xmm) (rule (blendvpd src1 src2 mask) ;; Move the mask into `xmm0`, as `blendvpd` implicitly operates on that ;; register. (This kind of thing would normally happen inside of ;; `Inst::mov_mitosis`, but has to happen here, where we still have the ;; mask register, because the mask is implicit and doesn't appear in the ;; `Inst` itself.) - (let ((mask2 WritableReg (xmm0)) - (_ Unit (emit (MInst.XmmUnaryRmR (SseOpcode.Movapd) (RegMem.Reg mask) mask2)))) + (let ((mask2 WritableXmm (xmm0)) + (_ Unit (emit (MInst.XmmUnaryRmR (SseOpcode.Movapd) + (xmm_to_xmm_mem mask) + mask2)))) (xmm_rm_r $F64X2 (SseOpcode.Blendvpd) src1 src2))) ;; Helper for creating `movsd` instructions. -(decl movsd (Reg RegMem) Reg) +(decl movsd (Xmm XmmMem) Xmm) (rule (movsd src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Movsd) src1 src2)) ;; Helper for creating `movlhps` instructions. -(decl movlhps (Reg RegMem) Reg) +(decl movlhps (Xmm XmmMem) Xmm) (rule (movlhps src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Movlhps) src1 src2)) ;; Helper for creating `pmaxsb` instructions. -(decl pmaxsb (Reg RegMem) Reg) +(decl pmaxsb (Xmm XmmMem) Xmm) (rule (pmaxsb src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Pmaxsb) src1 src2)) ;; Helper for creating `pmaxsw` instructions. -(decl pmaxsw (Reg RegMem) Reg) +(decl pmaxsw (Xmm XmmMem) Xmm) (rule (pmaxsw src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Pmaxsw) src1 src2)) ;; Helper for creating `pmaxsd` instructions. -(decl pmaxsd (Reg RegMem) Reg) +(decl pmaxsd (Xmm XmmMem) Xmm) (rule (pmaxsd src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Pmaxsd) src1 src2)) ;; Helper for creating `pminsb` instructions. -(decl pminsb (Reg RegMem) Reg) +(decl pminsb (Xmm XmmMem) Xmm) (rule (pminsb src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Pminsb) src1 src2)) ;; Helper for creating `pminsw` instructions. -(decl pminsw (Reg RegMem) Reg) +(decl pminsw (Xmm XmmMem) Xmm) (rule (pminsw src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Pminsw) src1 src2)) ;; Helper for creating `pminsd` instructions. -(decl pminsd (Reg RegMem) Reg) +(decl pminsd (Xmm XmmMem) Xmm) (rule (pminsd src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Pminsd) src1 src2)) ;; Helper for creating `pmaxub` instructions. -(decl pmaxub (Reg RegMem) Reg) +(decl pmaxub (Xmm XmmMem) Xmm) (rule (pmaxub src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Pmaxub) src1 src2)) ;; Helper for creating `pmaxuw` instructions. -(decl pmaxuw (Reg RegMem) Reg) +(decl pmaxuw (Xmm XmmMem) Xmm) (rule (pmaxuw src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Pmaxuw) src1 src2)) ;; Helper for creating `pmaxud` instructions. -(decl pmaxud (Reg RegMem) Reg) +(decl pmaxud (Xmm XmmMem) Xmm) (rule (pmaxud src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Pmaxud) src1 src2)) ;; Helper for creating `pminub` instructions. -(decl pminub (Reg RegMem) Reg) +(decl pminub (Xmm XmmMem) Xmm) (rule (pminub src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Pminub) src1 src2)) ;; Helper for creating `pminuw` instructions. -(decl pminuw (Reg RegMem) Reg) +(decl pminuw (Xmm XmmMem) Xmm) (rule (pminuw src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Pminuw) src1 src2)) ;; Helper for creating `pminud` instructions. -(decl pminud (Reg RegMem) Reg) +(decl pminud (Xmm XmmMem) Xmm) (rule (pminud src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Pminud) src1 src2)) ;; Helper for creating `punpcklbw` instructions. -(decl punpcklbw (Reg RegMem) Reg) +(decl punpcklbw (Xmm XmmMem) Xmm) (rule (punpcklbw src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Punpcklbw) src1 src2)) ;; Helper for creating `punpckhbw` instructions. -(decl punpckhbw (Reg RegMem) Reg) +(decl punpckhbw (Xmm XmmMem) Xmm) (rule (punpckhbw src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Punpckhbw) src1 src2)) ;; Helper for creating `packsswb` instructions. -(decl packsswb (Reg RegMem) Reg) +(decl packsswb (Xmm XmmMem) Xmm) (rule (packsswb src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Packsswb) src1 src2)) ;; Helper for creating `MInst.XmmRmRImm` instructions. -(decl xmm_rm_r_imm (SseOpcode Reg RegMem u8 OperandSize) Reg) +(decl xmm_rm_r_imm (SseOpcode Reg RegMem u8 OperandSize) Xmm) (rule (xmm_rm_r_imm op src1 src2 imm size) - (let ((dst WritableReg (temp_writable_reg $I8X16)) + (let ((dst WritableXmm (temp_writable_xmm)) (_ Unit (emit (MInst.XmmRmRImm op src1 src2 - dst + (writable_xmm_to_reg dst) imm size)))) - (writable_reg_to_reg dst))) + (writable_xmm_to_xmm dst))) ;; Helper for creating `palignr` instructions. -(decl palignr (Reg RegMem u8 OperandSize) Reg) +(decl palignr (Xmm XmmMem u8 OperandSize) Xmm) (rule (palignr src1 src2 imm size) (xmm_rm_r_imm (SseOpcode.Palignr) - src1 - src2 + (xmm_to_reg src1) + (xmm_mem_to_reg_mem src2) imm size)) +;; Helper for creating `cmpps` instructions. +(decl cmpps (Xmm XmmMem FcmpImm) Xmm) +(rule (cmpps src1 src2 imm) + (xmm_rm_r_imm (SseOpcode.Cmpps) + (xmm_to_reg src1) + (xmm_mem_to_reg_mem src2) + (encode_fcmp_imm imm) + (OperandSize.Size32))) + +;; Helper for creating `pinsrb` instructions. +(decl pinsrb (Xmm GprMem u8) Xmm) +(rule (pinsrb src1 src2 lane) + (xmm_rm_r_imm (SseOpcode.Pinsrb) + (xmm_to_reg src1) + (gpr_mem_to_reg_mem src2) + lane + (OperandSize.Size32))) + +;; Helper for creating `pinsrw` instructions. +(decl pinsrw (Xmm GprMem u8) Xmm) +(rule (pinsrw src1 src2 lane) + (xmm_rm_r_imm (SseOpcode.Pinsrw) + (xmm_to_reg src1) + (gpr_mem_to_reg_mem src2) + lane + (OperandSize.Size32))) + +;; Helper for creating `pinsrd` instructions. +(decl pinsrd (Xmm GprMem u8 OperandSize) Xmm) +(rule (pinsrd src1 src2 lane size) + (xmm_rm_r_imm (SseOpcode.Pinsrd) + (xmm_to_reg src1) + (gpr_mem_to_reg_mem src2) + lane + size)) + +;; Helper for creating `insertps` instructions. +(decl insertps (Xmm XmmMem u8) Xmm) +(rule (insertps src1 src2 lane) + (xmm_rm_r_imm (SseOpcode.Insertps) + (xmm_to_reg src1) + (xmm_mem_to_reg_mem src2) + lane + (OperandSize.Size32))) + ;; Helper for creating `pshufd` instructions. -(decl pshufd (RegMem u8 OperandSize) Reg) +(decl pshufd (XmmMem u8 OperandSize) Xmm) (rule (pshufd src imm size) - (let ((w_dst WritableReg (temp_writable_reg $I8X16)) - (dst Reg (writable_reg_to_reg w_dst)) + (let ((w_dst WritableXmm (temp_writable_xmm)) + (dst Xmm (writable_xmm_to_xmm w_dst)) (_ Unit (emit (MInst.XmmRmRImm (SseOpcode.Pshufd) - dst - src - w_dst + (xmm_to_reg dst) + (xmm_mem_to_reg_mem src) + (writable_xmm_to_reg w_dst) imm size)))) dst)) ;; Helper for creating `MInst.XmmUnaryRmR` instructions. -(decl xmm_unary_rm_r (SseOpcode RegMem) Reg) +(decl xmm_unary_rm_r (SseOpcode XmmMem) Xmm) (rule (xmm_unary_rm_r op src) - (let ((dst WritableReg (temp_writable_reg $I8X16)) + (let ((dst WritableXmm (temp_writable_xmm)) (_ Unit (emit (MInst.XmmUnaryRmR op src dst)))) - (writable_reg_to_reg dst))) + (writable_xmm_to_xmm dst))) ;; Helper for creating `pmovsxbw` instructions. -(decl pmovsxbw (RegMem) Reg) +(decl pmovsxbw (XmmMem) Xmm) (rule (pmovsxbw src) (xmm_unary_rm_r (SseOpcode.Pmovsxbw) src)) ;; Helper for creating `pmovzxbw` instructions. -(decl pmovzxbw (RegMem) Reg) +(decl pmovzxbw (XmmMem) Xmm) (rule (pmovzxbw src) (xmm_unary_rm_r (SseOpcode.Pmovzxbw) src)) ;; Helper for creating `pabsb` instructions. -(decl pabsb (RegMem) Reg) +(decl pabsb (XmmMem) Xmm) (rule (pabsb src) (xmm_unary_rm_r (SseOpcode.Pabsb) src)) ;; Helper for creating `pabsw` instructions. -(decl pabsw (RegMem) Reg) +(decl pabsw (XmmMem) Xmm) (rule (pabsw src) (xmm_unary_rm_r (SseOpcode.Pabsw) src)) ;; Helper for creating `pabsd` instructions. -(decl pabsd (RegMem) Reg) +(decl pabsd (XmmMem) Xmm) (rule (pabsd src) (xmm_unary_rm_r (SseOpcode.Pabsd) src)) ;; Helper for creating `MInst.XmmUnaryRmREvex` instructions. -(decl xmm_unary_rm_r_evex (Avx512Opcode RegMem) Reg) +(decl xmm_unary_rm_r_evex (Avx512Opcode XmmMem) Xmm) (rule (xmm_unary_rm_r_evex op src) - (let ((dst WritableReg (temp_writable_reg $I8X16)) + (let ((dst WritableXmm (temp_writable_xmm)) (_ Unit (emit (MInst.XmmUnaryRmREvex op src dst)))) - (writable_reg_to_reg dst))) + (writable_xmm_to_xmm dst))) ;; Helper for creating `vpabsq` instructions. -(decl vpabsq (RegMem) Reg) +(decl vpabsq (XmmMem) Xmm) (rule (vpabsq src) (xmm_unary_rm_r_evex (Avx512Opcode.Vpabsq) src)) ;; Helper for creating `MInst.XmmRmREvex` instructions. -(decl xmm_rm_r_evex (Avx512Opcode RegMem Reg) Reg) +(decl xmm_rm_r_evex (Avx512Opcode XmmMem Xmm) Xmm) (rule (xmm_rm_r_evex op src1 src2) - (let ((dst WritableReg (temp_writable_reg $I8X16)) + (let ((dst WritableXmm (temp_writable_xmm)) (_ Unit (emit (MInst.XmmRmREvex op src1 src2 dst)))) - (writable_reg_to_reg dst))) + (writable_xmm_to_xmm dst))) ;; Helper for creating `vpmullq` instructions. ;; ;; Requires AVX-512 vl and dq. -(decl vpmullq (RegMem Reg) Reg) +(decl vpmullq (XmmMem Xmm) Xmm) (rule (vpmullq src1 src2) (xmm_rm_r_evex (Avx512Opcode.Vpmullq) src1 src2)) -;; Helper for creating `MInst.XmmRmiReg` instructions. -(decl xmm_rmi_reg (SseOpcode Reg RegMemImm) Reg) -(rule (xmm_rmi_reg op src1 src2) - (let ((dst WritableReg (temp_writable_reg $I8X16)) +;; Helper for creating `MInst.MulHi` instructions. +;; +;; Returns the (lo, hi) register halves of the multiplication. +(decl mul_hi (Type bool Reg RegMem) ValueRegs) +(rule (mul_hi ty signed src1 src2) + (let ((dst_lo WritableReg (temp_writable_reg ty)) + (dst_hi WritableReg (temp_writable_reg ty)) + (size OperandSize (operand_size_of_type_32_64 ty)) + (_ Unit (emit (MInst.MulHi size + signed + src1 + src2 + dst_lo + dst_hi)))) + (value_regs (writable_reg_to_reg dst_lo) + (writable_reg_to_reg dst_hi)))) + +;; Helper for creating `mul` instructions that return both the lower and +;; (unsigned) higher halves of the result. +(decl mulhi_u (Type Reg RegMem) ValueRegs) +(rule (mulhi_u ty src1 src2) + (mul_hi ty $false src1 src2)) + +;; Helper for creating `MInst.XmmRmiXmm` instructions. +(decl xmm_rmi_xmm (SseOpcode Xmm XmmMemImm) Xmm) +(rule (xmm_rmi_xmm op src1 src2) + (let ((dst WritableXmm (temp_writable_xmm)) (_ Unit (emit (MInst.XmmRmiReg op src1 src2 dst)))) - (writable_reg_to_reg dst))) + (writable_xmm_to_xmm dst))) ;; Helper for creating `psllw` instructions. -(decl psllw (Reg RegMemImm) Reg) +(decl psllw (Xmm XmmMemImm) Xmm) (rule (psllw src1 src2) - (xmm_rmi_reg (SseOpcode.Psllw) src1 src2)) + (xmm_rmi_xmm (SseOpcode.Psllw) src1 src2)) ;; Helper for creating `pslld` instructions. -(decl pslld (Reg RegMemImm) Reg) +(decl pslld (Xmm XmmMemImm) Xmm) (rule (pslld src1 src2) - (xmm_rmi_reg (SseOpcode.Pslld) src1 src2)) + (xmm_rmi_xmm (SseOpcode.Pslld) src1 src2)) ;; Helper for creating `psllq` instructions. -(decl psllq (Reg RegMemImm) Reg) +(decl psllq (Xmm XmmMemImm) Xmm) (rule (psllq src1 src2) - (xmm_rmi_reg (SseOpcode.Psllq) src1 src2)) + (xmm_rmi_xmm (SseOpcode.Psllq) src1 src2)) ;; Helper for creating `psrlw` instructions. -(decl psrlw (Reg RegMemImm) Reg) +(decl psrlw (Xmm XmmMemImm) Xmm) (rule (psrlw src1 src2) - (xmm_rmi_reg (SseOpcode.Psrlw) src1 src2)) + (xmm_rmi_xmm (SseOpcode.Psrlw) src1 src2)) ;; Helper for creating `psrld` instructions. -(decl psrld (Reg RegMemImm) Reg) +(decl psrld (Xmm XmmMemImm) Xmm) (rule (psrld src1 src2) - (xmm_rmi_reg (SseOpcode.Psrld) src1 src2)) + (xmm_rmi_xmm (SseOpcode.Psrld) src1 src2)) ;; Helper for creating `psrlq` instructions. -(decl psrlq (Reg RegMemImm) Reg) +(decl psrlq (Xmm XmmMemImm) Xmm) (rule (psrlq src1 src2) - (xmm_rmi_reg (SseOpcode.Psrlq) src1 src2)) + (xmm_rmi_xmm (SseOpcode.Psrlq) src1 src2)) ;; Helper for creating `psraw` instructions. -(decl psraw (Reg RegMemImm) Reg) +(decl psraw (Xmm XmmMemImm) Xmm) (rule (psraw src1 src2) - (xmm_rmi_reg (SseOpcode.Psraw) src1 src2)) + (xmm_rmi_xmm (SseOpcode.Psraw) src1 src2)) ;; Helper for creating `psrad` instructions. -(decl psrad (Reg RegMemImm) Reg) +(decl psrad (Xmm XmmMemImm) Xmm) (rule (psrad src1 src2) - (xmm_rmi_reg (SseOpcode.Psrad) src1 src2)) - -;; Helper for creating `MInst.MulHi` instructions. -;; -;; Returns the (lo, hi) register halves of the multiplication. -(decl mul_hi (Type bool Reg RegMem) ValueRegs) -(rule (mul_hi ty signed src1 src2) - (let ((dst_lo WritableReg (temp_writable_reg ty)) - (dst_hi WritableReg (temp_writable_reg ty)) - (size OperandSize (operand_size_of_type_32_64 ty)) - (_ Unit (emit (MInst.MulHi size - signed - src1 - src2 - dst_lo - dst_hi)))) - (value_regs (writable_reg_to_reg dst_lo) - (writable_reg_to_reg dst_hi)))) + (xmm_rmi_xmm (SseOpcode.Psrad) src1 src2)) -;; Helper for creating `mul` instructions that return both the lower and -;; (unsigned) higher halves of the result. -(decl mulhi_u (Type Reg RegMem) ValueRegs) -(rule (mulhi_u ty src1 src2) - (mul_hi ty $false src1 src2)) - -;; Helper for creating `cmpps` instructions. -(decl cmpps (Reg RegMem FcmpImm) Reg) -(rule (cmpps src1 src2 imm) - (xmm_rm_r_imm (SseOpcode.Cmpps) - src1 - src2 - (encode_fcmp_imm imm) - (OperandSize.Size32))) +;; Helper for creating `pextrd` instructions. +(decl pextrd (Type Xmm u8) Gpr) +(rule (pextrd ty src lane) + (let ((w_dst WritableGpr (temp_writable_gpr)) + (r_dst Gpr (writable_gpr_to_gpr w_dst)) + (_ Unit (emit (MInst.XmmRmRImm (SseOpcode.Pextrd) + (gpr_to_reg r_dst) + (RegMem.Reg (xmm_to_reg src)) + (writable_gpr_to_reg w_dst) + lane + (operand_size_of_type_32_64 (lane_type ty)))))) + r_dst)) ;; Helper for creating `cmppd` instructions. ;; ;; Note that `Size32` is intentional despite this being used for 64-bit ;; operations, since this presumably induces the correct encoding of the ;; instruction. -(decl cmppd (Reg RegMem FcmpImm) Reg) +(decl cmppd (Xmm XmmMem FcmpImm) Xmm) (rule (cmppd src1 src2 imm) (xmm_rm_r_imm (SseOpcode.Cmppd) - src1 - src2 + (xmm_to_reg src1) + (xmm_mem_to_reg_mem src2) (encode_fcmp_imm imm) (OperandSize.Size32))) ;; Helper for creating `MInst.GprToXmm` instructions. -(decl gpr_to_xmm (Type SseOpcode RegMem OperandSize) Reg) -(rule (gpr_to_xmm ty op src size) - (let ((dst WritableReg (temp_writable_reg ty)) - (_ Unit (emit (MInst.GprToXmm op src dst size)))) - (writable_reg_to_reg dst))) - -;; Helper for creating `pinsrb` instructions. -(decl pinsrb (Reg RegMem u8) Reg) -(rule (pinsrb src1 src2 lane) - (xmm_rm_r_imm (SseOpcode.Pinsrb) src1 src2 lane (OperandSize.Size32))) - -;; Helper for creating `pinsrw` instructions. -(decl pinsrw (Reg RegMem u8) Reg) -(rule (pinsrw src1 src2 lane) - (xmm_rm_r_imm (SseOpcode.Pinsrw) src1 src2 lane (OperandSize.Size32))) - -;; Helper for creating `pinsrd` instructions. -(decl pinsrd (Reg RegMem u8 OperandSize) Reg) -(rule (pinsrd src1 src2 lane size) - (xmm_rm_r_imm (SseOpcode.Pinsrd) src1 src2 lane size)) - -;; Helper for creating `insertps` instructions. -(decl insertps (Reg RegMem u8) Reg) -(rule (insertps src1 src2 lane) - (xmm_rm_r_imm (SseOpcode.Insertps) src1 src2 lane (OperandSize.Size32))) - -;; Helper for creating `pextrd` instructions. -(decl pextrd (Type Reg u8) Reg) -(rule (pextrd ty src lane) - (let ((w_dst WritableReg (temp_writable_reg ty)) - (r_dst Reg (writable_reg_to_reg w_dst)) - (_ Unit (emit (MInst.XmmRmRImm (SseOpcode.Pextrd) - r_dst - (RegMem.Reg src) - w_dst - lane - (operand_size_of_type_32_64 (lane_type ty)))))) - r_dst)) +(decl gpr_to_xmm (SseOpcode GprMem OperandSize) Xmm) +(rule (gpr_to_xmm op src size) + (let ((dst WritableXmm (temp_writable_xmm)) + (_ Unit (emit (MInst.GprToXmm op (gpr_mem_to_reg_mem src) dst size)))) + (writable_xmm_to_xmm dst))) ;; Helper for creating `not` instructions. -(decl not (Type Reg) Reg) +(decl not (Type Gpr) Gpr) (rule (not ty src) - (let ((dst WritableReg (temp_writable_reg ty)) + (let ((dst WritableGpr (temp_writable_gpr)) (size OperandSize (operand_size_of_type_32_64 ty)) (_ Unit (emit (MInst.Not size src dst)))) - (writable_reg_to_reg dst))) + (writable_gpr_to_gpr dst))) ;; Helper for creating `neg` instructions. -(decl neg (Type Reg) Reg) +(decl neg (Type Gpr) Gpr) (rule (neg ty src) - (let ((dst WritableReg (temp_writable_reg ty)) + (let ((dst WritableGpr (temp_writable_gpr)) (size OperandSize (operand_size_of_type_32_64 ty)) (_ Unit (emit (MInst.Neg size src dst)))) - (writable_reg_to_reg dst))) + (writable_gpr_to_gpr dst))) -(decl lea (SyntheticAmode) Reg) +(decl lea (SyntheticAmode) Gpr) (rule (lea addr) - (let ((dst WritableReg (temp_writable_reg $I64)) + (let ((dst WritableGpr (temp_writable_gpr)) (_ Unit (emit (MInst.LoadEffectiveAddress addr dst)))) - (writable_reg_to_reg dst))) + (writable_gpr_to_gpr dst))) ;; Helper for creating `ud2` instructions. (decl ud2 (TrapCode) SideEffectNoResult) diff --git a/cranelift/codegen/src/isa/x64/inst/args.rs b/cranelift/codegen/src/isa/x64/inst/args.rs index 9b9cf2fd0ca9..e27dc3130ec6 100644 --- a/cranelift/codegen/src/isa/x64/inst/args.rs +++ b/cranelift/codegen/src/isa/x64/inst/args.rs @@ -13,6 +13,309 @@ use smallvec::{smallvec, SmallVec}; use std::fmt; use std::string::String; +/// An extenstion trait for converting `Writable{Xmm,Gpr}` to `Writable`. +pub trait ToWritableReg { + fn to_writable_reg(&self) -> Writable; +} + +/// An extension trait for converting `Writable` to `Writable{Xmm,Gpr}`. +pub trait FromWritableReg: Sized { + fn from_writable_reg(w: Writable) -> Option; +} + +/// An extension trait for mapping register uses on `{Xmm,Gpr}`. +pub trait MapUseExt { + fn map_use(&mut self, mapper: &RM) + where + RM: RegMapper; +} + +/// An extension trait for mapping register mods and defs on +/// `Writable{Xmm,Gpr}`. +pub trait MapDefModExt { + fn map_def(&mut self, mapper: &RM) + where + RM: RegMapper; + + fn map_mod(&mut self, mapper: &RM) + where + RM: RegMapper; +} + +/// A macro for defining a newtype of `Reg` that enforces some invariant about +/// the wrapped `Reg` (such as that it is of a particular register class). +macro_rules! newtype_of_reg { + ( + $newtype_reg:ident, + $newtype_writable_reg:ident, + $newtype_reg_mem:ident, + $newtype_reg_mem_imm:ident, + |$check_reg:ident| $check:expr + ) => { + /// A newtype wrapper around `Reg`. + #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] + pub struct $newtype_reg(Reg); + + impl PartialEq for $newtype_reg { + fn eq(&self, other: &Reg) -> bool { + self.0 == *other + } + } + + impl From<$newtype_reg> for Reg { + fn from(r: $newtype_reg) -> Self { + r.0 + } + } + + impl PrettyPrint for $newtype_reg { + fn show_rru(&self, mb_rru: Option<&RealRegUniverse>) -> String { + self.0.show_rru(mb_rru) + } + } + + impl $newtype_reg { + /// Create this newtype from the given register, or return `None` if the register + /// is not a valid instance of this newtype. + pub fn new($check_reg: Reg) -> Option { + if $check { + Some(Self($check_reg)) + } else { + None + } + } + + /// Get this newtype's underlying `Reg`. + pub fn to_reg(self) -> Reg { + self.0 + } + } + + // Convenience impl so that people working with this newtype can use it + // "just like" a plain `Reg`. + // + // NB: We cannot implement `DerefMut` because that would let people do + // nasty stuff like `*my_gpr.deref_mut() = some_xmm_reg`, breaking the + // invariants that `Gpr` provides. + impl std::ops::Deref for $newtype_reg { + type Target = Reg; + + fn deref(&self) -> &Reg { + &self.0 + } + } + + impl MapUseExt for $newtype_reg { + fn map_use(&mut self, mapper: &RM) + where + RM: RegMapper, + { + let mut reg = self.0; + mapper.map_use(&mut reg); + debug_assert!({ + let $check_reg = reg; + $check + }); + *self = $newtype_reg(reg); + } + } + + pub type $newtype_writable_reg = Writable<$newtype_reg>; + + impl ToWritableReg for $newtype_writable_reg { + fn to_writable_reg(&self) -> Writable { + Writable::from_reg(self.to_reg().to_reg()) + } + } + + impl FromWritableReg for $newtype_writable_reg { + fn from_writable_reg(w: Writable) -> Option { + Some(Writable::from_reg($newtype_reg::new(w.to_reg())?)) + } + } + + impl MapDefModExt for $newtype_writable_reg { + fn map_def(&mut self, mapper: &RM) + where + RM: RegMapper, + { + let mut reg = self.to_writable_reg(); + mapper.map_def(&mut reg); + debug_assert!({ + let $check_reg = reg.to_reg(); + $check + }); + *self = Writable::from_reg($newtype_reg(reg.to_reg())); + } + + fn map_mod(&mut self, mapper: &RM) + where + RM: RegMapper, + { + let mut reg = self.to_writable_reg(); + mapper.map_mod(&mut reg); + debug_assert!({ + let $check_reg = reg.to_reg(); + $check + }); + *self = Writable::from_reg($newtype_reg(reg.to_reg())); + } + } + + /// A newtype wrapper around `RegMem` for general-purpose registers. + #[derive(Clone, Debug)] + pub struct $newtype_reg_mem(RegMem); + + impl From<$newtype_reg_mem> for RegMem { + fn from(rm: $newtype_reg_mem) -> Self { + rm.0 + } + } + + impl From<$newtype_reg> for $newtype_reg_mem { + fn from(r: $newtype_reg) -> Self { + $newtype_reg_mem(RegMem::reg(r.into())) + } + } + + impl $newtype_reg_mem { + /// Construct a `RegMem` newtype from the given `RegMem`, or return + /// `None` if the `RegMem` is not a valid instance of this `RegMem` + /// newtype. + pub fn new(rm: RegMem) -> Option { + match rm { + RegMem::Mem { addr: _ } => Some(Self(rm)), + RegMem::Reg { reg: $check_reg } if $check => Some(Self(rm)), + RegMem::Reg { reg: _ } => None, + } + } + + /// Convert this newtype into its underlying `RegMem`. + pub fn to_reg_mem(self) -> RegMem { + self.0 + } + + #[allow(dead_code)] // Used by some newtypes and not others. + pub fn map_uses(&mut self, mapper: &RM) + where + RM: RegMapper, + { + self.0.map_uses(mapper); + debug_assert!(match self.0 { + RegMem::Reg { reg: $check_reg } => $check, + _ => true, + }); + } + + #[allow(dead_code)] // Used by some newtypes and not others. + pub fn map_as_def(&mut self, mapper: &RM) + where + RM: RegMapper, + { + self.0.map_as_def(mapper); + debug_assert!(match self.0 { + RegMem::Reg { reg: $check_reg } => $check, + _ => true, + }); + } + } + + impl PrettyPrint for $newtype_reg_mem { + fn show_rru(&self, mb_rru: Option<&RealRegUniverse>) -> String { + self.0.show_rru(mb_rru) + } + } + + impl PrettyPrintSized for $newtype_reg_mem { + fn show_rru_sized(&self, mb_rru: Option<&RealRegUniverse>, size: u8) -> String { + self.0.show_rru_sized(mb_rru, size) + } + } + + /// A newtype wrapper around `RegMemImm`. + #[derive(Clone, Debug)] + pub struct $newtype_reg_mem_imm(RegMemImm); + + impl From<$newtype_reg_mem_imm> for RegMemImm { + fn from(rmi: $newtype_reg_mem_imm) -> RegMemImm { + rmi.0 + } + } + + impl From<$newtype_reg> for $newtype_reg_mem_imm { + fn from(r: $newtype_reg) -> Self { + $newtype_reg_mem_imm(RegMemImm::reg(r.into())) + } + } + + impl $newtype_reg_mem_imm { + /// Construct this newtype from the given `RegMemImm`, or return + /// `None` if the `RegMemImm` is not a valid instance of this + /// newtype. + pub fn new(rmi: RegMemImm) -> Option { + match rmi { + RegMemImm::Imm { .. } => Some(Self(rmi)), + RegMemImm::Mem { addr: _ } => Some(Self(rmi)), + RegMemImm::Reg { reg: $check_reg } if $check => Some(Self(rmi)), + RegMemImm::Reg { reg: _ } => None, + } + } + + /// Convert this newtype into its underlying `RegMemImm`. + #[allow(dead_code)] // Used by some newtypes and not others. + pub fn to_reg_mem_imm(self) -> RegMemImm { + self.0 + } + + #[allow(dead_code)] // Used by some newtypes and not others. + pub fn map_uses(&mut self, mapper: &RM) + where + RM: RegMapper, + { + self.0.map_uses(mapper); + debug_assert!(match self.0 { + RegMemImm::Reg { reg: $check_reg } => $check, + _ => true, + }); + } + + #[allow(dead_code)] // Used by some newtypes and not others. + pub fn map_as_def(&mut self, mapper: &RM) + where + RM: RegMapper, + { + self.0.map_as_def(mapper); + debug_assert!(match self.0 { + RegMemImm::Reg { reg: $check_reg } => $check, + _ => true, + }); + } + } + + impl PrettyPrint for $newtype_reg_mem_imm { + fn show_rru(&self, mb_rru: Option<&RealRegUniverse>) -> String { + self.0.show_rru(mb_rru) + } + } + + impl PrettyPrintSized for $newtype_reg_mem_imm { + fn show_rru_sized(&self, mb_rru: Option<&RealRegUniverse>, size: u8) -> String { + self.0.show_rru_sized(mb_rru, size) + } + } + }; +} + +// Define a newtype of `Reg` for general-purpose registers. +newtype_of_reg!(Gpr, WritableGpr, GprMem, GprMemImm, |reg| { + reg.get_class() == RegClass::I64 +}); + +// Define a newtype of `Reg` for XMM registers. +newtype_of_reg!(Xmm, WritableXmm, XmmMem, XmmMemImm, |reg| { + reg.get_class() == RegClass::V128 +}); + /// A possible addressing mode (amode) that can be used in instructions. /// These denote a 64-bit value only. #[derive(Clone, Debug)] @@ -27,8 +330,8 @@ pub enum Amode { /// sign-extend-32-to-64(Immediate) + Register1 + (Register2 << Shift) ImmRegRegShift { simm32: u32, - base: Reg, - index: Reg, + base: Gpr, + index: Gpr, shift: u8, /* 0 .. 3 only */ flags: MemFlags, }, @@ -48,7 +351,7 @@ impl Amode { } } - pub(crate) fn imm_reg_reg_shift(simm32: u32, base: Reg, index: Reg, shift: u8) -> Self { + pub(crate) fn imm_reg_reg_shift(simm32: u32, base: Gpr, index: Gpr, shift: u8) -> Self { debug_assert!(base.get_class() == RegClass::I64); debug_assert!(index.get_class() == RegClass::I64); debug_assert!(shift <= 3); @@ -96,8 +399,8 @@ impl Amode { collector.add_use(*base); } Amode::ImmRegRegShift { base, index, .. } => { - collector.add_use(*base); - collector.add_use(*index); + collector.add_use(base.to_reg()); + collector.add_use(index.to_reg()); } Amode::RipRelative { .. } => { // RIP isn't involved in regalloc. @@ -225,7 +528,7 @@ impl PrettyPrint for SyntheticAmode { /// denote an 8, 16, 32 or 64 bit value. For the Immediate form, in the 8- and 16-bit case, only /// the lower 8 or 16 bits of `simm32` is relevant. In the 64-bit case, the value denoted by /// `simm32` is its sign-extension out to 64 bits. -#[derive(Clone)] +#[derive(Clone, Debug)] pub enum RegMemImm { Reg { reg: Reg }, Mem { addr: SyntheticAmode }, diff --git a/cranelift/codegen/src/isa/x64/inst/emit.rs b/cranelift/codegen/src/isa/x64/inst/emit.rs index c4dc3f3d7a5d..bda2ddc0a241 100644 --- a/cranelift/codegen/src/isa/x64/inst/emit.rs +++ b/cranelift/codegen/src/isa/x64/inst/emit.rs @@ -298,14 +298,14 @@ pub(crate) fn emit( Popcnt => (0x0fb8, 2), }; - match src { + match src.clone().into() { RegMem::Reg { reg: src } => emit_std_reg_reg( sink, prefix, opcode, num_opcodes, - dst.to_reg(), - *src, + dst.to_reg().to_reg(), + src, rex_flags, ), RegMem::Mem { addr: src } => { @@ -317,7 +317,7 @@ pub(crate) fn emit( prefix, opcode, num_opcodes, - dst.to_reg(), + dst.to_reg().to_reg(), &amode, rex_flags, ); @@ -327,7 +327,7 @@ pub(crate) fn emit( Inst::Not { size, src, dst } => { debug_assert_eq!(*src, dst.to_reg()); - let rex_flags = RexFlags::from((*size, dst.to_reg())); + let rex_flags = RexFlags::from((*size, dst.to_writable_reg().to_reg())); let (opcode, prefix) = match size { OperandSize::Size8 => (0xF6, LegacyPrefixes::None), OperandSize::Size16 => (0xF7, LegacyPrefixes::_66), @@ -342,7 +342,7 @@ pub(crate) fn emit( Inst::Neg { size, src, dst } => { debug_assert_eq!(*src, dst.to_reg()); - let rex_flags = RexFlags::from((*size, dst.to_reg())); + let rex_flags = RexFlags::from((*size, dst.to_writable_reg().to_reg())); let (opcode, prefix) = match size { OperandSize::Size8 => (0xF6, LegacyPrefixes::None), OperandSize::Size16 => (0xF7, LegacyPrefixes::_66), @@ -728,7 +728,7 @@ pub(crate) fn emit( LegacyPrefixes::None, 0x8D, 1, - dst.to_reg(), + dst.to_reg().to_reg(), &amode, RexFlags::set_w(), ); @@ -884,6 +884,7 @@ pub(crate) fn emit( debug_assert_eq!(*src1, dst.to_reg()); let rex = RexFlags::clear_w(); let prefix = LegacyPrefixes::_66; + let src2 = src2.clone().to_reg_mem_imm(); if let RegMemImm::Imm { simm32 } = src2 { let (opcode_bytes, reg_digit) = match opcode { SseOpcode::Psllw => (0x0F71, 6), @@ -898,7 +899,7 @@ pub(crate) fn emit( }; let dst_enc = reg_enc(dst.to_reg()); emit_std_enc_enc(sink, prefix, opcode_bytes, 2, reg_digit, dst_enc, rex); - let imm = (*simm32) + let imm = (simm32) .try_into() .expect("the immediate must be convertible to a u8"); sink.put1(imm); @@ -917,7 +918,15 @@ pub(crate) fn emit( match src2 { RegMemImm::Reg { reg } => { - emit_std_reg_reg(sink, prefix, opcode_bytes, 2, dst.to_reg(), *reg, rex); + emit_std_reg_reg( + sink, + prefix, + opcode_bytes, + 2, + dst.to_reg().to_reg(), + reg, + rex, + ); } RegMemImm::Mem { addr } => { let addr = &addr.finalize(state, sink); @@ -928,7 +937,7 @@ pub(crate) fn emit( prefix, opcode_bytes, 2, - dst.to_reg(), + dst.to_reg().to_reg(), addr, rex, ); @@ -1335,7 +1344,12 @@ pub(crate) fn emit( // might be negative; use a sign-extension. let inst = Inst::movsx_rm_r( ExtMode::LQ, - RegMem::mem(Amode::imm_reg_reg_shift(0, tmp1.to_reg(), tmp2.to_reg(), 2)), + RegMem::mem(Amode::imm_reg_reg_shift( + 0, + Gpr::new(tmp1.to_reg()).unwrap(), + Gpr::new(tmp2.to_reg()).unwrap(), + 2, + )), *tmp2, ); inst.emit(sink, info, state); @@ -1424,15 +1438,15 @@ pub(crate) fn emit( _ => unimplemented!("Opcode {:?} not implemented", op), }; - match src_e { + match src_e.clone().to_reg_mem() { RegMem::Reg { reg: reg_e } => { emit_std_reg_reg( sink, prefix, opcode, num_opcodes, - reg_g.to_reg(), - *reg_e, + reg_g.to_reg().to_reg(), + reg_e, rex, ); } @@ -1445,7 +1459,7 @@ pub(crate) fn emit( prefix, opcode, num_opcodes, - reg_g.to_reg(), + reg_g.to_reg().to_reg(), addr, rex, ); @@ -1460,7 +1474,7 @@ pub(crate) fn emit( Avx512Opcode::Vpopcntb => (LegacyPrefixes::_66, OpcodeMap::_0F38, false, 0x54), _ => unimplemented!("Opcode {:?} not implemented", op), }; - match src { + match src.clone().to_reg_mem() { RegMem::Reg { reg: src } => EvexInstruction::new() .length(EvexVectorLength::V128) .prefix(prefix) @@ -1587,9 +1601,17 @@ pub(crate) fn emit( _ => unimplemented!("Opcode {:?} not implemented", op), }; - match src_e { + match src_e.clone().to_reg_mem() { RegMem::Reg { reg: reg_e } => { - emit_std_reg_reg(sink, prefix, opcode, length, reg_g.to_reg(), *reg_e, rex); + emit_std_reg_reg( + sink, + prefix, + opcode, + length, + reg_g.to_reg().to_reg(), + reg_e, + rex, + ); } RegMem::Mem { addr } => { let addr = &addr.finalize(state, sink); @@ -1600,7 +1622,7 @@ pub(crate) fn emit( prefix, opcode, length, - reg_g.to_reg(), + reg_g.to_reg().to_reg(), addr, rex, ); @@ -1619,7 +1641,7 @@ pub(crate) fn emit( Avx512Opcode::Vpmullq => (true, 0x40), _ => unimplemented!("Opcode {:?} not implemented", op), }; - match src1 { + match src1.clone().to_reg_mem() { RegMem::Reg { reg: src } => EvexInstruction::new() .length(EvexVectorLength::V128) .prefix(LegacyPrefixes::_66) @@ -1845,9 +1867,9 @@ pub(crate) fn emit( }; let rex = RexFlags::from(*dst_size); let (src, dst) = if dst_first { - (dst.to_reg(), *src) + (dst.to_reg().to_reg(), src.to_reg()) } else { - (*src, dst.to_reg()) + (src.to_reg(), dst.to_reg().to_reg()) }; emit_std_reg_reg(sink, prefix, opcode, 2, src, dst, rex); @@ -1870,7 +1892,15 @@ pub(crate) fn emit( let rex = RexFlags::from(*src_size); match src_e { RegMem::Reg { reg: reg_e } => { - emit_std_reg_reg(sink, prefix, opcode, 2, reg_g.to_reg(), *reg_e, rex); + emit_std_reg_reg( + sink, + prefix, + opcode, + 2, + reg_g.to_reg().to_reg(), + *reg_e, + rex, + ); } RegMem::Mem { addr } => { let addr = &addr.finalize(state, sink); @@ -1881,7 +1911,7 @@ pub(crate) fn emit( prefix, opcode, 2, - reg_g.to_reg(), + reg_g.to_reg().to_reg(), addr, rex, ); @@ -1950,7 +1980,11 @@ pub(crate) fn emit( // If x seen as a signed int64 is not negative, a signed-conversion will do the right // thing. // TODO use tst src, src here. - let inst = Inst::cmp_rmi_r(OperandSize::Size64, RegMemImm::imm(0), src.to_reg()); + let inst = Inst::cmp_rmi_r( + OperandSize::Size64, + RegMemImm::imm(0), + src.to_reg().to_reg(), + ); inst.emit(sink, info, state); one_way_jmp(sink, CC::L, handle_negative); @@ -1961,8 +1995,8 @@ pub(crate) fn emit( sink, info, state, - src.to_reg(), - *dst, + src.to_reg().to_reg(), + dst.to_writable_reg(), *dst_size == OperandSize::Size64, ); @@ -1973,7 +2007,11 @@ pub(crate) fn emit( // Divide x by two to get it in range for the signed conversion, keep the LSB, and // scale it back up on the FP side. - let inst = Inst::gen_move(*tmp_gpr1, src.to_reg(), types::I64); + let inst = Inst::gen_move( + tmp_gpr1.to_writable_reg(), + src.to_reg().to_reg(), + types::I64, + ); inst.emit(sink, info, state); // tmp_gpr1 := src >> 1 @@ -1981,26 +2019,30 @@ pub(crate) fn emit( OperandSize::Size64, ShiftKind::ShiftRightLogical, Some(1), - *tmp_gpr1, + tmp_gpr1.to_writable_reg(), ); inst.emit(sink, info, state); - let inst = Inst::gen_move(*tmp_gpr2, src.to_reg(), types::I64); + let inst = Inst::gen_move( + tmp_gpr2.to_writable_reg(), + src.to_reg().to_reg(), + types::I64, + ); inst.emit(sink, info, state); let inst = Inst::alu_rmi_r( OperandSize::Size64, AluRmiROpcode::And, RegMemImm::imm(1), - *tmp_gpr2, + tmp_gpr2.to_writable_reg(), ); inst.emit(sink, info, state); let inst = Inst::alu_rmi_r( OperandSize::Size64, AluRmiROpcode::Or, - RegMemImm::reg(tmp_gpr1.to_reg()), - *tmp_gpr2, + RegMemImm::reg(tmp_gpr1.to_reg().to_reg()), + tmp_gpr2.to_writable_reg(), ); inst.emit(sink, info, state); @@ -2008,8 +2050,8 @@ pub(crate) fn emit( sink, info, state, - tmp_gpr2.to_reg(), - *dst, + tmp_gpr2.to_reg().to_reg(), + dst.to_writable_reg(), *dst_size == OperandSize::Size64, ); @@ -2018,7 +2060,11 @@ pub(crate) fn emit( } else { SseOpcode::Addss }; - let inst = Inst::xmm_rm_r(add_op, RegMem::reg(dst.to_reg()), *dst); + let inst = Inst::xmm_rm_r( + add_op, + RegMem::reg(dst.to_reg().to_reg()), + dst.to_writable_reg(), + ); inst.emit(sink, info, state); sink.bind_label(done); @@ -2091,18 +2137,18 @@ pub(crate) fn emit( let not_nan = sink.get_label(); // The truncation. - let inst = Inst::xmm_to_gpr(trunc_op, src, *dst, *dst_size); + let inst = Inst::xmm_to_gpr(trunc_op, src.to_reg(), dst.to_writable_reg(), *dst_size); inst.emit(sink, info, state); // Compare against 1, in case of overflow the dst operand was INT_MIN. - let inst = Inst::cmp_rmi_r(*dst_size, RegMemImm::imm(1), dst.to_reg()); + let inst = Inst::cmp_rmi_r(*dst_size, RegMemImm::imm(1), dst.to_reg().to_reg()); inst.emit(sink, info, state); one_way_jmp(sink, CC::NO, done); // no overflow => done // Check for NaN. - let inst = Inst::xmm_cmp_rm_r(cmp_op, RegMem::reg(src), src); + let inst = Inst::xmm_cmp_rm_r(cmp_op, RegMem::reg(src.to_reg()), src.to_reg()); inst.emit(sink, info, state); one_way_jmp(sink, CC::NP, not_nan); // go to not_nan if not a NaN @@ -2112,8 +2158,8 @@ pub(crate) fn emit( let inst = Inst::alu_rmi_r( *dst_size, AluRmiROpcode::Xor, - RegMemImm::reg(dst.to_reg()), - *dst, + RegMemImm::reg(dst.to_reg().to_reg()), + dst.to_writable_reg(), ); inst.emit(sink, info, state); @@ -2125,11 +2171,18 @@ pub(crate) fn emit( // If the input was positive, saturate to INT_MAX. // Zero out tmp_xmm. - let inst = - Inst::xmm_rm_r(SseOpcode::Xorpd, RegMem::reg(tmp_xmm.to_reg()), *tmp_xmm); + let inst = Inst::xmm_rm_r( + SseOpcode::Xorpd, + RegMem::reg(tmp_xmm.to_reg().to_reg()), + tmp_xmm.to_writable_reg(), + ); inst.emit(sink, info, state); - let inst = Inst::xmm_cmp_rm_r(cmp_op, RegMem::reg(src), tmp_xmm.to_reg()); + let inst = Inst::xmm_cmp_rm_r( + cmp_op, + RegMem::reg(src.to_reg()), + tmp_xmm.to_reg().to_reg(), + ); inst.emit(sink, info, state); // Jump if >= to done. @@ -2137,10 +2190,14 @@ pub(crate) fn emit( // Otherwise, put INT_MAX. if *dst_size == OperandSize::Size64 { - let inst = Inst::imm(OperandSize::Size64, 0x7fffffffffffffff, *dst); + let inst = Inst::imm( + OperandSize::Size64, + 0x7fffffffffffffff, + dst.to_writable_reg(), + ); inst.emit(sink, info, state); } else { - let inst = Inst::imm(OperandSize::Size32, 0x7fffffff, *dst); + let inst = Inst::imm(OperandSize::Size32, 0x7fffffff, dst.to_writable_reg()); inst.emit(sink, info, state); } } else { @@ -2162,7 +2219,8 @@ pub(crate) fn emit( match *src_size { OperandSize::Size32 => { let cst = Ieee32::pow2(output_bits - 1).neg().bits(); - let inst = Inst::imm(OperandSize::Size32, cst as u64, *tmp_gpr); + let inst = + Inst::imm(OperandSize::Size32, cst as u64, tmp_gpr.to_writable_reg()); inst.emit(sink, info, state); } OperandSize::Size64 => { @@ -2174,17 +2232,26 @@ pub(crate) fn emit( } else { Ieee64::pow2(output_bits - 1).neg() }; - let inst = Inst::imm(OperandSize::Size64, cst.bits(), *tmp_gpr); + let inst = + Inst::imm(OperandSize::Size64, cst.bits(), tmp_gpr.to_writable_reg()); inst.emit(sink, info, state); } _ => unreachable!(), } - let inst = - Inst::gpr_to_xmm(cast_op, RegMem::reg(tmp_gpr.to_reg()), *src_size, *tmp_xmm); + let inst = Inst::gpr_to_xmm( + cast_op, + RegMem::reg(tmp_gpr.to_reg().to_reg()), + *src_size, + tmp_xmm.to_writable_reg(), + ); inst.emit(sink, info, state); - let inst = Inst::xmm_cmp_rm_r(cmp_op, RegMem::reg(tmp_xmm.to_reg()), src); + let inst = Inst::xmm_cmp_rm_r( + cmp_op, + RegMem::reg(tmp_xmm.to_reg().to_reg()), + src.to_reg(), + ); inst.emit(sink, info, state); // jump over trap if src >= or > threshold @@ -2198,11 +2265,18 @@ pub(crate) fn emit( sink.bind_label(check_positive); // Zero out the tmp_xmm register. - let inst = - Inst::xmm_rm_r(SseOpcode::Xorpd, RegMem::reg(tmp_xmm.to_reg()), *tmp_xmm); + let inst = Inst::xmm_rm_r( + SseOpcode::Xorpd, + RegMem::reg(tmp_xmm.to_reg().to_reg()), + tmp_xmm.to_writable_reg(), + ); inst.emit(sink, info, state); - let inst = Inst::xmm_cmp_rm_r(cmp_op, RegMem::reg(src), tmp_xmm.to_reg()); + let inst = Inst::xmm_cmp_rm_r( + cmp_op, + RegMem::reg(src.to_reg()), + tmp_xmm.to_reg().to_reg(), + ); inst.emit(sink, info, state); one_way_jmp(sink, CC::NB, done); // jump over trap if 0 >= src @@ -2282,14 +2356,22 @@ pub(crate) fn emit( _ => unreachable!(), }; - let inst = Inst::imm(*src_size, cst, *tmp_gpr); + let inst = Inst::imm(*src_size, cst, tmp_gpr.to_writable_reg()); inst.emit(sink, info, state); - let inst = - Inst::gpr_to_xmm(cast_op, RegMem::reg(tmp_gpr.to_reg()), *src_size, *tmp_xmm); + let inst = Inst::gpr_to_xmm( + cast_op, + RegMem::reg(tmp_gpr.to_reg().to_reg()), + *src_size, + tmp_xmm.to_writable_reg(), + ); inst.emit(sink, info, state); - let inst = Inst::xmm_cmp_rm_r(cmp_op, RegMem::reg(tmp_xmm.to_reg()), src.to_reg()); + let inst = Inst::xmm_cmp_rm_r( + cmp_op, + RegMem::reg(tmp_xmm.to_reg().to_reg()), + src.to_reg().to_reg(), + ); inst.emit(sink, info, state); let handle_large = sink.get_label(); @@ -2303,8 +2385,8 @@ pub(crate) fn emit( let inst = Inst::alu_rmi_r( *dst_size, AluRmiROpcode::Xor, - RegMemImm::reg(dst.to_reg()), - *dst, + RegMemImm::reg(dst.to_reg().to_reg()), + dst.to_writable_reg(), ); inst.emit(sink, info, state); @@ -2321,10 +2403,15 @@ pub(crate) fn emit( // Actual truncation for small inputs: if the result is not positive, then we had an // overflow. - let inst = Inst::xmm_to_gpr(trunc_op, src.to_reg(), *dst, *dst_size); + let inst = Inst::xmm_to_gpr( + trunc_op, + src.to_reg().to_reg(), + dst.to_writable_reg(), + *dst_size, + ); inst.emit(sink, info, state); - let inst = Inst::cmp_rmi_r(*dst_size, RegMemImm::imm(0), dst.to_reg()); + let inst = Inst::cmp_rmi_r(*dst_size, RegMemImm::imm(0), dst.to_reg().to_reg()); inst.emit(sink, info, state); one_way_jmp(sink, CC::NL, done); // if dst >= 0, jump to done @@ -2335,8 +2422,8 @@ pub(crate) fn emit( let inst = Inst::alu_rmi_r( *dst_size, AluRmiROpcode::Xor, - RegMemImm::reg(dst.to_reg()), - *dst, + RegMemImm::reg(dst.to_reg().to_reg()), + dst.to_writable_reg(), ); inst.emit(sink, info, state); @@ -2352,13 +2439,22 @@ pub(crate) fn emit( sink.bind_label(handle_large); - let inst = Inst::xmm_rm_r(sub_op, RegMem::reg(tmp_xmm.to_reg()), *src); + let inst = Inst::xmm_rm_r( + sub_op, + RegMem::reg(tmp_xmm.to_reg().to_reg()), + src.to_writable_reg(), + ); inst.emit(sink, info, state); - let inst = Inst::xmm_to_gpr(trunc_op, src.to_reg(), *dst, *dst_size); + let inst = Inst::xmm_to_gpr( + trunc_op, + src.to_reg().to_reg(), + dst.to_writable_reg(), + *dst_size, + ); inst.emit(sink, info, state); - let inst = Inst::cmp_rmi_r(*dst_size, RegMemImm::imm(0), dst.to_reg()); + let inst = Inst::cmp_rmi_r(*dst_size, RegMemImm::imm(0), dst.to_reg().to_reg()); inst.emit(sink, info, state); let next_is_large = sink.get_label(); @@ -2374,7 +2470,7 @@ pub(crate) fn emit( } else { u32::max_value() as u64 }, - *dst, + dst.to_writable_reg(), ); inst.emit(sink, info, state); @@ -2388,14 +2484,14 @@ pub(crate) fn emit( sink.bind_label(next_is_large); if *dst_size == OperandSize::Size64 { - let inst = Inst::imm(OperandSize::Size64, 1 << 63, *tmp_gpr); + let inst = Inst::imm(OperandSize::Size64, 1 << 63, tmp_gpr.to_writable_reg()); inst.emit(sink, info, state); let inst = Inst::alu_rmi_r( OperandSize::Size64, AluRmiROpcode::Add, - RegMemImm::reg(tmp_gpr.to_reg()), - *dst, + RegMemImm::reg(tmp_gpr.to_reg().to_reg()), + dst.to_writable_reg(), ); inst.emit(sink, info, state); } else { @@ -2403,7 +2499,7 @@ pub(crate) fn emit( OperandSize::Size32, AluRmiROpcode::Add, RegMemImm::imm(1 << 31), - *dst, + dst.to_writable_reg(), ); inst.emit(sink, info, state); } diff --git a/cranelift/codegen/src/isa/x64/inst/mod.rs b/cranelift/codegen/src/isa/x64/inst/mod.rs index 6f00370ea09d..bd7c039b9436 100644 --- a/cranelift/codegen/src/isa/x64/inst/mod.rs +++ b/cranelift/codegen/src/isa/x64/inst/mod.rs @@ -2,13 +2,11 @@ use crate::binemit::{Addend, CodeOffset, Reloc, StackMap}; use crate::ir::{types, ExternalName, Opcode, SourceLoc, TrapCode, Type, ValueLabel}; -use crate::isa::unwind::UnwindInst; use crate::isa::x64::abi::X64ABIMachineSpec; use crate::isa::x64::settings as x64_settings; use crate::isa::CallConv; use crate::machinst::*; use crate::{settings, CodegenError, CodegenResult}; -use alloc::boxed::Box; use alloc::vec::Vec; use regalloc::{ PrettyPrint, PrettyPrintSized, RealRegUniverse, Reg, RegClass, RegUsageCollector, SpillSlot, @@ -31,510 +29,8 @@ use regs::show_ireg_sized; //============================================================================= // Instructions (top level): definition -// Don't build these directly. Instead use the Inst:: functions to create them. - -/// Instructions. -#[derive(Clone)] -pub enum Inst { - /// Nops of various sizes, including zero. - Nop { len: u8 }, - - // ===================================== - // Integer instructions. - /// Integer arithmetic/bit-twiddling: (add sub and or xor mul adc? sbb?) (32 64) (reg addr imm) reg - AluRmiR { - size: OperandSize, // 4 or 8 - op: AluRmiROpcode, - src1: Reg, - src2: RegMemImm, - dst: Writable, - }, - - /// Instructions on GPR that only read src and defines dst (dst is not modified): bsr, etc. - UnaryRmR { - size: OperandSize, // 2, 4 or 8 - op: UnaryRmROpcode, - src: RegMem, - dst: Writable, - }, - - /// Bitwise not - Not { - size: OperandSize, // 1, 2, 4 or 8 - src: Reg, - dst: Writable, - }, - - /// Integer negation - Neg { - size: OperandSize, // 1, 2, 4 or 8 - src: Reg, - dst: Writable, - }, - - /// Integer quotient and remainder: (div idiv) $rax $rdx (reg addr) - Div { - size: OperandSize, // 1, 2, 4 or 8 - signed: bool, - divisor: RegMem, - dividend: Reg, - dst_quotient: Writable, - dst_remainder: Writable, - }, - - /// The high bits (RDX) of a (un)signed multiply: RDX:RAX := RAX * rhs. - MulHi { - size: OperandSize, // 2, 4, or 8 - signed: bool, - src1: Reg, - src2: RegMem, - dst_lo: Writable, - dst_hi: Writable, - }, - - /// A synthetic sequence to implement the right inline checks for remainder and division, - /// assuming the dividend is in %rax. - /// - /// Puts the result back into %rax if is_div, %rdx if !is_div, to mimic what the div - /// instruction does. - /// - /// The generated code sequence is described in the emit's function match arm for this - /// instruction. - /// - /// Note: %rdx is marked as modified by this instruction, to avoid an early clobber problem - /// with the temporary and divisor registers. Make sure to zero %rdx right before this - /// instruction, or you might run into regalloc failures where %rdx is live before its first - /// def! - CheckedDivOrRemSeq { - kind: DivOrRemKind, - size: OperandSize, - dividend: Reg, - /// The divisor operand. Note it's marked as modified so that it gets assigned a register - /// different from the temporary. - divisor: Writable, - dst_quotient: Writable, - dst_remainder: Writable, - tmp: Option>, - }, - - /// Do a sign-extend based on the sign of the value in rax into rdx: (cwd cdq cqo) - /// or al into ah: (cbw) - SignExtendData { - size: OperandSize, // 1, 2, 4 or 8 - src: Reg, - dst: Writable, - }, - - /// Constant materialization: (imm32 imm64) reg. - /// - /// Either: movl $imm32, %reg32 or movabsq $imm64, %reg32. - Imm { - dst_size: OperandSize, // 4 or 8 - simm64: u64, - dst: Writable, - }, - - /// GPR to GPR move: mov (64 32) reg reg. - MovRR { - size: OperandSize, // 4 or 8 - src: Reg, - dst: Writable, - }, - - /// Zero-extended loads, except for 64 bits: movz (bl bq wl wq lq) addr reg. - /// Note that the lq variant doesn't really exist since the default zero-extend rule makes it - /// unnecessary. For that case we emit the equivalent "movl AM, reg32". - MovzxRmR { - ext_mode: ExtMode, - src: RegMem, - dst: Writable, - }, - - /// A plain 64-bit integer load, since MovZX_RM_R can't represent that. - Mov64MR { - src: SyntheticAmode, - dst: Writable, - }, - - /// Loads the memory address of addr into dst. - LoadEffectiveAddress { - addr: SyntheticAmode, - dst: Writable, - }, - - /// Sign-extended loads and moves: movs (bl bq wl wq lq) addr reg. - MovsxRmR { - ext_mode: ExtMode, - src: RegMem, - dst: Writable, - }, - - /// Integer stores: mov (b w l q) reg addr. - MovRM { - size: OperandSize, // 1, 2, 4 or 8. - src: Reg, - dst: SyntheticAmode, - }, - - /// Arithmetic shifts: (shl shr sar) (b w l q) imm reg. - ShiftR { - size: OperandSize, // 1, 2, 4 or 8 - kind: ShiftKind, - src: Reg, - /// shift count: Some(0 .. #bits-in-type - 1), or None to mean "%cl". - num_bits: Imm8Reg, - dst: Writable, - }, - - /// Arithmetic SIMD shifts. - XmmRmiReg { - opcode: SseOpcode, - src1: Reg, - src2: RegMemImm, - dst: Writable, - }, - - /// Integer comparisons/tests: cmp or test (b w l q) (reg addr imm) reg. - CmpRmiR { - size: OperandSize, // 1, 2, 4 or 8 - opcode: CmpOpcode, - src: RegMemImm, - dst: Reg, - }, - - /// Materializes the requested condition code in the destination reg. - Setcc { cc: CC, dst: Writable }, - - /// Integer conditional move. - /// Overwrites the destination register. - Cmove { - size: OperandSize, // 2, 4, or 8 - cc: CC, - consequent: RegMem, - alternative: Reg, - dst: Writable, - }, - - // ===================================== - // Stack manipulation. - /// pushq (reg addr imm) - Push64 { src: RegMemImm }, - - /// popq reg - Pop64 { dst: Writable }, - - // ===================================== - // Floating-point operations. - /// XMM (scalar or vector) binary op: (add sub and or xor mul adc? sbb?) (32 64) (reg addr) reg - XmmRmR { - op: SseOpcode, - src1: Reg, - src2: RegMem, - dst: Writable, - }, - - XmmRmREvex { - op: Avx512Opcode, - src1: RegMem, - src2: Reg, - dst: Writable, - }, - - /// XMM (scalar or vector) unary op: mov between XMM registers (32 64) (reg addr) reg, sqrt, - /// etc. - /// - /// This differs from XMM_RM_R in that the dst register of XmmUnaryRmR is not used in the - /// computation of the instruction dst value and so does not have to be a previously valid - /// value. This is characteristic of mov instructions. - XmmUnaryRmR { - op: SseOpcode, - src: RegMem, - dst: Writable, - }, - - XmmUnaryRmREvex { - op: Avx512Opcode, - src: RegMem, - dst: Writable, - }, - - /// XMM (scalar or vector) unary op (from xmm to reg/mem): stores, movd, movq - XmmMovRM { - op: SseOpcode, - src: Reg, - dst: SyntheticAmode, - }, - - /// XMM (vector) unary op (to move a constant value into an xmm register): movups - XmmLoadConst { - src: VCodeConstant, - dst: Writable, - ty: Type, - }, - - /// XMM (scalar) unary op (from xmm to integer reg): movd, movq, cvtts{s,d}2si - XmmToGpr { - op: SseOpcode, - src: Reg, - dst: Writable, - dst_size: OperandSize, - }, - - /// XMM (scalar) unary op (from integer to float reg): movd, movq, cvtsi2s{s,d} - GprToXmm { - op: SseOpcode, - src: RegMem, - dst: Writable, - src_size: OperandSize, - }, - - /// Converts an unsigned int64 to a float32/float64. - CvtUint64ToFloatSeq { - dst_size: OperandSize, // 4 or 8 - /// A copy of the source register, fed by lowering. It is marked as modified during - /// register allocation to make sure that the temporary registers differ from the src - /// register, since both registers are live at the same time in the generated code - /// sequence. - src: Writable, - dst: Writable, - tmp_gpr1: Writable, - tmp_gpr2: Writable, - }, - - /// Converts a scalar xmm to a signed int32/int64. - CvtFloatToSintSeq { - dst_size: OperandSize, - src_size: OperandSize, - is_saturating: bool, - /// A copy of the source register, fed by lowering. It is marked as modified during - /// register allocation to make sure that the temporary xmm register differs from the src - /// register, since both registers are live at the same time in the generated code - /// sequence. - src: Writable, - dst: Writable, - tmp_gpr: Writable, - tmp_xmm: Writable, - }, - - /// Converts a scalar xmm to an unsigned int32/int64. - CvtFloatToUintSeq { - src_size: OperandSize, - dst_size: OperandSize, - is_saturating: bool, - /// A copy of the source register, fed by lowering, reused as a temporary. It is marked as - /// modified during register allocation to make sure that the temporary xmm register - /// differs from the src register, since both registers are live at the same time in the - /// generated code sequence. - src: Writable, - dst: Writable, - tmp_gpr: Writable, - tmp_xmm: Writable, - }, - - /// A sequence to compute min/max with the proper NaN semantics for xmm registers. - XmmMinMaxSeq { - size: OperandSize, - is_min: bool, - lhs: Reg, - rhs_dst: Writable, - }, - - /// XMM (scalar) conditional move. - /// Overwrites the destination register if cc is set. - XmmCmove { - size: OperandSize, // 4 or 8 - cc: CC, - src: RegMem, - dst: Writable, - }, - - /// Float comparisons/tests: cmp (b w l q) (reg addr imm) reg. - XmmCmpRmR { - op: SseOpcode, - src: RegMem, - dst: Reg, - }, - - /// A binary XMM instruction with an 8-bit immediate: e.g. cmp (ps pd) imm (reg addr) reg - XmmRmRImm { - op: SseOpcode, - src1: Reg, - src2: RegMem, - dst: Writable, - imm: u8, - size: OperandSize, // 4 or 8 - }, - - // ===================================== - // Control flow instructions. - /// Direct call: call simm32. - CallKnown { - dest: ExternalName, - uses: Vec, - defs: Vec>, - opcode: Opcode, - }, - - /// Indirect call: callq (reg mem). - CallUnknown { - dest: RegMem, - uses: Vec, - defs: Vec>, - opcode: Opcode, - }, - - /// Return. - Ret, - - /// A placeholder instruction, generating no code, meaning that a function epilogue must be - /// inserted there. - EpiloguePlaceholder, - - /// Jump to a known target: jmp simm32. - JmpKnown { dst: MachLabel }, - - /// One-way conditional branch: jcond cond target. - /// - /// This instruction is useful when we have conditional jumps depending on more than two - /// conditions, see for instance the lowering of Brz/brnz with Fcmp inputs. - /// - /// A note of caution: in contexts where the branch target is another block, this has to be the - /// same successor as the one specified in the terminator branch of the current block. - /// Otherwise, this might confuse register allocation by creating new invisible edges. - JmpIf { cc: CC, taken: MachLabel }, - - /// Two-way conditional branch: jcond cond target target. - /// Emitted as a compound sequence; the MachBuffer will shrink it as appropriate. - JmpCond { - cc: CC, - taken: MachLabel, - not_taken: MachLabel, - }, - - /// Jump-table sequence, as one compound instruction (see note in lower.rs for rationale). - /// The generated code sequence is described in the emit's function match arm for this - /// instruction. - /// See comment in lowering about the temporaries signedness. - JmpTableSeq { - idx: Reg, - tmp1: Writable, - tmp2: Writable, - default_target: MachLabel, - targets: Vec, - targets_for_term: Vec, - }, - - /// Indirect jump: jmpq (reg mem). - JmpUnknown { target: RegMem }, - - /// Traps if the condition code is set. - TrapIf { cc: CC, trap_code: TrapCode }, - - /// A debug trap. - Hlt, - - /// An instruction that will always trigger the illegal instruction exception. - Ud2 { trap_code: TrapCode }, - - /// Loads an external symbol in a register, with a relocation: - /// - /// movq $name@GOTPCREL(%rip), dst if PIC is enabled, or - /// movabsq $name, dst otherwise. - LoadExtName { - dst: Writable, - name: Box, - offset: i64, - }, - - // ===================================== - // Instructions pertaining to atomic memory accesses. - /// A standard (native) `lock cmpxchg src, (amode)`, with register conventions: - /// - /// `mem` (read) address - /// `replacement` (read) replacement value - /// %rax (modified) in: expected value, out: value that was actually at `dst` - /// %rflags is written. Do not assume anything about it after the instruction. - /// - /// The instruction "succeeded" iff the lowest `ty` bits of %rax afterwards are the same as - /// they were before. - LockCmpxchg { - ty: Type, // I8, I16, I32 or I64 - replacement: Reg, - expected: Reg, - mem: SyntheticAmode, - dst_old: Writable, - }, - - /// A synthetic instruction, based on a loop around a native `lock cmpxchg` instruction. - /// This atomically modifies a value in memory and returns the old value. The sequence - /// consists of an initial "normal" load from `dst`, followed by a loop which computes the - /// new value and tries to compare-and-swap ("CAS") it into `dst`, using the native - /// instruction `lock cmpxchg{b,w,l,q}` . The loop iterates until the CAS is successful. - /// If there is no contention, there will be only one pass through the loop body. The - /// sequence does *not* perform any explicit memory fence instructions - /// (mfence/sfence/lfence). - /// - /// Note that the transaction is atomic in the sense that, as observed by some other thread, - /// `dst` either has the initial or final value, but no other. It isn't atomic in the sense - /// of guaranteeing that no other thread writes to `dst` in between the initial load and the - /// CAS -- but that would cause the CAS to fail unless the other thread's last write before - /// the CAS wrote the same value that was already there. In other words, this - /// implementation suffers (unavoidably) from the A-B-A problem. - /// - /// This instruction sequence has fixed register uses as follows: - /// - /// %r9 (read) address - /// %r10 (read) second operand for `op` - /// %r11 (written) scratch reg; value afterwards has no meaning - /// %rax (written) the old value at %r9 - /// %rflags is written. Do not assume anything about it after the instruction. - AtomicRmwSeq { - ty: Type, // I8, I16, I32 or I64 - op: inst_common::AtomicRmwOp, - address: Reg, - operand: Reg, - temp: Writable, - dst_old: Writable, - }, - - /// A memory fence (mfence, lfence or sfence). - Fence { kind: FenceKind }, - - // ===================================== - // Meta-instructions generating no code. - /// Marker, no-op in generated code: SP "virtual offset" is adjusted. This - /// controls how MemArg::NominalSPOffset args are lowered. - VirtualSPOffsetAdj { offset: i64 }, - - /// Provides a way to tell the register allocator that the upcoming sequence of instructions - /// will overwrite `dst` so it should be considered as a `def`; use this with care. - /// - /// This is useful when we have a sequence of instructions whose register usages are nominally - /// `mod`s, but such that the combination of operations creates a result that is independent of - /// the initial register value. It's thus semantically a `def`, not a `mod`, when all the - /// instructions are taken together, so we want to ensure the register is defined (its - /// live-range starts) prior to the sequence to keep analyses happy. - /// - /// One alternative would be a compound instruction that somehow encapsulates the others and - /// reports its own `def`s/`use`s/`mod`s; this adds complexity (the instruction list is no - /// longer flat) and requires knowledge about semantics and initial-value independence anyway. - XmmUninitializedValue { dst: Writable }, - - /// A call to the `ElfTlsGetAddr` libcall. Returns address - /// of TLS symbol in rax. - ElfTlsGetAddr { symbol: ExternalName }, - - /// A Mach-O TLS symbol access. Returns address of the TLS - /// symbol in rax. - MachOTlsGetAddr { symbol: ExternalName }, - - /// A definition of a value label. - ValueLabelMarker { reg: Reg, label: ValueLabel }, - - /// An unwind pseudoinstruction describing the state of the - /// machine at this program point. - Unwind { inst: UnwindInst }, -} +// `Inst` is defined inside ISLE as `MInst`. We publicly re-export it here. +pub use super::lower::isle::generated_code::MInst as Inst; pub(crate) fn low32_will_sign_extend_to_64(x: u64) -> bool { let xs = x as i64; @@ -656,15 +152,20 @@ impl Inst { OperandSize::Size32, OperandSize::Size64 ])); - Self::UnaryRmR { size, op, src, dst } + Self::UnaryRmR { + size, + op, + src: GprMem::new(src).unwrap(), + dst: WritableGpr::from_writable_reg(dst).unwrap(), + } } pub(crate) fn not(size: OperandSize, src: Writable) -> Inst { debug_assert_eq!(src.to_reg().get_class(), RegClass::I64); Inst::Not { size, - src: src.to_reg(), - dst: src, + src: Gpr::new(src.to_reg()).unwrap(), + dst: WritableGpr::from_writable_reg(src).unwrap(), } } @@ -753,7 +254,11 @@ impl Inst { pub(crate) fn xmm_mov(op: SseOpcode, src: RegMem, dst: Writable) -> Inst { src.assert_regclass_is(RegClass::V128); debug_assert!(dst.to_reg().get_class() == RegClass::V128); - Inst::XmmUnaryRmR { op, src, dst } + Inst::XmmUnaryRmR { + op, + src: XmmMem::new(src).unwrap(), + dst: WritableXmm::from_writable_reg(dst).unwrap(), + } } pub(crate) fn xmm_load_const(src: VCodeConstant, dst: Writable, ty: Type) -> Inst { @@ -766,13 +271,21 @@ impl Inst { pub(crate) fn xmm_unary_rm_r(op: SseOpcode, src: RegMem, dst: Writable) -> Inst { src.assert_regclass_is(RegClass::V128); debug_assert!(dst.to_reg().get_class() == RegClass::V128); - Inst::XmmUnaryRmR { op, src, dst } + Inst::XmmUnaryRmR { + op, + src: XmmMem::new(src).unwrap(), + dst: WritableXmm::from_writable_reg(dst).unwrap(), + } } pub(crate) fn xmm_unary_rm_r_evex(op: Avx512Opcode, src: RegMem, dst: Writable) -> Inst { src.assert_regclass_is(RegClass::V128); debug_assert!(dst.to_reg().get_class() == RegClass::V128); - Inst::XmmUnaryRmREvex { op, src, dst } + Inst::XmmUnaryRmREvex { + op, + src: XmmMem::new(src).unwrap(), + dst: WritableXmm::from_writable_reg(dst).unwrap(), + } } pub(crate) fn xmm_rm_r(op: SseOpcode, src: RegMem, dst: Writable) -> Self { @@ -780,9 +293,9 @@ impl Inst { debug_assert!(dst.to_reg().get_class() == RegClass::V128); Inst::XmmRmR { op, - src1: dst.to_reg(), - src2: src, - dst, + src1: Xmm::new(dst.to_reg()).unwrap(), + src2: XmmMem::new(src).unwrap(), + dst: WritableXmm::from_writable_reg(dst).unwrap(), } } @@ -797,15 +310,17 @@ impl Inst { debug_assert!(dst.to_reg().get_class() == RegClass::V128); Inst::XmmRmREvex { op, - src1, - src2, - dst, + src1: XmmMem::new(src1).unwrap(), + src2: Xmm::new(src2).unwrap(), + dst: WritableXmm::from_writable_reg(dst).unwrap(), } } pub(crate) fn xmm_uninit_value(dst: Writable) -> Self { debug_assert!(dst.to_reg().get_class() == RegClass::V128); - Inst::XmmUninitializedValue { dst } + Inst::XmmUninitializedValue { + dst: WritableXmm::from_writable_reg(dst).unwrap(), + } } pub(crate) fn xmm_mov_r_m(op: SseOpcode, src: Reg, dst: impl Into) -> Inst { @@ -828,8 +343,8 @@ impl Inst { debug_assert!(dst_size.is_one_of(&[OperandSize::Size32, OperandSize::Size64])); Inst::XmmToGpr { op, - src, - dst, + src: Xmm::new(src).unwrap(), + dst: WritableGpr::from_writable_reg(dst).unwrap(), dst_size, } } @@ -846,7 +361,7 @@ impl Inst { Inst::GprToXmm { op, src, - dst, + dst: WritableXmm::from_writable_reg(dst).unwrap(), src_size, } } @@ -870,10 +385,10 @@ impl Inst { debug_assert!(tmp_gpr2.to_reg().get_class() == RegClass::I64); debug_assert!(dst.to_reg().get_class() == RegClass::V128); Inst::CvtUint64ToFloatSeq { - src, - dst, - tmp_gpr1, - tmp_gpr2, + src: WritableGpr::from_writable_reg(src).unwrap(), + dst: WritableXmm::from_writable_reg(dst).unwrap(), + tmp_gpr1: WritableGpr::from_writable_reg(tmp_gpr1).unwrap(), + tmp_gpr2: WritableGpr::from_writable_reg(tmp_gpr2).unwrap(), dst_size, } } @@ -897,10 +412,10 @@ impl Inst { src_size, dst_size, is_saturating, - src, - dst, - tmp_gpr, - tmp_xmm, + src: WritableXmm::from_writable_reg(src).unwrap(), + dst: WritableGpr::from_writable_reg(dst).unwrap(), + tmp_gpr: WritableGpr::from_writable_reg(tmp_gpr).unwrap(), + tmp_xmm: WritableXmm::from_writable_reg(tmp_xmm).unwrap(), } } @@ -923,10 +438,10 @@ impl Inst { src_size, dst_size, is_saturating, - src, - dst, - tmp_gpr, - tmp_xmm, + src: WritableXmm::from_writable_reg(src).unwrap(), + dst: WritableGpr::from_writable_reg(dst).unwrap(), + tmp_gpr: WritableGpr::from_writable_reg(tmp_gpr).unwrap(), + tmp_xmm: WritableXmm::from_writable_reg(tmp_xmm).unwrap(), } } @@ -976,9 +491,9 @@ impl Inst { debug_assert!(dst.to_reg().get_class() == RegClass::V128); Inst::XmmRmiReg { opcode, - src1: dst.to_reg(), - src2: src, - dst, + src1: Xmm::new(dst.to_reg()).unwrap(), + src2: XmmMemImm::new(src).unwrap(), + dst: WritableXmm::from_writable_reg(dst).unwrap(), } } @@ -1018,7 +533,7 @@ impl Inst { debug_assert!(dst.to_reg().get_class() == RegClass::I64); Inst::LoadEffectiveAddress { addr: addr.into(), - dst, + dst: WritableGpr::from_writable_reg(dst).unwrap(), } } @@ -1270,7 +785,7 @@ impl Inst { } Self::XmmRmR { op, src2, dst, .. } => { - src2.to_reg() == Some(dst.to_reg()) + src2.clone().to_reg_mem().to_reg() == Some(dst.to_reg().to_reg()) && (*op == SseOpcode::Xorps || *op == SseOpcode::Xorpd || *op == SseOpcode::Pxor @@ -1331,7 +846,11 @@ impl Inst { Inst::XmmRmiReg { src1, dst, .. } => { if *src1 != dst.to_reg() { debug_assert!(src1.is_virtual()); - insts.push(Self::gen_move(*dst, *src1, types::I8X16)); + insts.push(Self::gen_move( + dst.to_writable_reg(), + src1.to_reg(), + types::I8X16, + )); *src1 = dst.to_reg(); } insts.push(self); @@ -1339,7 +858,11 @@ impl Inst { Inst::XmmRmR { src1, dst, .. } => { if *src1 != dst.to_reg() { debug_assert!(src1.is_virtual()); - insts.push(Self::gen_move(*dst, *src1, types::I8X16)); + insts.push(Self::gen_move( + dst.to_writable_reg(), + src1.to_reg(), + types::I8X16, + )); *src1 = dst.to_reg(); } insts.push(self); @@ -1368,7 +891,11 @@ impl Inst { Inst::Not { src, dst, .. } | Inst::Neg { src, dst, .. } => { if *src != dst.to_reg() { debug_assert!(src.is_virtual()); - insts.push(Self::gen_move(*dst, *src, types::I64)); + insts.push(Self::gen_move( + dst.to_writable_reg(), + src.to_reg(), + types::I64, + )); *src = dst.to_reg(); } insts.push(self); @@ -1626,19 +1153,19 @@ impl PrettyPrint for Inst { "{} {}, {}", ljustify2(op.to_string(), suffix_bwlq(*size)), src.show_rru_sized(mb_rru, size.to_bytes()), - show_ireg_sized(dst.to_reg(), mb_rru, size.to_bytes()), + show_ireg_sized(dst.to_reg().to_reg(), mb_rru, size.to_bytes()), ), Inst::Not { size, src: _, dst } => format!( "{} {}", ljustify2("not".to_string(), suffix_bwlq(*size)), - show_ireg_sized(dst.to_reg(), mb_rru, size.to_bytes()) + show_ireg_sized(dst.to_reg().to_reg(), mb_rru, size.to_bytes()) ), Inst::Neg { size, src: _, dst } => format!( "{} {}", ljustify2("neg".to_string(), suffix_bwlq(*size)), - show_ireg_sized(dst.to_reg(), mb_rru, size.to_bytes()) + show_ireg_sized(dst.to_reg().to_reg(), mb_rru, size.to_bytes()) ), Inst::Div { @@ -1696,14 +1223,14 @@ impl PrettyPrint for Inst { "{} {}, {}", ljustify(op.to_string()), src.show_rru_sized(mb_rru, op.src_size()), - show_ireg_sized(dst.to_reg(), mb_rru, 8), + show_ireg_sized(dst.to_reg().to_reg(), mb_rru, 8), ), Inst::XmmUnaryRmREvex { op, src, dst, .. } => format!( "{} {}, {}", ljustify(op.to_string()), src.show_rru_sized(mb_rru, 8), - show_ireg_sized(dst.to_reg(), mb_rru, 8), + show_ireg_sized(dst.to_reg().to_reg(), mb_rru, 8), ), Inst::XmmMovRM { op, src, dst, .. } => format!( @@ -1717,7 +1244,7 @@ impl PrettyPrint for Inst { "{} {}, {}", ljustify(op.to_string()), src2.show_rru_sized(mb_rru, 8), - show_ireg_sized(dst.to_reg(), mb_rru, 8), + show_ireg_sized(dst.to_reg().to_reg(), mb_rru, 8), ), Inst::XmmRmREvex { @@ -1730,8 +1257,8 @@ impl PrettyPrint for Inst { "{} {}, {}, {}", ljustify(op.to_string()), src1.show_rru_sized(mb_rru, 8), - show_ireg_sized(*src2, mb_rru, 8), - show_ireg_sized(dst.to_reg(), mb_rru, 8), + show_ireg_sized(src2.to_reg(), mb_rru, 8), + show_ireg_sized(dst.to_reg().to_reg(), mb_rru, 8), ), Inst::XmmMinMaxSeq { @@ -1795,7 +1322,7 @@ impl PrettyPrint for Inst { "{} {}, {}", ljustify(op.to_string()), src.show_rru(mb_rru), - show_ireg_sized(dst.to_reg(), mb_rru, dst_size), + show_ireg_sized(dst.to_reg().to_reg(), mb_rru, dst_size), ) } @@ -1830,7 +1357,7 @@ impl PrettyPrint for Inst { "f32" } )), - show_ireg_sized(src.to_reg(), mb_rru, 8), + show_ireg_sized(src.to_reg().to_reg(), mb_rru, 8), dst.show_rru(mb_rru), ), @@ -1847,8 +1374,8 @@ impl PrettyPrint for Inst { src_size.to_bits(), dst_size.to_bits() )), - show_ireg_sized(src.to_reg(), mb_rru, 8), - show_ireg_sized(dst.to_reg(), mb_rru, dst_size.to_bytes()), + show_ireg_sized(src.to_reg().to_reg(), mb_rru, 8), + show_ireg_sized(dst.to_reg().to_reg(), mb_rru, dst_size.to_bytes()), ), Inst::CvtFloatToUintSeq { @@ -1864,8 +1391,8 @@ impl PrettyPrint for Inst { src_size.to_bits(), dst_size.to_bits() )), - show_ireg_sized(src.to_reg(), mb_rru, 8), - show_ireg_sized(dst.to_reg(), mb_rru, dst_size.to_bytes()), + show_ireg_sized(src.to_reg().to_reg(), mb_rru, 8), + show_ireg_sized(dst.to_reg().to_reg(), mb_rru, dst_size.to_bytes()), ), Inst::Imm { @@ -2174,11 +1701,11 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) { } Inst::Not { src, dst, .. } => { debug_assert_eq!(*src, dst.to_reg()); - collector.add_mod(*dst); + collector.add_mod(dst.to_writable_reg()); } Inst::Neg { src, dst, .. } => { debug_assert_eq!(*src, dst.to_reg()); - collector.add_mod(*dst); + collector.add_mod(dst.to_writable_reg()); } Inst::Div { size, @@ -2249,11 +1776,13 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) { } } } - Inst::UnaryRmR { src, dst, .. } - | Inst::XmmUnaryRmR { src, dst, .. } - | Inst::XmmUnaryRmREvex { src, dst, .. } => { - src.get_regs_as_uses(collector); - collector.add_def(*dst); + Inst::UnaryRmR { src, dst, .. } => { + src.clone().to_reg_mem().get_regs_as_uses(collector); + collector.add_def(dst.to_writable_reg()); + } + Inst::XmmUnaryRmR { src, dst, .. } | Inst::XmmUnaryRmREvex { src, dst, .. } => { + src.clone().to_reg_mem().get_regs_as_uses(collector); + collector.add_def(dst.to_writable_reg()); } Inst::XmmRmR { src1, @@ -2265,10 +1794,10 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) { debug_assert_eq!(*src1, dst.to_reg()); if inst.produces_const() { // No need to account for src, since src == dst. - collector.add_def(*dst); + collector.add_def(dst.to_writable_reg()); } else { - src2.get_regs_as_uses(collector); - collector.add_mod(*dst); + src2.clone().to_reg_mem().get_regs_as_uses(collector); + collector.add_mod(dst.to_writable_reg()); // Some instructions have an implicit use of XMM0. if *op == SseOpcode::Blendvpd || *op == SseOpcode::Blendvps @@ -2285,11 +1814,11 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) { dst, .. } => { - src1.get_regs_as_uses(collector); - collector.add_use(*src2); + src1.clone().to_reg_mem().get_regs_as_uses(collector); + collector.add_use(src2.to_reg()); match *op { - Avx512Opcode::Vpermi2b => collector.add_mod(*dst), - _ => collector.add_def(*dst), + Avx512Opcode::Vpermi2b => collector.add_mod(dst.to_writable_reg()), + _ => collector.add_def(dst.to_writable_reg()), } } Inst::XmmRmRImm { @@ -2320,7 +1849,7 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) { collector.add_mod(*dst); } } - Inst::XmmUninitializedValue { dst } => collector.add_def(*dst), + Inst::XmmUninitializedValue { dst } => collector.add_def(dst.to_writable_reg()), Inst::XmmLoadConst { dst, .. } => collector.add_def(*dst), Inst::XmmMinMaxSeq { lhs, rhs_dst, .. } => { collector.add_use(*lhs); @@ -2330,8 +1859,8 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) { src1, src2, dst, .. } => { debug_assert_eq!(*src1, dst.to_reg()); - src2.get_regs_as_uses(collector); - collector.add_mod(*dst); + src2.clone().to_reg_mem_imm().get_regs_as_uses(collector); + collector.add_mod(dst.to_writable_reg()); } Inst::XmmMovRM { src, dst, .. } => { collector.add_use(*src); @@ -2344,13 +1873,17 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) { Inst::Imm { dst, .. } => { collector.add_def(*dst); } - Inst::MovRR { src, dst, .. } | Inst::XmmToGpr { src, dst, .. } => { + Inst::MovRR { src, dst, .. } => { collector.add_use(*src); collector.add_def(*dst); } + Inst::XmmToGpr { src, dst, .. } => { + collector.add_use(src.to_reg()); + collector.add_def(dst.to_writable_reg()); + } Inst::GprToXmm { src, dst, .. } => { src.get_regs_as_uses(collector); - collector.add_def(*dst); + collector.add_def(dst.to_writable_reg()); } Inst::CvtUint64ToFloatSeq { src, @@ -2359,10 +1892,10 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) { tmp_gpr2, .. } => { - collector.add_mod(*src); - collector.add_def(*dst); - collector.add_def(*tmp_gpr1); - collector.add_def(*tmp_gpr2); + collector.add_mod(src.to_writable_reg()); + collector.add_def(dst.to_writable_reg()); + collector.add_def(tmp_gpr1.to_writable_reg()); + collector.add_def(tmp_gpr2.to_writable_reg()); } Inst::CvtFloatToSintSeq { src, @@ -2378,19 +1911,23 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) { tmp_xmm, .. } => { - collector.add_mod(*src); - collector.add_def(*dst); - collector.add_def(*tmp_gpr); - collector.add_def(*tmp_xmm); + collector.add_mod(src.to_writable_reg()); + collector.add_def(dst.to_writable_reg()); + collector.add_def(tmp_gpr.to_writable_reg()); + collector.add_def(tmp_xmm.to_writable_reg()); } Inst::MovzxRmR { src, dst, .. } => { src.get_regs_as_uses(collector); collector.add_def(*dst); } - Inst::Mov64MR { src, dst, .. } | Inst::LoadEffectiveAddress { addr: src, dst } => { + Inst::Mov64MR { src, dst, .. } => { src.get_regs_as_uses(collector); collector.add_def(*dst) } + Inst::LoadEffectiveAddress { addr: src, dst } => { + src.get_regs_as_uses(collector); + collector.add_def(dst.to_writable_reg()) + } Inst::MovsxRmR { src, dst, .. } => { src.get_regs_as_uses(collector); collector.add_def(*dst); @@ -2535,8 +2072,8 @@ impl Amode { ref mut index, .. } => { - map.map_use(base); - map.map_use(index); + base.map_use(map); + index.map_use(map); } Amode::RipRelative { .. } => { // RIP isn't involved in regalloc. @@ -2622,7 +2159,7 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { } Inst::Not { src, dst, .. } | Inst::Neg { src, dst, .. } => { debug_assert_eq!(*src, dst.to_reg()); - mapper.map_mod(dst); + dst.map_mod(mapper); *src = dst.to_reg(); } Inst::Div { divisor, .. } => divisor.map_uses(mapper), @@ -2643,14 +2180,17 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { ref mut src, ref mut dst, .. + } => { + src.map_uses(mapper); + dst.map_def(mapper); } - | Inst::UnaryRmR { + Inst::UnaryRmR { ref mut src, ref mut dst, .. } => { src.map_uses(mapper); - mapper.map_def(dst); + dst.map_def(mapper); } Inst::XmmRmRImm { ref op, @@ -2691,11 +2231,11 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { debug_assert_eq!(*src1, dst.to_reg()); if produces_const { src2.map_as_def(mapper); - mapper.map_def(dst); + dst.map_def(mapper); *src1 = dst.to_reg(); } else { src2.map_uses(mapper); - mapper.map_mod(dst); + dst.map_mod(mapper); *src1 = dst.to_reg(); } } @@ -2707,10 +2247,10 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { .. } => { src1.map_uses(mapper); - mapper.map_use(src2); + src2.map_use(mapper); match *op { - Avx512Opcode::Vpermi2b => mapper.map_mod(dst), - _ => mapper.map_def(dst), + Avx512Opcode::Vpermi2b => dst.map_mod(mapper), + _ => dst.map_def(mapper), } } Inst::XmmRmiReg { @@ -2721,11 +2261,11 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { } => { debug_assert_eq!(*src1, dst.to_reg()); src2.map_uses(mapper); - mapper.map_mod(dst); + dst.map_mod(mapper); *src1 = dst.to_reg(); } Inst::XmmUninitializedValue { ref mut dst, .. } => { - mapper.map_def(dst); + dst.map_def(mapper); } Inst::XmmLoadConst { ref mut dst, .. } => { mapper.map_def(dst); @@ -2759,14 +2299,17 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { ref mut src, ref mut dst, .. + } => { + mapper.map_use(src); + mapper.map_def(dst); } - | Inst::XmmToGpr { + Inst::XmmToGpr { ref mut src, ref mut dst, .. } => { - mapper.map_use(src); - mapper.map_def(dst); + src.map_use(mapper); + dst.map_def(mapper); } Inst::GprToXmm { ref mut src, @@ -2774,7 +2317,7 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { .. } => { src.map_uses(mapper); - mapper.map_def(dst); + dst.map_def(mapper); } Inst::CvtUint64ToFloatSeq { ref mut src, @@ -2783,10 +2326,10 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { ref mut tmp_gpr2, .. } => { - mapper.map_mod(src); - mapper.map_def(dst); - mapper.map_def(tmp_gpr1); - mapper.map_def(tmp_gpr2); + src.map_mod(mapper); + dst.map_def(mapper); + tmp_gpr1.map_def(mapper); + tmp_gpr2.map_def(mapper); } Inst::CvtFloatToSintSeq { ref mut src, @@ -2802,10 +2345,10 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { ref mut tmp_xmm, .. } => { - mapper.map_mod(src); - mapper.map_def(dst); - mapper.map_def(tmp_gpr); - mapper.map_def(tmp_xmm); + src.map_mod(mapper); + dst.map_def(mapper); + tmp_gpr.map_def(mapper); + tmp_xmm.map_def(mapper); } Inst::MovzxRmR { ref mut src, @@ -2815,10 +2358,14 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { src.map_uses(mapper); mapper.map_def(dst); } - Inst::Mov64MR { src, dst, .. } | Inst::LoadEffectiveAddress { addr: src, dst } => { + Inst::Mov64MR { src, dst, .. } => { src.map_uses(mapper); mapper.map_def(dst); } + Inst::LoadEffectiveAddress { addr: src, dst } => { + src.map_uses(mapper); + dst.map_def(mapper); + } Inst::MovsxRmR { ref mut src, ref mut dst, @@ -2989,8 +2536,8 @@ impl MachInst for Inst { || *op == SseOpcode::Movdqa || *op == SseOpcode::Movdqu => { - if let RegMem::Reg { reg } = src { - Some((*dst, *reg)) + if let RegMem::Reg { reg } = src.clone().to_reg_mem() { + Some((dst.to_writable_reg(), reg)) } else { None } diff --git a/cranelift/codegen/src/isa/x64/lower.isle b/cranelift/codegen/src/isa/x64/lower.isle index 9b881b5f691c..8de9110bbdfe 100644 --- a/cranelift/codegen/src/isa/x64/lower.isle +++ b/cranelift/codegen/src/isa/x64/lower.isle @@ -95,23 +95,23 @@ (rule (lower (has_type (multi_lane 8 16) (iadd x y))) - (value_reg (paddb (put_in_reg x) - (put_in_reg_mem y)))) + (value_xmm (paddb (put_in_xmm x) + (put_in_xmm_mem y)))) (rule (lower (has_type (multi_lane 16 8) (iadd x y))) - (value_reg (paddw (put_in_reg x) - (put_in_reg_mem y)))) + (value_xmm (paddw (put_in_xmm x) + (put_in_xmm_mem y)))) (rule (lower (has_type (multi_lane 32 4) (iadd x y))) - (value_reg (paddd (put_in_reg x) - (put_in_reg_mem y)))) + (value_xmm (paddd (put_in_xmm x) + (put_in_xmm_mem y)))) (rule (lower (has_type (multi_lane 64 2) (iadd x y))) - (value_reg (paddq (put_in_reg x) - (put_in_reg_mem y)))) + (value_xmm (paddq (put_in_xmm x) + (put_in_xmm_mem y)))) ;; `i128` (rule (lower (has_type $I128 (iadd x y))) @@ -131,25 +131,25 @@ (rule (lower (has_type (multi_lane 8 16) (sadd_sat x y))) - (value_reg (paddsb (put_in_reg x) - (put_in_reg_mem y)))) + (value_xmm (paddsb (put_in_xmm x) + (put_in_xmm_mem y)))) (rule (lower (has_type (multi_lane 16 8) (sadd_sat x y))) - (value_reg (paddsw (put_in_reg x) - (put_in_reg_mem y)))) + (value_xmm (paddsw (put_in_xmm x) + (put_in_xmm_mem y)))) ;;;; Rules for `uadd_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (rule (lower (has_type (multi_lane 8 16) (uadd_sat x y))) - (value_reg (paddusb (put_in_reg x) - (put_in_reg_mem y)))) + (value_xmm (paddusb (put_in_xmm x) + (put_in_xmm_mem y)))) (rule (lower (has_type (multi_lane 16 8) (uadd_sat x y))) - (value_reg (paddusw (put_in_reg x) - (put_in_reg_mem y)))) + (value_xmm (paddusw (put_in_xmm x) + (put_in_xmm_mem y)))) ;;;; Rules for `iadd_ifcout` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -264,23 +264,23 @@ (rule (lower (has_type (multi_lane 8 16) (isub x y))) - (value_reg (psubb (put_in_reg x) - (put_in_reg_mem y)))) + (value_xmm (psubb (put_in_xmm x) + (put_in_xmm_mem y)))) (rule (lower (has_type (multi_lane 16 8) (isub x y))) - (value_reg (psubw (put_in_reg x) - (put_in_reg_mem y)))) + (value_xmm (psubw (put_in_xmm x) + (put_in_xmm_mem y)))) (rule (lower (has_type (multi_lane 32 4) (isub x y))) - (value_reg (psubd (put_in_reg x) - (put_in_reg_mem y)))) + (value_xmm (psubd (put_in_xmm x) + (put_in_xmm_mem y)))) (rule (lower (has_type (multi_lane 64 2) (isub x y))) - (value_reg (psubq (put_in_reg x) - (put_in_reg_mem y)))) + (value_xmm (psubq (put_in_xmm x) + (put_in_xmm_mem y)))) ;; `i128` (rule (lower (has_type $I128 (isub x y))) @@ -300,25 +300,25 @@ (rule (lower (has_type (multi_lane 8 16) (ssub_sat x y))) - (value_reg (psubsb (put_in_reg x) - (put_in_reg_mem y)))) + (value_xmm (psubsb (put_in_xmm x) + (put_in_xmm_mem y)))) (rule (lower (has_type (multi_lane 16 8) (ssub_sat x y))) - (value_reg (psubsw (put_in_reg x) - (put_in_reg_mem y)))) + (value_xmm (psubsw (put_in_xmm x) + (put_in_xmm_mem y)))) ;;;; Rules for `usub_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (rule (lower (has_type (multi_lane 8 16) (usub_sat x y))) - (value_reg (psubusb (put_in_reg x) - (put_in_reg_mem y)))) + (value_xmm (psubusb (put_in_xmm x) + (put_in_xmm_mem y)))) (rule (lower (has_type (multi_lane 16 8) (usub_sat x y))) - (value_reg (psubusw (put_in_reg x) - (put_in_reg_mem y)))) + (value_xmm (psubusw (put_in_xmm x) + (put_in_xmm_mem y)))) ;;;; Rules for `band` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -360,14 +360,16 @@ ;; SSE. -(decl sse_and (Type Reg RegMem) Reg) +(decl sse_and (Type Xmm XmmMem) Xmm) (rule (sse_and $F32X4 x y) (andps x y)) (rule (sse_and $F64X2 x y) (andpd x y)) (rule (sse_and (multi_lane _bits _lanes) x y) (pand x y)) (rule (lower (has_type ty @ (multi_lane _bits _lanes) (band x y))) - (value_reg (sse_and ty (put_in_reg x) (put_in_reg_mem y)))) + (value_xmm (sse_and ty + (put_in_xmm x) + (put_in_xmm_mem y)))) ;; `{i,b}128`. @@ -432,14 +434,16 @@ ;; SSE. -(decl sse_or (Type Reg RegMem) Reg) +(decl sse_or (Type Xmm XmmMem) Xmm) (rule (sse_or $F32X4 x y) (orps x y)) (rule (sse_or $F64X2 x y) (orpd x y)) (rule (sse_or (multi_lane _bits _lanes) x y) (por x y)) (rule (lower (has_type ty @ (multi_lane _bits _lanes) (bor x y))) - (value_reg (sse_or ty (put_in_reg x) (put_in_reg_mem y)))) + (value_xmm (sse_or ty + (put_in_xmm x) + (put_in_xmm_mem y)))) ;; `{i,b}128`. @@ -507,7 +511,7 @@ ;; SSE. (rule (lower (has_type ty @ (multi_lane _bits _lanes) (bxor x y))) - (value_reg (sse_xor ty (put_in_reg x) (put_in_reg_mem y)))) + (value_xmm (sse_xor ty (put_in_xmm x) (put_in_xmm_mem y)))) ;; `{i,b}128`. @@ -578,16 +582,16 @@ ;; instructions. The basic idea, whether the amount to shift by is an immediate ;; or not, is to use a 16x8 shift and then mask off the incorrect bits to 0s. (rule (lower (has_type $I8X16 (ishl src amt))) - (let ((src_ Reg (put_in_reg src)) + (let ((src_ Xmm (put_in_xmm src)) (amt_gpr RegMemImm (put_in_reg_mem_imm amt)) - (amt_xmm RegMemImm (reg_mem_imm_to_xmm amt_gpr)) + (amt_xmm XmmMemImm (mov_rmi_to_xmm amt_gpr)) ;; Shift `src` using 16x8. Unfortunately, a 16x8 shift will only be ;; correct for half of the lanes; the others must be fixed up with ;; the mask below. - (unmasked Reg (psllw src_ amt_xmm)) + (unmasked Xmm (psllw src_ amt_xmm)) (mask_addr SyntheticAmode (ishl_i8x16_mask amt_gpr)) (mask Reg (x64_load $I8X16 mask_addr (ExtKind.None)))) - (value_reg (sse_and $I8X16 unmasked (RegMem.Reg mask))))) + (value_xmm (sse_and $I8X16 unmasked (xmm_mem_new (RegMem.Reg mask)))))) ;; Get the address of the mask to use when fixing up the lanes that weren't ;; correctly generated by the 16x8 shift. @@ -608,25 +612,28 @@ (extern constructor ishl_i8x16_mask_table ishl_i8x16_mask_table) (rule (ishl_i8x16_mask (RegMemImm.Reg amt)) (let ((mask_table SyntheticAmode (ishl_i8x16_mask_table)) - (base_mask_addr Reg (lea mask_table)) + (base_mask_addr Gpr (lea mask_table)) (mask_offset Reg (shl $I64 amt (Imm8Reg.Imm8 4)))) (amode_to_synthetic_amode (amode_imm_reg_reg_shift 0 base_mask_addr - mask_offset + (gpr_new mask_offset) 0)))) (rule (ishl_i8x16_mask (RegMemImm.Mem amt)) (ishl_i8x16_mask (RegMemImm.Reg (x64_load $I64 amt (ExtKind.None))))) ;; 16x8, 32x4, and 64x2 shifts can each use a single instruction. + (rule (lower (has_type $I16X8 (ishl src amt))) - (value_reg (psllw (put_in_reg src) - (reg_mem_imm_to_xmm (put_in_reg_mem_imm amt))))) + (value_xmm (psllw (put_in_xmm src) + (mov_rmi_to_xmm (put_in_reg_mem_imm amt))))) + (rule (lower (has_type $I32X4 (ishl src amt))) - (value_reg (pslld (put_in_reg src) - (reg_mem_imm_to_xmm (put_in_reg_mem_imm amt))))) + (value_xmm (pslld (put_in_xmm src) + (mov_rmi_to_xmm (put_in_reg_mem_imm amt))))) + (rule (lower (has_type $I64X2 (ishl src amt))) - (value_reg (psllq (put_in_reg src) - (reg_mem_imm_to_xmm (put_in_reg_mem_imm amt))))) + (value_xmm (psllq (put_in_xmm src) + (mov_rmi_to_xmm (put_in_reg_mem_imm amt))))) ;;;; Rules for `ushr` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -672,16 +679,18 @@ ;; There are no 8x16 shifts in x64. Do the same 16x8-shift-and-mask thing we do ;; with 8x16 `ishl`. (rule (lower (has_type $I8X16 (ushr src amt))) - (let ((src_ Reg (put_in_reg src)) + (let ((src_ Xmm (put_in_xmm src)) (amt_gpr RegMemImm (put_in_reg_mem_imm amt)) - (amt_xmm RegMemImm (reg_mem_imm_to_xmm amt_gpr)) + (amt_xmm XmmMemImm (mov_rmi_to_xmm amt_gpr)) ;; Shift `src` using 16x8. Unfortunately, a 16x8 shift will only be ;; correct for half of the lanes; the others must be fixed up with ;; the mask below. - (unmasked Reg (psrlw src_ amt_xmm)) + (unmasked Xmm (psrlw src_ amt_xmm)) (mask_addr SyntheticAmode (ushr_i8x16_mask amt_gpr)) (mask Reg (x64_load $I8X16 mask_addr (ExtKind.None)))) - (value_reg (sse_and $I8X16 unmasked (RegMem.Reg mask))))) + (value_xmm (sse_and $I8X16 + unmasked + (xmm_mem_new (RegMem.Reg mask)))))) ;; Get the address of the mask to use when fixing up the lanes that weren't ;; correctly generated by the 16x8 shift. @@ -702,25 +711,28 @@ (extern constructor ushr_i8x16_mask_table ushr_i8x16_mask_table) (rule (ushr_i8x16_mask (RegMemImm.Reg amt)) (let ((mask_table SyntheticAmode (ushr_i8x16_mask_table)) - (base_mask_addr Reg (lea mask_table)) + (base_mask_addr Gpr (lea mask_table)) (mask_offset Reg (shl $I64 amt (Imm8Reg.Imm8 4)))) (amode_to_synthetic_amode (amode_imm_reg_reg_shift 0 base_mask_addr - mask_offset + (gpr_new mask_offset) 0)))) (rule (ushr_i8x16_mask (RegMemImm.Mem amt)) (ushr_i8x16_mask (RegMemImm.Reg (x64_load $I64 amt (ExtKind.None))))) ;; 16x8, 32x4, and 64x2 shifts can each use a single instruction. + (rule (lower (has_type $I16X8 (ushr src amt))) - (value_reg (psrlw (put_in_reg src) - (reg_mem_imm_to_xmm (put_in_reg_mem_imm amt))))) + (value_xmm (psrlw (put_in_xmm src) + (mov_rmi_to_xmm (put_in_reg_mem_imm amt))))) + (rule (lower (has_type $I32X4 (ushr src amt))) - (value_reg (psrld (put_in_reg src) - (reg_mem_imm_to_xmm (put_in_reg_mem_imm amt))))) + (value_xmm (psrld (put_in_xmm src) + (mov_rmi_to_xmm (put_in_reg_mem_imm amt))))) + (rule (lower (has_type $I64X2 (ushr src amt))) - (value_reg (psrlq (put_in_reg src) - (reg_mem_imm_to_xmm (put_in_reg_mem_imm amt))))) + (value_xmm (psrlq (put_in_xmm src) + (mov_rmi_to_xmm (put_in_reg_mem_imm amt))))) ;;;; Rules for `sshr` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -780,33 +792,35 @@ ;; shifted_hi.i16x8 = shift each lane of `high` ;; result = [s0'', s1'', ..., s15''] (rule (lower (has_type $I8X16 (sshr src amt @ (value_type amt_ty)))) - (let ((src_ Reg (put_in_reg src)) + (let ((src_ Xmm (put_in_xmm src)) ;; In order for `packsswb` later to only use the high byte of each ;; 16x8 lane, we shift right an extra 8 bits, relying on `psraw` to ;; fill in the upper bits appropriately. - (lo Reg (punpcklbw src_ (RegMem.Reg src_))) - (hi Reg (punpckhbw src_ (RegMem.Reg src_))) - (amt_ RegMemImm (sshr_i8x16_bigger_shift amt_ty (put_in_reg_mem_imm amt))) - (shifted_lo Reg (psraw lo amt_)) - (shifted_hi Reg (psraw hi amt_))) - (value_reg (packsswb shifted_lo (RegMem.Reg shifted_hi))))) - -(decl sshr_i8x16_bigger_shift (Type RegMemImm) RegMemImm) + (lo Xmm (punpcklbw src_ (xmm_to_xmm_mem src_))) + (hi Xmm (punpckhbw src_ (xmm_to_xmm_mem src_))) + (amt_ XmmMemImm (sshr_i8x16_bigger_shift amt_ty (put_in_reg_mem_imm amt))) + (shifted_lo Xmm (psraw lo amt_)) + (shifted_hi Xmm (psraw hi amt_))) + (value_xmm (packsswb shifted_lo (xmm_to_xmm_mem shifted_hi))))) + +(decl sshr_i8x16_bigger_shift (Type RegMemImm) XmmMemImm) (rule (sshr_i8x16_bigger_shift _ty (RegMemImm.Imm i)) - (RegMemImm.Imm (u32_add i 8))) + (xmm_mem_imm_new (RegMemImm.Imm (u32_add i 8)))) (rule (sshr_i8x16_bigger_shift ty (RegMemImm.Reg r)) - (reg_mem_imm_to_xmm (RegMemImm.Reg (add ty r (RegMemImm.Imm 8))))) + (mov_rmi_to_xmm (RegMemImm.Reg (add ty r (RegMemImm.Imm 8))))) (rule (sshr_i8x16_bigger_shift ty rmi @ (RegMemImm.Mem _m)) - (reg_mem_imm_to_xmm (RegMemImm.Reg (add ty (imm ty 8) rmi)))) + (mov_rmi_to_xmm (RegMemImm.Reg (add ty (imm ty 8) rmi)))) ;; `sshr.{i16x8,i32x4}` can be a simple `psra{w,d}`, we just have to make sure ;; that if the shift amount is in a register, it is in an XMM register. + (rule (lower (has_type $I16X8 (sshr src amt))) - (value_reg (psraw (put_in_reg src) - (reg_mem_imm_to_xmm (put_in_reg_mem_imm amt))))) + (value_xmm (psraw (put_in_xmm src) + (mov_rmi_to_xmm (put_in_reg_mem_imm amt))))) + (rule (lower (has_type $I32X4 (sshr src amt))) - (value_reg (psrad (put_in_reg src) - (reg_mem_imm_to_xmm (put_in_reg_mem_imm amt))))) + (value_xmm (psrad (put_in_xmm src) + (mov_rmi_to_xmm (put_in_reg_mem_imm amt))))) ;; The `sshr.i64x2` CLIF instruction has no single x86 instruction in the older ;; feature sets. Newer ones like AVX512VL + AVX512F include `vpsraq`, a 128-bit @@ -817,14 +831,15 @@ ;; ;; (TODO: when EVEX support is available, add an alternate lowering here). (rule (lower (has_type $I64X2 (sshr src amt))) - (let ((src_ Reg (put_in_reg src)) - (lo Reg (pextrd $I64 src_ 0)) - (hi Reg (pextrd $I64 src_ 1)) + (let ((src_ Xmm (put_in_xmm src)) + (lo Gpr (pextrd $I64 src_ 0)) + (hi Gpr (pextrd $I64 src_ 1)) (amt_ Imm8Reg (put_masked_in_imm8_reg amt $I64)) - (shifted_lo Reg (sar $I64 lo amt_)) - (shifted_hi Reg (sar $I64 hi amt_))) - (value_reg (make_i64x2_from_lanes (RegMem.Reg shifted_lo) - (RegMem.Reg shifted_hi))))) + (shifted_lo Reg (sar $I64 (gpr_to_reg lo) amt_)) + (shifted_hi Reg (sar $I64 (gpr_to_reg hi) amt_))) + (value_xmm (make_i64x2_from_lanes (gpr_mem_new (RegMem.Reg shifted_lo)) + (gpr_mem_new (RegMem.Reg shifted_hi)))))) + ;;;; Rules for `rotl` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; `i16` and `i8`: we need to extend the shift amount, or mask the @@ -910,35 +925,35 @@ ;; `i64` and smaller. (rule (lower (has_type (fits_in_64 ty) (ineg x))) - (value_reg (neg ty (put_in_reg x)))) + (value_gpr (neg ty (put_in_gpr x)))) ;; SSE. (rule (lower (has_type $I8X16 (ineg x))) - (value_reg (psubb (imm $I8X16 0) - (put_in_reg_mem x)))) + (value_xmm (psubb (xmm_new (imm $I8X16 0)) + (put_in_xmm_mem x)))) (rule (lower (has_type $I16X8 (ineg x))) - (value_reg (psubw (imm $I16X8 0) - (put_in_reg_mem x)))) + (value_xmm (psubw (xmm_new (imm $I16X8 0)) + (put_in_xmm_mem x)))) (rule (lower (has_type $I32X4 (ineg x))) - (value_reg (psubd (imm $I32X4 0) - (put_in_reg_mem x)))) + (value_xmm (psubd (xmm_new (imm $I32X4 0)) + (put_in_xmm_mem x)))) (rule (lower (has_type $I64X2 (ineg x))) - (value_reg (psubq (imm $I64X2 0) - (put_in_reg_mem x)))) + (value_xmm (psubq (xmm_new (imm $I64X2 0)) + (put_in_xmm_mem x)))) ;;;; Rules for `avg_round` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (rule (lower (has_type (multi_lane 8 16) (avg_round x y))) - (value_reg (pavgb (put_in_reg x) (put_in_reg_mem y)))) + (value_xmm (pavgb (put_in_xmm x) (put_in_xmm_mem y)))) (rule (lower (has_type (multi_lane 16 8) (avg_round x y))) - (value_reg (pavgw (put_in_reg x) (put_in_reg_mem y)))) + (value_xmm (pavgw (put_in_xmm x) (put_in_xmm_mem y)))) ;;;; Rules for `imul` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -1017,10 +1032,10 @@ ;; (No i8x16 multiply.) (rule (lower (has_type (multi_lane 16 8) (imul x y))) - (value_reg (pmullw (put_in_reg x) (put_in_reg_mem y)))) + (value_xmm (pmullw (put_in_xmm x) (put_in_xmm_mem y)))) (rule (lower (has_type (multi_lane 32 4) (imul x y))) - (value_reg (pmulld (put_in_reg x) (put_in_reg_mem y)))) + (value_xmm (pmulld (put_in_xmm x) (put_in_xmm_mem y)))) ;; With AVX-512 we can implement `i64x2` multiplication with a single ;; instruction. @@ -1028,7 +1043,7 @@ (avx512dq_enabled) (multi_lane 64 2)) (imul x y))) - (value_reg (vpmullq (put_in_reg_mem x) (put_in_reg y)))) + (value_xmm (vpmullq (put_in_xmm_mem x) (put_in_xmm y)))) ;; Otherwise, for i64x2 multiplication we describe a lane A as being composed of ;; a 32-bit upper half "Ah" and a 32-bit lower half "Al". The 32-bit long hand @@ -1052,24 +1067,24 @@ ;; 32-bits when doing calculations, i.e., `Ah == A >> 32`. (rule (lower (has_type (multi_lane 64 2) (imul a b))) - (let ((a0 Reg (put_in_reg a)) - (b0 Reg (put_in_reg b)) + (let ((a0 Xmm (put_in_xmm a)) + (b0 Xmm (put_in_xmm b)) ;; a_hi = A >> 32 - (a_hi Reg (psrlq a0 (RegMemImm.Imm 32))) + (a_hi Xmm (psrlq a0 (xmm_mem_imm_new (RegMemImm.Imm 32)))) ;; ah_bl = Ah * Bl - (ah_bl Reg (pmuludq a_hi (RegMem.Reg b0))) + (ah_bl Xmm (pmuludq a_hi (xmm_to_xmm_mem b0))) ;; b_hi = B >> 32 - (b_hi Reg (psrlq b0 (RegMemImm.Imm 32))) + (b_hi Xmm (psrlq b0 (xmm_mem_imm_new (RegMemImm.Imm 32)))) ;; al_bh = Al * Bh - (al_bh Reg (pmuludq a0 (RegMem.Reg b_hi))) + (al_bh Xmm (pmuludq a0 (xmm_to_xmm_mem b_hi))) ;; aa_bb = ah_bl + al_bh - (aa_bb Reg (paddq ah_bl (RegMem.Reg al_bh))) + (aa_bb Xmm (paddq ah_bl (xmm_to_xmm_mem al_bh))) ;; aa_bb_shifted = aa_bb << 32 - (aa_bb_shifted Reg (psllq aa_bb (RegMemImm.Imm 32))) + (aa_bb_shifted Xmm (psllq aa_bb (xmm_mem_imm_new (RegMemImm.Imm 32)))) ;; al_bl = Al * Bl - (al_bl Reg (pmuludq a0 (RegMem.Reg b0)))) + (al_bl Xmm (pmuludq a0 (xmm_to_xmm_mem b0)))) ;; al_bl + aa_bb_shifted - (value_reg (paddq al_bl (RegMem.Reg aa_bb_shifted))))) + (value_xmm (paddq al_bl (xmm_to_xmm_mem aa_bb_shifted))))) ;; Special case for `i16x8.extmul_high_i8x16_s`. (rule (lower (has_type (multi_lane 16 8) @@ -1077,13 +1092,13 @@ x))) (def_inst (swiden_high (and (value_type (multi_lane 8 16)) y)))))) - (let ((x1 Reg (put_in_reg x)) - (x2 Reg (palignr x1 (RegMem.Reg x1) 8 (OperandSize.Size32))) - (x3 Reg (pmovsxbw (RegMem.Reg x2))) - (y1 Reg (put_in_reg y)) - (y2 Reg (palignr y1 (RegMem.Reg y1) 8 (OperandSize.Size32))) - (y3 Reg (pmovsxbw (RegMem.Reg y2)))) - (value_reg (pmullw x3 (RegMem.Reg y3))))) + (let ((x1 Xmm (put_in_xmm x)) + (x2 Xmm (palignr x1 (xmm_to_xmm_mem x1) 8 (OperandSize.Size32))) + (x3 Xmm (pmovsxbw (xmm_to_xmm_mem x2))) + (y1 Xmm (put_in_xmm y)) + (y2 Xmm (palignr y1 (xmm_to_xmm_mem y1) 8 (OperandSize.Size32))) + (y3 Xmm (pmovsxbw (xmm_to_xmm_mem y2)))) + (value_xmm (pmullw x3 (xmm_to_xmm_mem y3))))) ;; Special case for `i32x4.extmul_high_i16x8_s`. (rule (lower (has_type (multi_lane 32 4) @@ -1091,11 +1106,11 @@ x))) (def_inst (swiden_high (and (value_type (multi_lane 16 8)) y)))))) - (let ((x2 Reg (put_in_reg x)) - (y2 Reg (put_in_reg y)) - (lo Reg (pmullw x2 (RegMem.Reg y2))) - (hi Reg (pmulhw x2 (RegMem.Reg y2)))) - (value_reg (punpckhwd lo (RegMem.Reg hi))))) + (let ((x2 Xmm (put_in_xmm x)) + (y2 Xmm (put_in_xmm y)) + (lo Xmm (pmullw x2 (xmm_to_xmm_mem y2))) + (hi Xmm (pmulhw x2 (xmm_to_xmm_mem y2)))) + (value_xmm (punpckhwd lo (xmm_to_xmm_mem hi))))) ;; Special case for `i64x2.extmul_high_i32x4_s`. (rule (lower (has_type (multi_lane 64 2) @@ -1103,13 +1118,13 @@ x))) (def_inst (swiden_high (and (value_type (multi_lane 32 4)) y)))))) - (let ((x2 Reg (pshufd (put_in_reg_mem x) + (let ((x2 Xmm (pshufd (put_in_xmm_mem x) 0xFA (OperandSize.Size32))) - (y2 Reg (pshufd (put_in_reg_mem y) + (y2 Xmm (pshufd (put_in_xmm_mem y) 0xFA (OperandSize.Size32)))) - (value_reg (pmuldq x2 (RegMem.Reg y2))))) + (value_xmm (pmuldq x2 (xmm_to_xmm_mem y2))))) ;; Special case for `i16x8.extmul_low_i8x16_s`. (rule (lower (has_type (multi_lane 16 8) @@ -1117,9 +1132,9 @@ x))) (def_inst (swiden_low (and (value_type (multi_lane 8 16)) y)))))) - (let ((x2 Reg (pmovsxbw (put_in_reg_mem x))) - (y2 Reg (pmovsxbw (put_in_reg_mem y)))) - (value_reg (pmullw x2 (RegMem.Reg y2))))) + (let ((x2 Xmm (pmovsxbw (put_in_xmm_mem x))) + (y2 Xmm (pmovsxbw (put_in_xmm_mem y)))) + (value_xmm (pmullw x2 (xmm_to_xmm_mem y2))))) ;; Special case for `i32x4.extmul_low_i16x8_s`. (rule (lower (has_type (multi_lane 32 4) @@ -1127,11 +1142,11 @@ x))) (def_inst (swiden_low (and (value_type (multi_lane 16 8)) y)))))) - (let ((x2 Reg (put_in_reg x)) - (y2 Reg (put_in_reg y)) - (lo Reg (pmullw x2 (RegMem.Reg y2))) - (hi Reg (pmulhw x2 (RegMem.Reg y2)))) - (value_reg (punpcklwd lo (RegMem.Reg hi))))) + (let ((x2 Xmm (put_in_xmm x)) + (y2 Xmm (put_in_xmm y)) + (lo Xmm (pmullw x2 (xmm_to_xmm_mem y2))) + (hi Xmm (pmulhw x2 (xmm_to_xmm_mem y2)))) + (value_xmm (punpcklwd lo (xmm_to_xmm_mem hi))))) ;; Special case for `i64x2.extmul_low_i32x4_s`. (rule (lower (has_type (multi_lane 64 2) @@ -1139,13 +1154,13 @@ x))) (def_inst (swiden_low (and (value_type (multi_lane 32 4)) y)))))) - (let ((x2 Reg (pshufd (put_in_reg_mem x) + (let ((x2 Xmm (pshufd (put_in_xmm_mem x) 0x50 (OperandSize.Size32))) - (y2 Reg (pshufd (put_in_reg_mem y) + (y2 Xmm (pshufd (put_in_xmm_mem y) 0x50 (OperandSize.Size32)))) - (value_reg (pmuldq x2 (RegMem.Reg y2))))) + (value_xmm (pmuldq x2 (xmm_to_xmm_mem y2))))) ;; Special case for `i16x8.extmul_high_i8x16_u`. (rule (lower (has_type (multi_lane 16 8) @@ -1153,13 +1168,13 @@ x))) (def_inst (uwiden_high (and (value_type (multi_lane 8 16)) y)))))) - (let ((x1 Reg (put_in_reg x)) - (x2 Reg (palignr x1 (RegMem.Reg x1) 8 (OperandSize.Size32))) - (x3 Reg (pmovzxbw (RegMem.Reg x2))) - (y1 Reg (put_in_reg y)) - (y2 Reg (palignr y1 (RegMem.Reg y1) 8 (OperandSize.Size32))) - (y3 Reg (pmovzxbw (RegMem.Reg y2)))) - (value_reg (pmullw x3 (RegMem.Reg y3))))) + (let ((x1 Xmm (put_in_xmm x)) + (x2 Xmm (palignr x1 (xmm_to_xmm_mem x1) 8 (OperandSize.Size32))) + (x3 Xmm (pmovzxbw (xmm_to_xmm_mem x2))) + (y1 Xmm (put_in_xmm y)) + (y2 Xmm (palignr y1 (xmm_to_xmm_mem y1) 8 (OperandSize.Size32))) + (y3 Xmm (pmovzxbw (xmm_to_xmm_mem y2)))) + (value_xmm (pmullw x3 (xmm_to_xmm_mem y3))))) ;; Special case for `i32x4.extmul_high_i16x8_u`. (rule (lower (has_type (multi_lane 32 4) @@ -1167,11 +1182,11 @@ x))) (def_inst (uwiden_high (and (value_type (multi_lane 16 8)) y)))))) - (let ((x2 Reg (put_in_reg x)) - (y2 Reg (put_in_reg y)) - (lo Reg (pmullw x2 (RegMem.Reg y2))) - (hi Reg (pmulhuw x2 (RegMem.Reg y2)))) - (value_reg (punpckhwd lo (RegMem.Reg hi))))) + (let ((x2 Xmm (put_in_xmm x)) + (y2 Xmm (put_in_xmm y)) + (lo Xmm (pmullw x2 (xmm_to_xmm_mem y2))) + (hi Xmm (pmulhuw x2 (xmm_to_xmm_mem y2)))) + (value_xmm (punpckhwd lo (xmm_to_xmm_mem hi))))) ;; Special case for `i64x2.extmul_high_i32x4_u`. (rule (lower (has_type (multi_lane 64 2) @@ -1179,13 +1194,13 @@ x))) (def_inst (uwiden_high (and (value_type (multi_lane 32 4)) y)))))) - (let ((x2 Reg (pshufd (put_in_reg_mem x) + (let ((x2 Xmm (pshufd (put_in_xmm_mem x) 0xFA (OperandSize.Size32))) - (y2 Reg (pshufd (put_in_reg_mem y) + (y2 Xmm (pshufd (put_in_xmm_mem y) 0xFA (OperandSize.Size32)))) - (value_reg (pmuludq x2 (RegMem.Reg y2))))) + (value_xmm (pmuludq x2 (xmm_to_xmm_mem y2))))) ;; Special case for `i16x8.extmul_low_i8x16_u`. (rule (lower (has_type (multi_lane 16 8) @@ -1193,9 +1208,9 @@ x))) (def_inst (uwiden_low (and (value_type (multi_lane 8 16)) y)))))) - (let ((x2 Reg (pmovzxbw (put_in_reg_mem x))) - (y2 Reg (pmovzxbw (put_in_reg_mem y)))) - (value_reg (pmullw x2 (RegMem.Reg y2))))) + (let ((x2 Xmm (pmovzxbw (put_in_xmm_mem x))) + (y2 Xmm (pmovzxbw (put_in_xmm_mem y)))) + (value_xmm (pmullw x2 (xmm_to_xmm_mem y2))))) ;; Special case for `i32x4.extmul_low_i16x8_u`. (rule (lower (has_type (multi_lane 32 4) @@ -1203,11 +1218,11 @@ x))) (def_inst (uwiden_low (and (value_type (multi_lane 16 8)) y)))))) - (let ((x2 Reg (put_in_reg x)) - (y2 Reg (put_in_reg y)) - (lo Reg (pmullw x2 (RegMem.Reg y2))) - (hi Reg (pmulhuw x2 (RegMem.Reg y2)))) - (value_reg (punpcklwd lo (RegMem.Reg hi))))) + (let ((x2 Xmm (put_in_xmm x)) + (y2 Xmm (put_in_xmm y)) + (lo Xmm (pmullw x2 (xmm_to_xmm_mem y2))) + (hi Xmm (pmulhuw x2 (xmm_to_xmm_mem y2)))) + (value_xmm (punpcklwd lo (xmm_to_xmm_mem hi))))) ;; Special case for `i64x2.extmul_low_i32x4_u`. (rule (lower (has_type (multi_lane 64 2) @@ -1215,17 +1230,17 @@ x))) (def_inst (uwiden_low (and (value_type (multi_lane 32 4)) y)))))) - (let ((x2 Reg (pshufd (put_in_reg_mem x) + (let ((x2 Xmm (pshufd (put_in_xmm_mem x) 0x50 (OperandSize.Size32))) - (y2 Reg (pshufd (put_in_reg_mem y) + (y2 Xmm (pshufd (put_in_xmm_mem y) 0x50 (OperandSize.Size32)))) - (value_reg (pmuludq x2 (RegMem.Reg y2))))) + (value_xmm (pmuludq x2 (xmm_to_xmm_mem y2))))) ;;;; Rules for `band_not` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -(decl sse_and_not (Type Reg RegMem) Reg) +(decl sse_and_not (Type Xmm XmmMem) Xmm) (rule (sse_and_not $F32X4 x y) (andnps x y)) (rule (sse_and_not $F64X2 x y) (andnpd x y)) (rule (sse_and_not (multi_lane _bits _lanes) x y) (pandn x y)) @@ -1238,64 +1253,66 @@ ;; ;; pandn(x, y) = and(not(x), y) (rule (lower (has_type ty (band_not x y))) - (value_reg (sse_and_not ty - (put_in_reg y) - (put_in_reg_mem x)))) + (value_xmm (sse_and_not ty + (put_in_xmm y) + (put_in_xmm_mem x)))) ;;;; Rules for `iabs` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (rule (lower (has_type $I8X16 (iabs x))) - (value_reg (pabsb (put_in_reg_mem x)))) + (value_xmm (pabsb (put_in_xmm_mem x)))) (rule (lower (has_type $I16X8 (iabs x))) - (value_reg (pabsw (put_in_reg_mem x)))) + (value_xmm (pabsw (put_in_xmm_mem x)))) (rule (lower (has_type $I32X4 (iabs x))) - (value_reg (pabsd (put_in_reg_mem x)))) + (value_xmm (pabsd (put_in_xmm_mem x)))) ;; When AVX512 is available, we can use a single `vpabsq` instruction. (rule (lower (has_type (and (avx512vl_enabled) (avx512f_enabled) $I64X2) (iabs x))) - (value_reg (vpabsq (put_in_reg_mem x)))) + (value_xmm (vpabsq (put_in_xmm_mem x)))) -;; Otherwise, we use a separate register, `neg`, to contain the results of `0 - +;; Otherwise, we use a separate xmmister, `neg`, to contain the results of `0 - ;; x` and then blend in those results with `blendvpd` if the MSB of `neg` was ;; set to 1 (i.e. if `neg` was negative or, conversely, if `x` was originally ;; positive). (rule (lower (has_type $I64X2 (iabs x))) - (let ((rx Reg (put_in_reg x)) - (neg Reg (psubq (imm $I64X2 0) (RegMem.Reg rx)))) - (value_reg (blendvpd neg (RegMem.Reg rx) neg)))) + (let ((rx Xmm (put_in_xmm x)) + (neg Xmm (psubq (xmm_new (imm $I64X2 0)) (xmm_to_xmm_mem rx)))) + (value_xmm (blendvpd neg (xmm_to_xmm_mem rx) neg)))) ;;;; Rules for `fabs` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; Special case for `f32x4.abs`. (rule (lower (has_type $F32X4 (fabs x))) - (value_reg (andps (put_in_reg x) - (RegMem.Reg (psrld (vector_all_ones $F32X4) (RegMemImm.Imm 1)))))) + (value_xmm (andps (put_in_xmm x) + (xmm_to_xmm_mem (psrld (vector_all_ones $F32X4) + (xmm_mem_imm_new (RegMemImm.Imm 1))))))) ;; Special case for `f64x2.abs`. (rule (lower (has_type $F64X2 (fabs x))) - (value_reg (andpd (put_in_reg x) - (RegMem.Reg (psrlq (vector_all_ones $F64X2) (RegMemImm.Imm 1)))))) + (value_xmm (andpd (put_in_xmm x) + (xmm_to_xmm_mem (psrlq (vector_all_ones $F64X2) + (xmm_mem_imm_new (RegMemImm.Imm 1))))))) ;;;; Rules for `bnot` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; `i64` and smaller. (rule (lower (has_type (fits_in_64 ty) (bnot x))) - (value_reg (not ty (put_in_reg x)))) + (value_gpr (not ty (put_in_gpr x)))) ;; `i128`. (decl i128_not (Value) ValueRegs) (rule (i128_not x) (let ((x_regs ValueRegs (put_in_regs x)) - (x_lo Reg (value_regs_get x_regs 0)) - (x_hi Reg (value_regs_get x_regs 1))) - (value_regs (not $I64 x_lo) + (x_lo Gpr (gpr_new (value_regs_get x_regs 0))) + (x_hi Gpr (gpr_new (value_regs_get x_regs 1)))) + (value_gprs (not $I64 x_lo) (not $I64 x_hi)))) (rule (lower (has_type $I128 (bnot x))) @@ -1307,7 +1324,7 @@ ;; Special case for vector-types where bit-negation is an xor against an ;; all-one value (rule (lower (has_type ty @ (multi_lane _bits _lanes) (bnot x))) - (value_reg (sse_xor ty (put_in_reg x) (RegMem.Reg (vector_all_ones ty))))) + (value_xmm (sse_xor ty (put_in_xmm x) (xmm_to_xmm_mem (vector_all_ones ty))))) ;;;; Rules for `bitselect` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -1318,48 +1335,53 @@ ;; a = and if_true, condition ;; b = and_not condition, if_false ;; or b, a - (let ((cond_reg Reg (put_in_reg condition)) - (a Reg (sse_and ty (put_in_reg if_true) (RegMem.Reg cond_reg))) - (b Reg (sse_and_not ty cond_reg (put_in_reg_mem if_false)))) - (value_reg (sse_or ty b (RegMem.Reg a))))) + (let ((cond_xmm Xmm (put_in_xmm condition)) + (a Xmm (sse_and ty (put_in_xmm if_true) (xmm_to_xmm_mem cond_xmm))) + (b Xmm (sse_and_not ty cond_xmm (put_in_xmm_mem if_false)))) + (value_xmm (sse_or ty b (xmm_to_xmm_mem a))))) ;;;; Rules for `vselect` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (rule (lower (has_type ty @ (multi_lane _bits _lanes) (vselect condition if_true if_false))) - (value_reg (sse_blend ty - (put_in_reg_mem condition) - (put_in_reg_mem if_true) - (put_in_reg if_false)))) + (value_xmm (sse_blend ty + (put_in_xmm_mem condition) + (put_in_xmm_mem if_true) + (put_in_xmm if_false)))) ;;;; Rules for `insertlane` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (rule (lower (insertlane vec @ (value_type ty) val (u8_from_uimm8 idx))) - (value_reg (vec_insert_lane ty (put_in_reg vec) (put_in_reg_mem val) idx))) + (value_xmm (vec_insert_lane ty (put_in_xmm vec) (put_in_reg_mem val) idx))) ;; Helper function used below for `insertlane` but also here for other ;; lowerings. ;; ;; Note that the `Type` used here is the type of vector the insertion is ;; happening into, or the type of the first `Reg` argument. -(decl vec_insert_lane (Type Reg RegMem u8) Reg) +(decl vec_insert_lane (Type Xmm RegMem u8) Xmm) ;; i8x16.replace_lane -(rule (vec_insert_lane $I8X16 vec val idx) (pinsrb vec val idx)) +(rule (vec_insert_lane $I8X16 vec val idx) + (pinsrb vec (gpr_mem_new val) idx)) ;; i16x8.replace_lane -(rule (vec_insert_lane $I16X8 vec val idx) (pinsrw vec val idx)) +(rule (vec_insert_lane $I16X8 vec val idx) + (pinsrw vec (gpr_mem_new val) idx)) ;; i32x4.replace_lane -(rule (vec_insert_lane $I32X4 vec val idx) (pinsrd vec val idx (OperandSize.Size32))) +(rule (vec_insert_lane $I32X4 vec val idx) + (pinsrd vec (gpr_mem_new val) idx (OperandSize.Size32))) ;; i64x2.replace_lane -(rule (vec_insert_lane $I64X2 vec val idx) (pinsrd vec val idx (OperandSize.Size64))) +(rule (vec_insert_lane $I64X2 vec val idx) + (pinsrd vec (gpr_mem_new val) idx (OperandSize.Size64))) ;; f32x4.replace_lane -(rule (vec_insert_lane $F32X4 vec val idx) (insertps vec val (sse_insertps_lane_imm idx))) +(rule (vec_insert_lane $F32X4 vec val idx) + (insertps vec (xmm_mem_new val) (sse_insertps_lane_imm idx))) -;; external rust code used to calculate the immediate value to `insertps` +;; External rust code used to calculate the immediate value to `insertps`. (decl sse_insertps_lane_imm (u8) u8) (extern constructor sse_insertps_lane_imm sse_insertps_lane_imm) @@ -1378,60 +1400,63 @@ ;; load from memory into a temp register and then the second `movsd` (modeled ;; internally as `xmm_rm_r` will merge the temp register into our `vec` ;; register. -(rule (vec_insert_lane $F64X2 vec (RegMem.Reg val) 0) (movsd vec (RegMem.Reg val))) +(rule (vec_insert_lane $F64X2 vec (RegMem.Reg val) 0) + (movsd vec (xmm_mem_new (RegMem.Reg val)))) (rule (vec_insert_lane $F64X2 vec mem 0) - (movsd vec (RegMem.Reg (xmm_unary_rm_r (SseOpcode.Movsd) mem)))) + (movsd vec (xmm_to_xmm_mem (xmm_unary_rm_r (SseOpcode.Movsd) + (xmm_mem_new mem))))) ;; f64x2.replace_lane 1 ;; ;; Here the `movlhps` instruction is used specifically to specialize moving ;; into the second lane where unlike above cases we're not using the lane ;; immediate as an immediate to the instruction itself. -(rule (vec_insert_lane $F64X2 vec val 1) (movlhps vec val)) +(rule (vec_insert_lane $F64X2 vec val 1) + (movlhps vec (xmm_mem_new val))) ;;;; Rules for `imax` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (rule (lower (has_type $I8X16 (imax x y))) - (value_reg (pmaxsb (put_in_reg x) (put_in_reg_mem y)))) + (value_xmm (pmaxsb (put_in_xmm x) (put_in_xmm_mem y)))) (rule (lower (has_type $I16X8 (imax x y))) - (value_reg (pmaxsw (put_in_reg x) (put_in_reg_mem y)))) + (value_xmm (pmaxsw (put_in_xmm x) (put_in_xmm_mem y)))) (rule (lower (has_type $I32X4 (imax x y))) - (value_reg (pmaxsd (put_in_reg x) (put_in_reg_mem y)))) + (value_xmm (pmaxsd (put_in_xmm x) (put_in_xmm_mem y)))) ;;;; Rules for `imin` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (rule (lower (has_type $I8X16 (imin x y))) - (value_reg (pminsb (put_in_reg x) (put_in_reg_mem y)))) + (value_xmm (pminsb (put_in_xmm x) (put_in_xmm_mem y)))) (rule (lower (has_type $I16X8 (imin x y))) - (value_reg (pminsw (put_in_reg x) (put_in_reg_mem y)))) + (value_xmm (pminsw (put_in_xmm x) (put_in_xmm_mem y)))) (rule (lower (has_type $I32X4 (imin x y))) - (value_reg (pminsd (put_in_reg x) (put_in_reg_mem y)))) + (value_xmm (pminsd (put_in_xmm x) (put_in_xmm_mem y)))) ;;;; Rules for `umax` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (rule (lower (has_type $I8X16 (umax x y))) - (value_reg (pmaxub (put_in_reg x) (put_in_reg_mem y)))) + (value_xmm (pmaxub (put_in_xmm x) (put_in_xmm_mem y)))) (rule (lower (has_type $I16X8 (umax x y))) - (value_reg (pmaxuw (put_in_reg x) (put_in_reg_mem y)))) + (value_xmm (pmaxuw (put_in_xmm x) (put_in_xmm_mem y)))) (rule (lower (has_type $I32X4 (umax x y))) - (value_reg (pmaxud (put_in_reg x) (put_in_reg_mem y)))) + (value_xmm (pmaxud (put_in_xmm x) (put_in_xmm_mem y)))) ;;;; Rules for `umin` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (rule (lower (has_type $I8X16 (umin x y))) - (value_reg (pminub (put_in_reg x) (put_in_reg_mem y)))) + (value_xmm (pminub (put_in_xmm x) (put_in_xmm_mem y)))) (rule (lower (has_type $I16X8 (umin x y))) - (value_reg (pminuw (put_in_reg x) (put_in_reg_mem y)))) + (value_xmm (pminuw (put_in_xmm x) (put_in_xmm_mem y)))) (rule (lower (has_type $I32X4 (umin x y))) - (value_reg (pminud (put_in_reg x) (put_in_reg_mem y)))) + (value_xmm (pminud (put_in_xmm x) (put_in_xmm_mem y)))) ;;;; Rules for `trap` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; diff --git a/cranelift/codegen/src/isa/x64/lower.rs b/cranelift/codegen/src/isa/x64/lower.rs index d5503416de9f..3e74dddda93d 100644 --- a/cranelift/codegen/src/isa/x64/lower.rs +++ b/cranelift/codegen/src/isa/x64/lower.rs @@ -1,7 +1,7 @@ //! Lowering rules for X64. // ISLE integration glue. -mod isle; +pub(super) mod isle; use crate::data_value::DataValue; use crate::ir::{ @@ -1057,7 +1057,13 @@ fn lower_to_amode>(ctx: &mut C, spec: InsnInput, offset: i ) }; - return Amode::imm_reg_reg_shift(offset as u32, base, index, shift).with_flags(flags); + return Amode::imm_reg_reg_shift( + offset as u32, + Gpr::new(base).unwrap(), + Gpr::new(index).unwrap(), + shift, + ) + .with_flags(flags); } let input = put_input_in_reg(ctx, spec); @@ -3950,7 +3956,13 @@ fn lower_insn_to_regs>( let index = put_input_in_reg(ctx, inputs[1]); let shift = 0; let flags = ctx.memflags(insn).expect("load should have memflags"); - Amode::imm_reg_reg_shift(offset as u32, base, index, shift).with_flags(flags) + Amode::imm_reg_reg_shift( + offset as u32, + Gpr::new(base).unwrap(), + Gpr::new(index).unwrap(), + shift, + ) + .with_flags(flags) } _ => unreachable!(), }; @@ -4054,7 +4066,13 @@ fn lower_insn_to_regs>( let index = put_input_in_reg(ctx, inputs[2]); let shift = 0; let flags = ctx.memflags(insn).expect("store should have memflags"); - Amode::imm_reg_reg_shift(offset as u32, base, index, shift).with_flags(flags) + Amode::imm_reg_reg_shift( + offset as u32, + Gpr::new(base).unwrap(), + Gpr::new(index).unwrap(), + shift, + ) + .with_flags(flags) } _ => unreachable!(), diff --git a/cranelift/codegen/src/isa/x64/lower/isle.rs b/cranelift/codegen/src/isa/x64/lower/isle.rs index c6bd7651b413..3aff4de79037 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle.rs +++ b/cranelift/codegen/src/isa/x64/lower/isle.rs @@ -1,26 +1,28 @@ //! ISLE integration glue code for x64 lowering. // Pull in the ISLE generated code. -mod generated_code; +pub(crate) mod generated_code; +use generated_code::MInst; +use regalloc::Writable; // Types that the generated ISLE code uses via `use super::*`. -use super::{ - is_mergeable_load, lower_to_amode, AluRmiROpcode, Inst as MInst, OperandSize, Reg, RegMemImm, -}; +use super::{is_mergeable_load, lower_to_amode, Reg}; use crate::{ - ir::{immediates::*, types::*, Inst, InstructionData, Opcode, TrapCode, Value, ValueList}, - isa::x64::{ - inst::{ - args::{ - Amode, Avx512Opcode, CmpOpcode, ExtKind, ExtMode, FcmpImm, Imm8Reg, RegMem, - ShiftKind, SseOpcode, SyntheticAmode, CC, - }, - regs, x64_map_regs, + ir::{ + immediates::*, types::*, Inst, InstructionData, Opcode, TrapCode, Value, ValueLabel, + ValueList, + }, + isa::{ + settings::Flags, + unwind::UnwindInst, + x64::{ + inst::{args::*, regs, x64_map_regs}, + settings::Flags as IsaFlags, }, - settings::Flags as IsaFlags, }, - machinst::{isle::*, InsnInput, InsnOutput, LowerCtx, VCodeConstantData}, - settings::Flags, + machinst::{ + isle::*, AtomicRmwOp, InsnInput, InsnOutput, LowerCtx, VCodeConstant, VCodeConstantData, + }, }; use std::convert::TryFrom; @@ -248,8 +250,8 @@ where } #[inline] - fn xmm0(&mut self) -> WritableReg { - WritableReg::from_reg(regs::xmm0()) + fn xmm0(&mut self) -> WritableXmm { + WritableXmm::from_reg(Xmm::new(regs::xmm0()).unwrap()) } #[inline] @@ -258,7 +260,7 @@ where } #[inline] - fn amode_imm_reg_reg_shift(&mut self, simm32: u32, base: Reg, index: Reg, shift: u8) -> Amode { + fn amode_imm_reg_reg_shift(&mut self, simm32: u32, base: Gpr, index: Gpr, shift: u8) -> Amode { Amode::imm_reg_reg_shift(simm32, base, index, shift) } @@ -267,6 +269,16 @@ where amode.clone().into() } + #[inline] + fn writable_gpr_to_reg(&mut self, r: WritableGpr) -> WritableReg { + r.to_writable_reg() + } + + #[inline] + fn writable_xmm_to_reg(&mut self, r: WritableXmm) -> WritableReg { + r.to_writable_reg() + } + fn ishl_i8x16_mask_for_const(&mut self, amt: u32) -> SyntheticAmode { // When the shift amount is known, we can statically (i.e. at compile // time) determine the mask to use and only emit that. @@ -302,6 +314,96 @@ where .use_constant(VCodeConstantData::WellKnown(&I8X16_USHR_MASKS)); SyntheticAmode::ConstantOffset(mask_table) } + + #[inline] + fn writable_reg_to_xmm(&mut self, r: WritableReg) -> WritableXmm { + Writable::from_reg(Xmm::new(r.to_reg()).unwrap()) + } + + #[inline] + fn writable_xmm_to_xmm(&mut self, r: WritableXmm) -> Xmm { + r.to_reg() + } + + #[inline] + fn writable_gpr_to_gpr(&mut self, r: WritableGpr) -> Gpr { + r.to_reg() + } + + #[inline] + fn gpr_to_reg(&mut self, r: Gpr) -> Reg { + r.into() + } + + #[inline] + fn xmm_to_reg(&mut self, r: Xmm) -> Reg { + r.into() + } + + #[inline] + fn xmm_to_xmm_mem_imm(&mut self, r: Xmm) -> XmmMemImm { + r.into() + } + + #[inline] + fn temp_writable_gpr(&mut self) -> WritableGpr { + Writable::from_reg(Gpr::new(self.temp_writable_reg(I64).to_reg()).unwrap()) + } + + #[inline] + fn temp_writable_xmm(&mut self) -> WritableXmm { + Writable::from_reg(Xmm::new(self.temp_writable_reg(I8X16).to_reg()).unwrap()) + } + + #[inline] + fn xmm_mem_new(&mut self, rm: &RegMem) -> XmmMem { + XmmMem::new(rm.clone()).unwrap() + } + + #[inline] + fn gpr_mem_imm_new(&mut self, rmi: &RegMemImm) -> GprMemImm { + GprMemImm::new(rmi.clone()).unwrap() + } + + #[inline] + fn xmm_mem_imm_new(&mut self, rmi: &RegMemImm) -> XmmMemImm { + XmmMemImm::new(rmi.clone()).unwrap() + } + + #[inline] + fn xmm_to_xmm_mem(&mut self, r: Xmm) -> XmmMem { + r.into() + } + + #[inline] + fn xmm_mem_to_reg_mem(&mut self, xm: &XmmMem) -> RegMem { + xm.clone().into() + } + + #[inline] + fn gpr_mem_to_reg_mem(&mut self, gm: &GprMem) -> RegMem { + gm.clone().into() + } + + #[inline] + fn xmm_new(&mut self, r: Reg) -> Xmm { + Xmm::new(r).unwrap() + } + + #[inline] + fn gpr_new(&mut self, r: Reg) -> Gpr { + Gpr::new(r).unwrap() + } + + #[inline] + fn gpr_mem_new(&mut self, rm: &RegMem) -> GprMem { + GprMem::new(rm.clone()).unwrap() + } + + #[inline] + fn reg_to_gpr_mem(&mut self, r: Reg) -> GprMem { + GprMem::new(RegMem::reg(r)).unwrap() + } } // Since x64 doesn't have 8x16 shifts and we must use a 16x8 shift instead, we diff --git a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest index 6d2b8256ac8d..a1e5d89a632c 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest +++ b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest @@ -1,4 +1,4 @@ src/clif.isle 9ea75a6f790b5c03 -src/prelude.isle 51d2aef2566c1c96 -src/isa/x64/inst.isle 61004acbb1289816 -src/isa/x64/lower.isle 82db7f7d47ac7809 +src/prelude.isle 3d00cfa50c4d52d9 +src/isa/x64/inst.isle 1e5ee6f51e890a17 +src/isa/x64/lower.isle 144c33c4e64a17a7 diff --git a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs index 9c2ad6974087..64f7149ef56f 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs +++ b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs @@ -68,11 +68,31 @@ pub trait Context { fn put_in_reg_mem_imm(&mut self, arg0: Value) -> RegMemImm; fn put_in_reg_mem(&mut self, arg0: Value) -> RegMem; fn synthetic_amode_to_reg_mem(&mut self, arg0: &SyntheticAmode) -> RegMem; - fn amode_imm_reg_reg_shift(&mut self, arg0: u32, arg1: Reg, arg2: Reg, arg3: u8) -> Amode; + fn amode_imm_reg_reg_shift(&mut self, arg0: u32, arg1: Gpr, arg2: Gpr, arg3: u8) -> Amode; fn amode_to_synthetic_amode(&mut self, arg0: &Amode) -> SyntheticAmode; fn put_masked_in_imm8_reg(&mut self, arg0: Value, arg1: Type) -> Imm8Reg; fn encode_fcmp_imm(&mut self, arg0: &FcmpImm) -> u8; - fn xmm0(&mut self) -> WritableReg; + fn writable_gpr_to_reg(&mut self, arg0: WritableGpr) -> WritableReg; + fn writable_xmm_to_reg(&mut self, arg0: WritableXmm) -> WritableReg; + fn writable_reg_to_xmm(&mut self, arg0: WritableReg) -> WritableXmm; + fn writable_xmm_to_xmm(&mut self, arg0: WritableXmm) -> Xmm; + fn writable_gpr_to_gpr(&mut self, arg0: WritableGpr) -> Gpr; + fn gpr_to_reg(&mut self, arg0: Gpr) -> Reg; + fn xmm_to_reg(&mut self, arg0: Xmm) -> Reg; + fn xmm_to_xmm_mem_imm(&mut self, arg0: Xmm) -> XmmMemImm; + fn temp_writable_gpr(&mut self) -> WritableGpr; + fn temp_writable_xmm(&mut self) -> WritableXmm; + fn xmm_mem_new(&mut self, arg0: &RegMem) -> XmmMem; + fn gpr_mem_imm_new(&mut self, arg0: &RegMemImm) -> GprMemImm; + fn xmm_mem_imm_new(&mut self, arg0: &RegMemImm) -> XmmMemImm; + fn xmm_to_xmm_mem(&mut self, arg0: Xmm) -> XmmMem; + fn xmm_mem_to_reg_mem(&mut self, arg0: &XmmMem) -> RegMem; + fn gpr_mem_to_reg_mem(&mut self, arg0: &GprMem) -> RegMem; + fn xmm_new(&mut self, arg0: Reg) -> Xmm; + fn gpr_new(&mut self, arg0: Reg) -> Gpr; + fn gpr_mem_new(&mut self, arg0: &RegMem) -> GprMem; + fn reg_to_gpr_mem(&mut self, arg0: Reg) -> GprMem; + fn xmm0(&mut self) -> WritableXmm; fn avx512vl_enabled(&mut self, arg0: Type) -> Option<()>; fn avx512dq_enabled(&mut self, arg0: Type) -> Option<()>; fn avx512f_enabled(&mut self, arg0: Type) -> Option<()>; @@ -92,26 +112,337 @@ pub trait Context { fn sse_insertps_lane_imm(&mut self, arg0: u8) -> u8; } -/// Internal type SideEffectNoResult: defined at src/prelude.isle line 282. +/// Internal type SideEffectNoResult: defined at src/prelude.isle line 294. #[derive(Clone, Debug)] pub enum SideEffectNoResult { Inst { inst: MInst }, } -/// Internal type ProducesFlags: defined at src/prelude.isle line 295. +/// Internal type ProducesFlags: defined at src/prelude.isle line 307. #[derive(Clone, Debug)] pub enum ProducesFlags { ProducesFlags { inst: MInst, result: Reg }, } -/// Internal type ConsumesFlags: defined at src/prelude.isle line 298. +/// Internal type ConsumesFlags: defined at src/prelude.isle line 310. #[derive(Clone, Debug)] pub enum ConsumesFlags { ConsumesFlags { inst: MInst, result: Reg }, } -/// Internal type ExtendKind: defined at src/isa/x64/inst.isle line 464. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type MInst: defined at src/isa/x64/inst.isle line 8. +#[derive(Clone)] +pub enum MInst { + Nop { + len: u8, + }, + AluRmiR { + size: OperandSize, + op: AluRmiROpcode, + src1: Reg, + src2: RegMemImm, + dst: WritableReg, + }, + UnaryRmR { + size: OperandSize, + op: UnaryRmROpcode, + src: GprMem, + dst: WritableGpr, + }, + Not { + size: OperandSize, + src: Gpr, + dst: WritableGpr, + }, + Neg { + size: OperandSize, + src: Gpr, + dst: WritableGpr, + }, + Div { + size: OperandSize, + signed: bool, + divisor: RegMem, + dividend: Reg, + dst_quotient: WritableReg, + dst_remainder: WritableReg, + }, + MulHi { + size: OperandSize, + signed: bool, + src1: Reg, + src2: RegMem, + dst_lo: WritableReg, + dst_hi: WritableReg, + }, + CheckedDivOrRemSeq { + kind: DivOrRemKind, + size: OperandSize, + dividend: Reg, + divisor: WritableReg, + dst_quotient: WritableReg, + dst_remainder: WritableReg, + tmp: OptionWritableReg, + }, + SignExtendData { + size: OperandSize, + src: Reg, + dst: WritableReg, + }, + Imm { + dst_size: OperandSize, + simm64: u64, + dst: WritableReg, + }, + MovRR { + size: OperandSize, + src: Reg, + dst: WritableReg, + }, + MovzxRmR { + ext_mode: ExtMode, + src: RegMem, + dst: WritableReg, + }, + Mov64MR { + src: SyntheticAmode, + dst: WritableReg, + }, + LoadEffectiveAddress { + addr: SyntheticAmode, + dst: WritableGpr, + }, + MovsxRmR { + ext_mode: ExtMode, + src: RegMem, + dst: WritableReg, + }, + MovRM { + size: OperandSize, + src: Reg, + dst: SyntheticAmode, + }, + ShiftR { + size: OperandSize, + kind: ShiftKind, + src: Reg, + num_bits: Imm8Reg, + dst: WritableReg, + }, + XmmRmiReg { + opcode: SseOpcode, + src1: Xmm, + src2: XmmMemImm, + dst: WritableXmm, + }, + CmpRmiR { + size: OperandSize, + opcode: CmpOpcode, + src: RegMemImm, + dst: Reg, + }, + Setcc { + cc: CC, + dst: WritableReg, + }, + Cmove { + size: OperandSize, + cc: CC, + consequent: RegMem, + alternative: Reg, + dst: WritableReg, + }, + Push64 { + src: RegMemImm, + }, + Pop64 { + dst: WritableReg, + }, + XmmRmR { + op: SseOpcode, + src1: Xmm, + src2: XmmMem, + dst: WritableXmm, + }, + XmmRmREvex { + op: Avx512Opcode, + src1: XmmMem, + src2: Xmm, + dst: WritableXmm, + }, + XmmUnaryRmR { + op: SseOpcode, + src: XmmMem, + dst: WritableXmm, + }, + XmmUnaryRmREvex { + op: Avx512Opcode, + src: XmmMem, + dst: WritableXmm, + }, + XmmMovRM { + op: SseOpcode, + src: Reg, + dst: SyntheticAmode, + }, + XmmLoadConst { + src: VCodeConstant, + dst: WritableReg, + ty: Type, + }, + XmmToGpr { + op: SseOpcode, + src: Xmm, + dst: WritableGpr, + dst_size: OperandSize, + }, + GprToXmm { + op: SseOpcode, + src: RegMem, + dst: WritableXmm, + src_size: OperandSize, + }, + CvtUint64ToFloatSeq { + dst_size: OperandSize, + src: WritableGpr, + dst: WritableXmm, + tmp_gpr1: WritableGpr, + tmp_gpr2: WritableGpr, + }, + CvtFloatToSintSeq { + dst_size: OperandSize, + src_size: OperandSize, + is_saturating: bool, + src: WritableXmm, + dst: WritableGpr, + tmp_gpr: WritableGpr, + tmp_xmm: WritableXmm, + }, + CvtFloatToUintSeq { + dst_size: OperandSize, + src_size: OperandSize, + is_saturating: bool, + src: WritableXmm, + dst: WritableGpr, + tmp_gpr: WritableGpr, + tmp_xmm: WritableXmm, + }, + XmmMinMaxSeq { + size: OperandSize, + is_min: bool, + lhs: Reg, + rhs_dst: WritableReg, + }, + XmmCmove { + size: OperandSize, + cc: CC, + src: RegMem, + dst: WritableReg, + }, + XmmCmpRmR { + op: SseOpcode, + src: RegMem, + dst: Reg, + }, + XmmRmRImm { + op: SseOpcode, + src1: Reg, + src2: RegMem, + dst: WritableReg, + imm: u8, + size: OperandSize, + }, + CallKnown { + dest: ExternalName, + uses: VecReg, + defs: VecWritableReg, + opcode: Opcode, + }, + CallUnknown { + dest: RegMem, + uses: VecReg, + defs: VecWritableReg, + opcode: Opcode, + }, + Ret, + EpiloguePlaceholder, + JmpKnown { + dst: MachLabel, + }, + JmpIf { + cc: CC, + taken: MachLabel, + }, + JmpCond { + cc: CC, + taken: MachLabel, + not_taken: MachLabel, + }, + JmpTableSeq { + idx: Reg, + tmp1: WritableReg, + tmp2: WritableReg, + default_target: MachLabel, + targets: VecMachLabel, + targets_for_term: VecMachLabel, + }, + JmpUnknown { + target: RegMem, + }, + TrapIf { + cc: CC, + trap_code: TrapCode, + }, + Hlt, + Ud2 { + trap_code: TrapCode, + }, + LoadExtName { + dst: WritableReg, + name: BoxExternalName, + offset: i64, + }, + LockCmpxchg { + ty: Type, + replacement: Reg, + expected: Reg, + mem: SyntheticAmode, + dst_old: WritableReg, + }, + AtomicRmwSeq { + ty: Type, + op: AtomicRmwOp, + address: Reg, + operand: Reg, + temp: WritableReg, + dst_old: WritableReg, + }, + Fence { + kind: FenceKind, + }, + VirtualSPOffsetAdj { + offset: i64, + }, + XmmUninitializedValue { + dst: WritableXmm, + }, + ElfTlsGetAddr { + symbol: ExternalName, + }, + MachOTlsGetAddr { + symbol: ExternalName, + }, + ValueLabelMarker { + reg: Reg, + label: ValueLabel, + }, + Unwind { + inst: UnwindInst, + }, +} + +/// Internal type ExtendKind: defined at src/isa/x64/inst.isle line 1051. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum ExtendKind { Sign, Zero, @@ -120,7 +451,7 @@ pub enum ExtendKind { // Generated as internal constructor for term temp_reg. pub fn constructor_temp_reg(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; - // Rule at src/prelude.isle line 70. + // Rule at src/prelude.isle line 73. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); return Some(expr1_0); @@ -129,7 +460,7 @@ pub fn constructor_temp_reg(ctx: &mut C, arg0: Type) -> Option // Generated as internal constructor for term lo_reg. pub fn constructor_lo_reg(ctx: &mut C, arg0: Value) -> Option { let pattern0_0 = arg0; - // Rule at src/prelude.isle line 105. + // Rule at src/prelude.isle line 108. let expr0_0 = C::put_in_regs(ctx, pattern0_0); let expr1_0: usize = 0; let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); @@ -146,7 +477,7 @@ pub fn constructor_value_regs_none( inst: ref pattern1_0, } = pattern0_0 { - // Rule at src/prelude.isle line 287. + // Rule at src/prelude.isle line 299. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::value_regs_invalid(ctx); return Some(expr1_0); @@ -172,7 +503,7 @@ pub fn constructor_with_flags( result: pattern3_1, } = pattern2_0 { - // Rule at src/prelude.isle line 308. + // Rule at src/prelude.isle line 320. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern3_0); let expr2_0 = C::value_regs(ctx, pattern1_1, pattern3_1); @@ -200,7 +531,7 @@ pub fn constructor_with_flags_1( result: pattern3_1, } = pattern2_0 { - // Rule at src/prelude.isle line 316. + // Rule at src/prelude.isle line 328. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern3_0); return Some(pattern3_1); @@ -234,7 +565,7 @@ pub fn constructor_with_flags_2( result: pattern5_1, } = pattern4_0 { - // Rule at src/prelude.isle line 326. + // Rule at src/prelude.isle line 338. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern5_0); let expr2_0 = C::emit(ctx, &pattern3_0); @@ -251,22 +582,22 @@ pub fn constructor_operand_size_bits(ctx: &mut C, arg0: &OperandSize let pattern0_0 = arg0; match pattern0_0 { &OperandSize::Size8 => { - // Rule at src/isa/x64/inst.isle line 98. + // Rule at src/isa/x64/inst.isle line 509. let expr0_0: u16 = 8; return Some(expr0_0); } &OperandSize::Size16 => { - // Rule at src/isa/x64/inst.isle line 99. + // Rule at src/isa/x64/inst.isle line 510. let expr0_0: u16 = 16; return Some(expr0_0); } &OperandSize::Size32 => { - // Rule at src/isa/x64/inst.isle line 100. + // Rule at src/isa/x64/inst.isle line 511. let expr0_0: u16 = 32; return Some(expr0_0); } &OperandSize::Size64 => { - // Rule at src/isa/x64/inst.isle line 101. + // Rule at src/isa/x64/inst.isle line 512. let expr0_0: u16 = 64; return Some(expr0_0); } @@ -275,6 +606,89 @@ pub fn constructor_operand_size_bits(ctx: &mut C, arg0: &OperandSize return None; } +// Generated as internal constructor for term put_in_gpr. +pub fn constructor_put_in_gpr(ctx: &mut C, arg0: Value) -> Option { + let pattern0_0 = arg0; + // Rule at src/isa/x64/inst.isle line 934. + let expr0_0 = C::put_in_reg(ctx, pattern0_0); + let expr1_0 = C::gpr_new(ctx, expr0_0); + return Some(expr1_0); +} + +// Generated as internal constructor for term put_in_gpr_mem. +pub fn constructor_put_in_gpr_mem(ctx: &mut C, arg0: Value) -> Option { + let pattern0_0 = arg0; + // Rule at src/isa/x64/inst.isle line 941. + let expr0_0 = C::put_in_reg_mem(ctx, pattern0_0); + let expr1_0 = C::gpr_mem_new(ctx, &expr0_0); + return Some(expr1_0); +} + +// Generated as internal constructor for term put_in_gpr_mem_imm. +pub fn constructor_put_in_gpr_mem_imm(ctx: &mut C, arg0: Value) -> Option { + let pattern0_0 = arg0; + // Rule at src/isa/x64/inst.isle line 948. + let expr0_0 = C::put_in_reg_mem_imm(ctx, pattern0_0); + let expr1_0 = C::gpr_mem_imm_new(ctx, &expr0_0); + return Some(expr1_0); +} + +// Generated as internal constructor for term put_in_xmm. +pub fn constructor_put_in_xmm(ctx: &mut C, arg0: Value) -> Option { + let pattern0_0 = arg0; + // Rule at src/isa/x64/inst.isle line 955. + let expr0_0 = C::put_in_reg(ctx, pattern0_0); + let expr1_0 = C::xmm_new(ctx, expr0_0); + return Some(expr1_0); +} + +// Generated as internal constructor for term put_in_xmm_mem. +pub fn constructor_put_in_xmm_mem(ctx: &mut C, arg0: Value) -> Option { + let pattern0_0 = arg0; + // Rule at src/isa/x64/inst.isle line 962. + let expr0_0 = C::put_in_reg_mem(ctx, pattern0_0); + let expr1_0 = C::xmm_mem_new(ctx, &expr0_0); + return Some(expr1_0); +} + +// Generated as internal constructor for term put_in_xmm_mem_imm. +pub fn constructor_put_in_xmm_mem_imm(ctx: &mut C, arg0: Value) -> Option { + let pattern0_0 = arg0; + // Rule at src/isa/x64/inst.isle line 969. + let expr0_0 = C::put_in_reg_mem_imm(ctx, pattern0_0); + let expr1_0 = C::xmm_mem_imm_new(ctx, &expr0_0); + return Some(expr1_0); +} + +// Generated as internal constructor for term value_gpr. +pub fn constructor_value_gpr(ctx: &mut C, arg0: Gpr) -> Option { + let pattern0_0 = arg0; + // Rule at src/isa/x64/inst.isle line 974. + let expr0_0 = C::gpr_to_reg(ctx, pattern0_0); + let expr1_0 = C::value_reg(ctx, expr0_0); + return Some(expr1_0); +} + +// Generated as internal constructor for term value_gprs. +pub fn constructor_value_gprs(ctx: &mut C, arg0: Gpr, arg1: Gpr) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 979. + let expr0_0 = C::gpr_to_reg(ctx, pattern0_0); + let expr1_0 = C::gpr_to_reg(ctx, pattern1_0); + let expr2_0 = C::value_regs(ctx, expr0_0, expr1_0); + return Some(expr2_0); +} + +// Generated as internal constructor for term value_xmm. +pub fn constructor_value_xmm(ctx: &mut C, arg0: Xmm) -> Option { + let pattern0_0 = arg0; + // Rule at src/isa/x64/inst.isle line 984. + let expr0_0 = C::xmm_to_reg(ctx, pattern0_0); + let expr1_0 = C::value_reg(ctx, expr0_0); + return Some(expr1_0); +} + // Generated as internal constructor for term extend_to_reg. pub fn constructor_extend_to_reg( ctx: &mut C, @@ -287,12 +701,12 @@ pub fn constructor_extend_to_reg( let pattern2_0 = arg1; if pattern2_0 == pattern1_0 { let pattern4_0 = arg2; - // Rule at src/isa/x64/inst.isle line 476. + // Rule at src/isa/x64/inst.isle line 1063. let expr0_0 = C::put_in_reg(ctx, pattern0_0); return Some(expr0_0); } let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 479. + // Rule at src/isa/x64/inst.isle line 1066. let expr0_0 = C::ty_bits_u16(ctx, pattern1_0); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern2_0); let expr2_0 = constructor_operand_size_bits(ctx, &expr1_0)?; @@ -316,7 +730,7 @@ pub fn constructor_extend( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/inst.isle line 499. + // Rule at src/isa/x64/inst.isle line 1086. let expr0_0 = constructor_movsx(ctx, pattern2_0, pattern3_0, pattern4_0)?; return Some(expr0_0); } @@ -324,7 +738,7 @@ pub fn constructor_extend( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/inst.isle line 495. + // Rule at src/isa/x64/inst.isle line 1082. let expr0_0 = constructor_movzx(ctx, pattern2_0, pattern3_0, pattern4_0)?; return Some(expr0_0); } @@ -337,17 +751,17 @@ pub fn constructor_extend( pub fn constructor_sse_xor_op(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32X4 { - // Rule at src/isa/x64/inst.isle line 506. + // Rule at src/isa/x64/inst.isle line 1093. let expr0_0 = SseOpcode::Xorps; return Some(expr0_0); } if pattern0_0 == F64X2 { - // Rule at src/isa/x64/inst.isle line 507. + // Rule at src/isa/x64/inst.isle line 1094. let expr0_0 = SseOpcode::Xorpd; return Some(expr0_0); } if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { - // Rule at src/isa/x64/inst.isle line 508. + // Rule at src/isa/x64/inst.isle line 1095. let expr0_0 = SseOpcode::Pxor; return Some(expr0_0); } @@ -358,13 +772,13 @@ pub fn constructor_sse_xor_op(ctx: &mut C, arg0: Type) -> Option( ctx: &mut C, arg0: Type, - arg1: Reg, - arg2: &RegMem, -) -> Option { + arg1: Xmm, + arg2: &XmmMem, +) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 512. + // Rule at src/isa/x64/inst.isle line 1099. let expr0_0 = constructor_sse_xor_op(ctx, pattern0_0)?; let expr1_0 = constructor_xmm_rm_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -374,40 +788,40 @@ pub fn constructor_sse_xor( pub fn constructor_sse_cmp_op(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32X4 { - // Rule at src/isa/x64/inst.isle line 521. + // Rule at src/isa/x64/inst.isle line 1108. let expr0_0 = SseOpcode::Cmpps; return Some(expr0_0); } if pattern0_0 == F64X2 { - // Rule at src/isa/x64/inst.isle line 522. + // Rule at src/isa/x64/inst.isle line 1109. let expr0_0 = SseOpcode::Cmppd; return Some(expr0_0); } if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { if pattern1_0 == 8 { if pattern1_1 == 16 { - // Rule at src/isa/x64/inst.isle line 517. + // Rule at src/isa/x64/inst.isle line 1104. let expr0_0 = SseOpcode::Pcmpeqb; return Some(expr0_0); } } if pattern1_0 == 16 { if pattern1_1 == 8 { - // Rule at src/isa/x64/inst.isle line 518. + // Rule at src/isa/x64/inst.isle line 1105. let expr0_0 = SseOpcode::Pcmpeqw; return Some(expr0_0); } } if pattern1_0 == 32 { if pattern1_1 == 4 { - // Rule at src/isa/x64/inst.isle line 519. + // Rule at src/isa/x64/inst.isle line 1106. let expr0_0 = SseOpcode::Pcmpeqd; return Some(expr0_0); } } if pattern1_0 == 64 { if pattern1_1 == 2 { - // Rule at src/isa/x64/inst.isle line 520. + // Rule at src/isa/x64/inst.isle line 1107. let expr0_0 = SseOpcode::Pcmpeqq; return Some(expr0_0); } @@ -417,14 +831,14 @@ pub fn constructor_sse_cmp_op(ctx: &mut C, arg0: Type) -> Option(ctx: &mut C, arg0: Type) -> Option { +pub fn constructor_vector_all_ones(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 536. - let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); - let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); + // Rule at src/isa/x64/inst.isle line 1123. + let expr0_0 = C::temp_writable_xmm(ctx); + let expr1_0 = C::writable_xmm_to_xmm(ctx, expr0_0); let expr2_0: Type = I32X4; let expr3_0 = constructor_sse_cmp_op(ctx, expr2_0)?; - let expr4_0 = RegMem::Reg { reg: expr1_0 }; + let expr4_0 = C::xmm_to_xmm_mem(ctx, expr1_0); let expr5_0 = MInst::XmmRmR { op: expr3_0, src1: expr1_0, @@ -438,70 +852,71 @@ pub fn constructor_vector_all_ones(ctx: &mut C, arg0: Type) -> Optio // Generated as internal constructor for term make_i64x2_from_lanes. pub fn constructor_make_i64x2_from_lanes( ctx: &mut C, - arg0: &RegMem, - arg1: &RegMem, -) -> Option { + arg0: &GprMem, + arg1: &GprMem, +) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 547. - let expr0_0: Type = I64X2; - let expr1_0 = C::temp_writable_reg(ctx, expr0_0); - let expr2_0 = C::writable_reg_to_reg(ctx, expr1_0); - let expr3_0 = MInst::XmmUninitializedValue { dst: expr1_0 }; - let expr4_0 = C::emit(ctx, &expr3_0); - let expr5_0 = SseOpcode::Pinsrd; - let expr6_0: u8 = 0; - let expr7_0 = OperandSize::Size64; - let expr8_0 = MInst::XmmRmRImm { - op: expr5_0, - src1: expr2_0, - src2: pattern0_0.clone(), + // Rule at src/isa/x64/inst.isle line 1134. + let expr0_0 = C::temp_writable_xmm(ctx); + let expr1_0 = C::writable_xmm_to_reg(ctx, expr0_0); + let expr2_0 = C::writable_xmm_to_xmm(ctx, expr0_0); + let expr3_0 = C::xmm_to_reg(ctx, expr2_0); + let expr4_0 = MInst::XmmUninitializedValue { dst: expr0_0 }; + let expr5_0 = C::emit(ctx, &expr4_0); + let expr6_0 = SseOpcode::Pinsrd; + let expr7_0 = C::gpr_mem_to_reg_mem(ctx, pattern0_0); + let expr8_0: u8 = 0; + let expr9_0 = OperandSize::Size64; + let expr10_0 = MInst::XmmRmRImm { + op: expr6_0, + src1: expr3_0, + src2: expr7_0, dst: expr1_0, - imm: expr6_0, - size: expr7_0, + imm: expr8_0, + size: expr9_0, }; - let expr9_0 = C::emit(ctx, &expr8_0); - let expr10_0 = SseOpcode::Pinsrd; - let expr11_0: u8 = 1; - let expr12_0 = OperandSize::Size64; - let expr13_0 = MInst::XmmRmRImm { - op: expr10_0, - src1: expr2_0, - src2: pattern1_0.clone(), + let expr11_0 = C::emit(ctx, &expr10_0); + let expr12_0 = SseOpcode::Pinsrd; + let expr13_0 = C::gpr_mem_to_reg_mem(ctx, pattern1_0); + let expr14_0: u8 = 1; + let expr15_0 = OperandSize::Size64; + let expr16_0 = MInst::XmmRmRImm { + op: expr12_0, + src1: expr3_0, + src2: expr13_0, dst: expr1_0, - imm: expr11_0, - size: expr12_0, + imm: expr14_0, + size: expr15_0, }; - let expr14_0 = C::emit(ctx, &expr13_0); + let expr17_0 = C::emit(ctx, &expr16_0); return Some(expr2_0); } -// Generated as internal constructor for term reg_mem_imm_to_xmm. -pub fn constructor_reg_mem_imm_to_xmm( - ctx: &mut C, - arg0: &RegMemImm, -) -> Option { +// Generated as internal constructor for term mov_rmi_to_xmm. +pub fn constructor_mov_rmi_to_xmm(ctx: &mut C, arg0: &RegMemImm) -> Option { let pattern0_0 = arg0; match pattern0_0 { &RegMemImm::Imm { simm32: pattern1_0 } => { - // Rule at src/isa/x64/inst.isle line 568. - return Some(pattern0_0.clone()); + // Rule at src/isa/x64/inst.isle line 1157. + let expr0_0 = C::xmm_mem_imm_new(ctx, pattern0_0); + return Some(expr0_0); } &RegMemImm::Reg { reg: pattern1_0 } => { - // Rule at src/isa/x64/inst.isle line 569. - let expr0_0: Type = I8X16; - let expr1_0 = SseOpcode::Movd; - let expr2_0 = RegMem::Reg { reg: pattern1_0 }; - let expr3_0 = OperandSize::Size32; - let expr4_0 = constructor_gpr_to_xmm(ctx, expr0_0, &expr1_0, &expr2_0, &expr3_0)?; - let expr5_0 = RegMemImm::Reg { reg: expr4_0 }; - return Some(expr5_0); + // Rule at src/isa/x64/inst.isle line 1158. + let expr0_0 = SseOpcode::Movd; + let expr1_0 = C::reg_to_gpr_mem(ctx, pattern1_0); + let expr2_0 = OperandSize::Size32; + let expr3_0 = constructor_gpr_to_xmm(ctx, &expr0_0, &expr1_0, &expr2_0)?; + let expr4_0 = C::xmm_to_xmm_mem_imm(ctx, expr3_0); + return Some(expr4_0); } &RegMemImm::Mem { addr: ref pattern1_0, } => { - // Rule at src/isa/x64/inst.isle line 567. - return Some(pattern0_0.clone()); + // Rule at src/isa/x64/inst.isle line 1156. + let expr0_0 = C::xmm_mem_imm_new(ctx, pattern0_0); + return Some(expr0_0); } _ => {} } @@ -519,7 +934,7 @@ pub fn constructor_x64_load( if pattern0_0 == I64 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 585. + // Rule at src/isa/x64/inst.isle line 1173. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::Mov64MR { @@ -533,53 +948,63 @@ pub fn constructor_x64_load( if pattern0_0 == F32 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 590. + // Rule at src/isa/x64/inst.isle line 1178. let expr0_0 = SseOpcode::Movss; let expr1_0 = C::synthetic_amode_to_reg_mem(ctx, pattern2_0); - let expr2_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, &expr1_0)?; - return Some(expr2_0); + let expr2_0 = C::xmm_mem_new(ctx, &expr1_0); + let expr3_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, &expr2_0)?; + let expr4_0 = C::xmm_to_reg(ctx, expr3_0); + return Some(expr4_0); } if pattern0_0 == F64 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 594. + // Rule at src/isa/x64/inst.isle line 1182. let expr0_0 = SseOpcode::Movsd; let expr1_0 = C::synthetic_amode_to_reg_mem(ctx, pattern2_0); - let expr2_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, &expr1_0)?; - return Some(expr2_0); + let expr2_0 = C::xmm_mem_new(ctx, &expr1_0); + let expr3_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, &expr2_0)?; + let expr4_0 = C::xmm_to_reg(ctx, expr3_0); + return Some(expr4_0); } if pattern0_0 == F32X4 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 598. + // Rule at src/isa/x64/inst.isle line 1186. let expr0_0 = SseOpcode::Movups; let expr1_0 = C::synthetic_amode_to_reg_mem(ctx, pattern2_0); - let expr2_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, &expr1_0)?; - return Some(expr2_0); + let expr2_0 = C::xmm_mem_new(ctx, &expr1_0); + let expr3_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, &expr2_0)?; + let expr4_0 = C::xmm_to_reg(ctx, expr3_0); + return Some(expr4_0); } if pattern0_0 == F64X2 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 602. + // Rule at src/isa/x64/inst.isle line 1190. let expr0_0 = SseOpcode::Movupd; let expr1_0 = C::synthetic_amode_to_reg_mem(ctx, pattern2_0); - let expr2_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, &expr1_0)?; - return Some(expr2_0); + let expr2_0 = C::xmm_mem_new(ctx, &expr1_0); + let expr3_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, &expr2_0)?; + let expr4_0 = C::xmm_to_reg(ctx, expr3_0); + return Some(expr4_0); } if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 606. + // Rule at src/isa/x64/inst.isle line 1194. let expr0_0 = SseOpcode::Movdqu; let expr1_0 = C::synthetic_amode_to_reg_mem(ctx, pattern2_0); - let expr2_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, &expr1_0)?; - return Some(expr2_0); + let expr2_0 = C::xmm_mem_new(ctx, &expr1_0); + let expr3_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, &expr2_0)?; + let expr4_0 = C::xmm_to_reg(ctx, expr3_0); + return Some(expr4_0); } if let Some(pattern1_0) = C::fits_in_32(ctx, pattern0_0) { let pattern2_0 = arg1; let pattern3_0 = arg2; if let &ExtKind::SignExtend = pattern3_0 { - // Rule at src/isa/x64/inst.isle line 580. + // Rule at src/isa/x64/inst.isle line 1168. let expr0_0 = C::ty_bytes(ctx, pattern1_0); let expr1_0: u16 = 8; let expr2_0 = C::ext_mode(ctx, expr0_0, expr1_0); @@ -603,7 +1028,7 @@ pub fn constructor_alu_rmi_r( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 625. + // Rule at src/isa/x64/inst.isle line 1213. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = MInst::AluRmiR { @@ -628,7 +1053,7 @@ pub fn constructor_add( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 633. + // Rule at src/isa/x64/inst.isle line 1221. let expr0_0 = AluRmiROpcode::Add; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -644,7 +1069,7 @@ pub fn constructor_add_with_flags( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 641. + // Rule at src/isa/x64/inst.isle line 1229. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = AluRmiROpcode::Add; @@ -673,7 +1098,7 @@ pub fn constructor_adc( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 652. + // Rule at src/isa/x64/inst.isle line 1240. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = AluRmiROpcode::Adc; @@ -702,7 +1127,7 @@ pub fn constructor_sub( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 663. + // Rule at src/isa/x64/inst.isle line 1251. let expr0_0 = AluRmiROpcode::Sub; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -718,7 +1143,7 @@ pub fn constructor_sub_with_flags( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 671. + // Rule at src/isa/x64/inst.isle line 1259. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = AluRmiROpcode::Sub; @@ -747,7 +1172,7 @@ pub fn constructor_sbb( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 682. + // Rule at src/isa/x64/inst.isle line 1270. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = AluRmiROpcode::Sbb; @@ -776,7 +1201,7 @@ pub fn constructor_mul( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 693. + // Rule at src/isa/x64/inst.isle line 1281. let expr0_0 = AluRmiROpcode::Mul; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -792,7 +1217,7 @@ pub fn constructor_x64_and( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 704. + // Rule at src/isa/x64/inst.isle line 1292. let expr0_0 = AluRmiROpcode::And; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -808,7 +1233,7 @@ pub fn constructor_or( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 712. + // Rule at src/isa/x64/inst.isle line 1300. let expr0_0 = AluRmiROpcode::Or; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -824,7 +1249,7 @@ pub fn constructor_xor( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 720. + // Rule at src/isa/x64/inst.isle line 1308. let expr0_0 = AluRmiROpcode::Xor; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -836,7 +1261,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option if pattern0_0 == I64 { let pattern2_0 = arg1; if let Some(pattern3_0) = C::nonzero_u64_fits_in_u32(ctx, pattern2_0) { - // Rule at src/isa/x64/inst.isle line 749. + // Rule at src/isa/x64/inst.isle line 1341. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = OperandSize::Size32; @@ -853,67 +1278,69 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option if pattern0_0 == F32 { let pattern2_0 = arg1; if pattern2_0 == 0 { - // Rule at src/isa/x64/inst.isle line 778. - let expr0_0: Type = F32; - let expr1_0 = C::temp_writable_reg(ctx, expr0_0); - let expr2_0 = C::writable_reg_to_reg(ctx, expr1_0); - let expr3_0 = SseOpcode::Xorps; - let expr4_0 = RegMem::Reg { reg: expr2_0 }; - let expr5_0 = MInst::XmmRmR { - op: expr3_0, - src1: expr2_0, - src2: expr4_0, - dst: expr1_0, + // Rule at src/isa/x64/inst.isle line 1370. + let expr0_0 = C::temp_writable_xmm(ctx); + let expr1_0 = C::writable_xmm_to_xmm(ctx, expr0_0); + let expr2_0 = SseOpcode::Xorps; + let expr3_0 = C::xmm_to_xmm_mem(ctx, expr1_0); + let expr4_0 = MInst::XmmRmR { + op: expr2_0, + src1: expr1_0, + src2: expr3_0, + dst: expr0_0, }; - let expr6_0 = C::emit(ctx, &expr5_0); - return Some(expr2_0); + let expr5_0 = C::emit(ctx, &expr4_0); + let expr6_0 = C::xmm_to_reg(ctx, expr1_0); + return Some(expr6_0); } - // Rule at src/isa/x64/inst.isle line 737. - let expr0_0: Type = F32; - let expr1_0 = SseOpcode::Movd; - let expr2_0: Type = I32; - let expr3_0 = constructor_imm(ctx, expr2_0, pattern2_0)?; - let expr4_0 = RegMem::Reg { reg: expr3_0 }; + // Rule at src/isa/x64/inst.isle line 1325. + let expr0_0 = SseOpcode::Movd; + let expr1_0: Type = I32; + let expr2_0 = constructor_imm(ctx, expr1_0, pattern2_0)?; + let expr3_0 = RegMem::Reg { reg: expr2_0 }; + let expr4_0 = C::gpr_mem_new(ctx, &expr3_0); let expr5_0 = OperandSize::Size32; - let expr6_0 = constructor_gpr_to_xmm(ctx, expr0_0, &expr1_0, &expr4_0, &expr5_0)?; - return Some(expr6_0); + let expr6_0 = constructor_gpr_to_xmm(ctx, &expr0_0, &expr4_0, &expr5_0)?; + let expr7_0 = C::xmm_to_reg(ctx, expr6_0); + return Some(expr7_0); } if pattern0_0 == F64 { let pattern2_0 = arg1; if pattern2_0 == 0 { - // Rule at src/isa/x64/inst.isle line 790. - let expr0_0: Type = F64; - let expr1_0 = C::temp_writable_reg(ctx, expr0_0); - let expr2_0 = C::writable_reg_to_reg(ctx, expr1_0); - let expr3_0 = SseOpcode::Xorpd; - let expr4_0 = RegMem::Reg { reg: expr2_0 }; - let expr5_0 = MInst::XmmRmR { - op: expr3_0, - src1: expr2_0, - src2: expr4_0, - dst: expr1_0, + // Rule at src/isa/x64/inst.isle line 1382. + let expr0_0 = C::temp_writable_xmm(ctx); + let expr1_0 = C::writable_xmm_to_xmm(ctx, expr0_0); + let expr2_0 = SseOpcode::Xorpd; + let expr3_0 = C::xmm_to_xmm_mem(ctx, expr1_0); + let expr4_0 = MInst::XmmRmR { + op: expr2_0, + src1: expr1_0, + src2: expr3_0, + dst: expr0_0, }; - let expr6_0 = C::emit(ctx, &expr5_0); - return Some(expr2_0); + let expr5_0 = C::emit(ctx, &expr4_0); + let expr6_0 = C::xmm_to_reg(ctx, expr1_0); + return Some(expr6_0); } - // Rule at src/isa/x64/inst.isle line 741. - let expr0_0: Type = F64; - let expr1_0 = SseOpcode::Movq; - let expr2_0: Type = I64; - let expr3_0 = constructor_imm(ctx, expr2_0, pattern2_0)?; - let expr4_0 = RegMem::Reg { reg: expr3_0 }; + // Rule at src/isa/x64/inst.isle line 1331. + let expr0_0 = SseOpcode::Movq; + let expr1_0: Type = I64; + let expr2_0 = constructor_imm(ctx, expr1_0, pattern2_0)?; + let expr3_0 = RegMem::Reg { reg: expr2_0 }; + let expr4_0 = C::gpr_mem_new(ctx, &expr3_0); let expr5_0 = OperandSize::Size64; - let expr6_0 = constructor_gpr_to_xmm(ctx, expr0_0, &expr1_0, &expr4_0, &expr5_0)?; - return Some(expr6_0); + let expr6_0 = constructor_gpr_to_xmm(ctx, &expr0_0, &expr4_0, &expr5_0)?; + let expr7_0 = C::xmm_to_reg(ctx, expr6_0); + return Some(expr7_0); } if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { let pattern2_0 = arg1; if pattern2_0 == 0 { - // Rule at src/isa/x64/inst.isle line 768. - let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); - let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); + // Rule at src/isa/x64/inst.isle line 1360. + let expr0_0 = C::temp_writable_xmm(ctx); + let expr1_0 = C::writable_xmm_to_xmm(ctx, expr0_0); let expr2_0 = constructor_sse_xor_op(ctx, pattern0_0)?; - let expr3_0 = RegMem::Reg { reg: expr1_0 }; + let expr3_0 = C::xmm_to_xmm_mem(ctx, expr1_0); let expr4_0 = MInst::XmmRmR { op: expr2_0, src1: expr1_0, @@ -921,13 +1348,14 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option dst: expr0_0, }; let expr5_0 = C::emit(ctx, &expr4_0); - return Some(expr1_0); + let expr6_0 = C::xmm_to_reg(ctx, expr1_0); + return Some(expr6_0); } } if let Some(pattern1_0) = C::fits_in_64(ctx, pattern0_0) { let pattern2_0 = arg1; if pattern2_0 == 0 { - // Rule at src/isa/x64/inst.isle line 755. + // Rule at src/isa/x64/inst.isle line 1347. let expr0_0 = C::temp_writable_reg(ctx, pattern1_0); let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); let expr2_0 = C::operand_size_of_type_32_64(ctx, pattern1_0); @@ -943,7 +1371,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option let expr6_0 = C::emit(ctx, &expr5_0); return Some(expr1_0); } - // Rule at src/isa/x64/inst.isle line 730. + // Rule at src/isa/x64/inst.isle line 1318. let expr0_0 = C::temp_writable_reg(ctx, pattern1_0); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern1_0); let expr2_0 = MInst::Imm { @@ -970,7 +1398,7 @@ pub fn constructor_shift_r( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 803. + // Rule at src/isa/x64/inst.isle line 1395. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::raw_operand_size_of_type(ctx, pattern0_0); let expr2_0 = MInst::ShiftR { @@ -995,7 +1423,7 @@ pub fn constructor_x64_rotl( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 814. + // Rule at src/isa/x64/inst.isle line 1406. let expr0_0 = ShiftKind::RotateLeft; let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -1011,7 +1439,7 @@ pub fn constructor_x64_rotr( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 820. + // Rule at src/isa/x64/inst.isle line 1412. let expr0_0 = ShiftKind::RotateRight; let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -1027,7 +1455,7 @@ pub fn constructor_shl( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 825. + // Rule at src/isa/x64/inst.isle line 1417. let expr0_0 = ShiftKind::ShiftLeft; let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -1043,7 +1471,7 @@ pub fn constructor_shr( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 830. + // Rule at src/isa/x64/inst.isle line 1422. let expr0_0 = ShiftKind::ShiftRightLogical; let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -1059,7 +1487,7 @@ pub fn constructor_sar( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 835. + // Rule at src/isa/x64/inst.isle line 1427. let expr0_0 = ShiftKind::ShiftRightArithmetic; let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -1077,7 +1505,7 @@ pub fn constructor_cmp_rmi_r( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 840. + // Rule at src/isa/x64/inst.isle line 1432. let expr0_0 = MInst::CmpRmiR { size: pattern0_0.clone(), opcode: pattern1_0.clone(), @@ -1102,7 +1530,7 @@ pub fn constructor_cmp( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 849. + // Rule at src/isa/x64/inst.isle line 1441. let expr0_0 = CmpOpcode::Cmp; let expr1_0 = constructor_cmp_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -1118,7 +1546,7 @@ pub fn constructor_test( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 854. + // Rule at src/isa/x64/inst.isle line 1446. let expr0_0 = CmpOpcode::Test; let expr1_0 = constructor_cmp_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -1136,7 +1564,7 @@ pub fn constructor_cmove( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 859. + // Rule at src/isa/x64/inst.isle line 1451. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = MInst::Cmove { @@ -1164,7 +1592,7 @@ pub fn constructor_movzx( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 867. + // Rule at src/isa/x64/inst.isle line 1459. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::MovzxRmR { ext_mode: pattern1_0.clone(), @@ -1186,7 +1614,7 @@ pub fn constructor_movsx( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 874. + // Rule at src/isa/x64/inst.isle line 1466. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::MovsxRmR { ext_mode: pattern1_0.clone(), @@ -1203,15 +1631,15 @@ pub fn constructor_xmm_rm_r( ctx: &mut C, arg0: Type, arg1: &SseOpcode, - arg2: Reg, - arg3: &RegMem, -) -> Option { + arg2: Xmm, + arg3: &XmmMem, +) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 881. - let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); + // Rule at src/isa/x64/inst.isle line 1473. + let expr0_0 = C::temp_writable_xmm(ctx); let expr1_0 = MInst::XmmRmR { op: pattern1_0.clone(), src1: pattern2_0, @@ -1219,15 +1647,15 @@ pub fn constructor_xmm_rm_r( dst: expr0_0, }; let expr2_0 = C::emit(ctx, &expr1_0); - let expr3_0 = C::writable_reg_to_reg(ctx, expr0_0); + let expr3_0 = C::writable_xmm_to_xmm(ctx, expr0_0); return Some(expr3_0); } // Generated as internal constructor for term paddb. -pub fn constructor_paddb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_paddb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 888. + // Rule at src/isa/x64/inst.isle line 1480. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Paddb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1235,10 +1663,10 @@ pub fn constructor_paddb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O } // Generated as internal constructor for term paddw. -pub fn constructor_paddw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_paddw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 893. + // Rule at src/isa/x64/inst.isle line 1485. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Paddw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1246,10 +1674,10 @@ pub fn constructor_paddw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O } // Generated as internal constructor for term paddd. -pub fn constructor_paddd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_paddd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 898. + // Rule at src/isa/x64/inst.isle line 1490. let expr0_0: Type = I32X4; let expr1_0 = SseOpcode::Paddd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1257,10 +1685,10 @@ pub fn constructor_paddd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O } // Generated as internal constructor for term paddq. -pub fn constructor_paddq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_paddq(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 903. + // Rule at src/isa/x64/inst.isle line 1495. let expr0_0: Type = I64X2; let expr1_0 = SseOpcode::Paddq; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1268,10 +1696,10 @@ pub fn constructor_paddq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O } // Generated as internal constructor for term paddsb. -pub fn constructor_paddsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_paddsb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 908. + // Rule at src/isa/x64/inst.isle line 1500. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Paddsb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1279,10 +1707,10 @@ pub fn constructor_paddsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term paddsw. -pub fn constructor_paddsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_paddsw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 913. + // Rule at src/isa/x64/inst.isle line 1505. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Paddsw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1290,10 +1718,10 @@ pub fn constructor_paddsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term paddusb. -pub fn constructor_paddusb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_paddusb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 918. + // Rule at src/isa/x64/inst.isle line 1510. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Paddusb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1301,10 +1729,10 @@ pub fn constructor_paddusb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term paddusw. -pub fn constructor_paddusw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_paddusw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 923. + // Rule at src/isa/x64/inst.isle line 1515. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Paddusw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1312,10 +1740,10 @@ pub fn constructor_paddusw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term psubb. -pub fn constructor_psubb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_psubb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 928. + // Rule at src/isa/x64/inst.isle line 1520. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Psubb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1323,10 +1751,10 @@ pub fn constructor_psubb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O } // Generated as internal constructor for term psubw. -pub fn constructor_psubw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_psubw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 933. + // Rule at src/isa/x64/inst.isle line 1525. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Psubw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1334,10 +1762,10 @@ pub fn constructor_psubw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O } // Generated as internal constructor for term psubd. -pub fn constructor_psubd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_psubd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 938. + // Rule at src/isa/x64/inst.isle line 1530. let expr0_0: Type = I32X4; let expr1_0 = SseOpcode::Psubd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1345,10 +1773,10 @@ pub fn constructor_psubd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O } // Generated as internal constructor for term psubq. -pub fn constructor_psubq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_psubq(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 943. + // Rule at src/isa/x64/inst.isle line 1535. let expr0_0: Type = I64X2; let expr1_0 = SseOpcode::Psubq; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1356,10 +1784,10 @@ pub fn constructor_psubq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O } // Generated as internal constructor for term psubsb. -pub fn constructor_psubsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_psubsb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 948. + // Rule at src/isa/x64/inst.isle line 1540. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Psubsb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1367,10 +1795,10 @@ pub fn constructor_psubsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term psubsw. -pub fn constructor_psubsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_psubsw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 953. + // Rule at src/isa/x64/inst.isle line 1545. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Psubsw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1378,10 +1806,10 @@ pub fn constructor_psubsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term psubusb. -pub fn constructor_psubusb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_psubusb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 958. + // Rule at src/isa/x64/inst.isle line 1550. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Psubusb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1389,10 +1817,10 @@ pub fn constructor_psubusb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term psubusw. -pub fn constructor_psubusw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_psubusw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 963. + // Rule at src/isa/x64/inst.isle line 1555. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Psubusw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1400,10 +1828,10 @@ pub fn constructor_psubusw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term pavgb. -pub fn constructor_pavgb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pavgb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 968. + // Rule at src/isa/x64/inst.isle line 1560. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pavgb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1411,10 +1839,10 @@ pub fn constructor_pavgb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O } // Generated as internal constructor for term pavgw. -pub fn constructor_pavgw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pavgw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 973. + // Rule at src/isa/x64/inst.isle line 1565. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pavgw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1422,10 +1850,10 @@ pub fn constructor_pavgw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O } // Generated as internal constructor for term pand. -pub fn constructor_pand(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pand(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 978. + // Rule at src/isa/x64/inst.isle line 1570. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Pand; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1433,10 +1861,10 @@ pub fn constructor_pand(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Op } // Generated as internal constructor for term andps. -pub fn constructor_andps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_andps(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 983. + // Rule at src/isa/x64/inst.isle line 1575. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Andps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1444,10 +1872,10 @@ pub fn constructor_andps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O } // Generated as internal constructor for term andpd. -pub fn constructor_andpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_andpd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 988. + // Rule at src/isa/x64/inst.isle line 1580. let expr0_0: Type = F64X2; let expr1_0 = SseOpcode::Andpd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1455,10 +1883,10 @@ pub fn constructor_andpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O } // Generated as internal constructor for term por. -pub fn constructor_por(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_por(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 993. + // Rule at src/isa/x64/inst.isle line 1585. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Por; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1466,10 +1894,10 @@ pub fn constructor_por(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Opt } // Generated as internal constructor for term orps. -pub fn constructor_orps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_orps(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 998. + // Rule at src/isa/x64/inst.isle line 1590. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Orps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1477,10 +1905,10 @@ pub fn constructor_orps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Op } // Generated as internal constructor for term orpd. -pub fn constructor_orpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_orpd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1003. + // Rule at src/isa/x64/inst.isle line 1595. let expr0_0: Type = F64X2; let expr1_0 = SseOpcode::Orpd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1488,10 +1916,10 @@ pub fn constructor_orpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Op } // Generated as internal constructor for term pxor. -pub fn constructor_pxor(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pxor(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1008. + // Rule at src/isa/x64/inst.isle line 1600. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pxor; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1499,10 +1927,10 @@ pub fn constructor_pxor(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Op } // Generated as internal constructor for term xorps. -pub fn constructor_xorps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_xorps(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1013. + // Rule at src/isa/x64/inst.isle line 1605. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Xorps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1510,10 +1938,10 @@ pub fn constructor_xorps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O } // Generated as internal constructor for term xorpd. -pub fn constructor_xorpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_xorpd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1018. + // Rule at src/isa/x64/inst.isle line 1610. let expr0_0: Type = F64X2; let expr1_0 = SseOpcode::Xorpd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1521,10 +1949,10 @@ pub fn constructor_xorpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O } // Generated as internal constructor for term pmullw. -pub fn constructor_pmullw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pmullw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1023. + // Rule at src/isa/x64/inst.isle line 1615. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pmullw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1532,10 +1960,10 @@ pub fn constructor_pmullw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term pmulld. -pub fn constructor_pmulld(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pmulld(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1028. + // Rule at src/isa/x64/inst.isle line 1620. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pmulld; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1543,10 +1971,10 @@ pub fn constructor_pmulld(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term pmulhw. -pub fn constructor_pmulhw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pmulhw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1033. + // Rule at src/isa/x64/inst.isle line 1625. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pmulhw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1554,10 +1982,10 @@ pub fn constructor_pmulhw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term pmulhuw. -pub fn constructor_pmulhuw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pmulhuw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1038. + // Rule at src/isa/x64/inst.isle line 1630. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pmulhuw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1565,10 +1993,10 @@ pub fn constructor_pmulhuw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term pmuldq. -pub fn constructor_pmuldq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pmuldq(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1043. + // Rule at src/isa/x64/inst.isle line 1635. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pmuldq; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1576,10 +2004,10 @@ pub fn constructor_pmuldq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term pmuludq. -pub fn constructor_pmuludq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pmuludq(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1048. + // Rule at src/isa/x64/inst.isle line 1640. let expr0_0: Type = I64X2; let expr1_0 = SseOpcode::Pmuludq; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1587,10 +2015,10 @@ pub fn constructor_pmuludq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term punpckhwd. -pub fn constructor_punpckhwd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_punpckhwd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1053. + // Rule at src/isa/x64/inst.isle line 1645. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Punpckhwd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1598,10 +2026,10 @@ pub fn constructor_punpckhwd(ctx: &mut C, arg0: Reg, arg1: &RegMem) } // Generated as internal constructor for term punpcklwd. -pub fn constructor_punpcklwd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_punpcklwd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1058. + // Rule at src/isa/x64/inst.isle line 1650. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Punpcklwd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1609,10 +2037,10 @@ pub fn constructor_punpcklwd(ctx: &mut C, arg0: Reg, arg1: &RegMem) } // Generated as internal constructor for term andnps. -pub fn constructor_andnps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_andnps(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1063. + // Rule at src/isa/x64/inst.isle line 1655. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Andnps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1620,10 +2048,10 @@ pub fn constructor_andnps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term andnpd. -pub fn constructor_andnpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_andnpd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1068. + // Rule at src/isa/x64/inst.isle line 1660. let expr0_0: Type = F64X2; let expr1_0 = SseOpcode::Andnpd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1631,10 +2059,10 @@ pub fn constructor_andnpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term pandn. -pub fn constructor_pandn(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pandn(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1073. + // Rule at src/isa/x64/inst.isle line 1665. let expr0_0: Type = F64X2; let expr1_0 = SseOpcode::Pandn; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1645,17 +2073,17 @@ pub fn constructor_pandn(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_sse_blend_op(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32X4 { - // Rule at src/isa/x64/inst.isle line 1077. + // Rule at src/isa/x64/inst.isle line 1669. let expr0_0 = SseOpcode::Blendvps; return Some(expr0_0); } if pattern0_0 == F64X2 { - // Rule at src/isa/x64/inst.isle line 1078. + // Rule at src/isa/x64/inst.isle line 1670. let expr0_0 = SseOpcode::Blendvpd; return Some(expr0_0); } if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { - // Rule at src/isa/x64/inst.isle line 1079. + // Rule at src/isa/x64/inst.isle line 1671. let expr0_0 = SseOpcode::Pblendvb; return Some(expr0_0); } @@ -1666,17 +2094,17 @@ pub fn constructor_sse_blend_op(ctx: &mut C, arg0: Type) -> Option(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32X4 { - // Rule at src/isa/x64/inst.isle line 1082. + // Rule at src/isa/x64/inst.isle line 1674. let expr0_0 = SseOpcode::Movaps; return Some(expr0_0); } if pattern0_0 == F64X2 { - // Rule at src/isa/x64/inst.isle line 1083. + // Rule at src/isa/x64/inst.isle line 1675. let expr0_0 = SseOpcode::Movapd; return Some(expr0_0); } if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { - // Rule at src/isa/x64/inst.isle line 1084. + // Rule at src/isa/x64/inst.isle line 1676. let expr0_0 = SseOpcode::Movdqa; return Some(expr0_0); } @@ -1687,15 +2115,15 @@ pub fn constructor_sse_mov_op(ctx: &mut C, arg0: Type) -> Option( ctx: &mut C, arg0: Type, - arg1: &RegMem, - arg2: &RegMem, - arg3: Reg, -) -> Option { + arg1: &XmmMem, + arg2: &XmmMem, + arg3: Xmm, +) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1088. + // Rule at src/isa/x64/inst.isle line 1680. let expr0_0 = C::xmm0(ctx); let expr1_0 = constructor_sse_mov_op(ctx, pattern0_0)?; let expr2_0 = MInst::XmmUnaryRmR { @@ -1712,17 +2140,17 @@ pub fn constructor_sse_blend( // Generated as internal constructor for term blendvpd. pub fn constructor_blendvpd( ctx: &mut C, - arg0: Reg, - arg1: &RegMem, - arg2: Reg, -) -> Option { + arg0: Xmm, + arg1: &XmmMem, + arg2: Xmm, +) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1100. + // Rule at src/isa/x64/inst.isle line 1694. let expr0_0 = C::xmm0(ctx); let expr1_0 = SseOpcode::Movapd; - let expr2_0 = RegMem::Reg { reg: pattern2_0 }; + let expr2_0 = C::xmm_to_xmm_mem(ctx, pattern2_0); let expr3_0 = MInst::XmmUnaryRmR { op: expr1_0, src: expr2_0, @@ -1736,10 +2164,10 @@ pub fn constructor_blendvpd( } // Generated as internal constructor for term movsd. -pub fn constructor_movsd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_movsd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1112. + // Rule at src/isa/x64/inst.isle line 1708. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Movsd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1747,10 +2175,10 @@ pub fn constructor_movsd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O } // Generated as internal constructor for term movlhps. -pub fn constructor_movlhps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_movlhps(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1117. + // Rule at src/isa/x64/inst.isle line 1713. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Movlhps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1758,10 +2186,10 @@ pub fn constructor_movlhps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term pmaxsb. -pub fn constructor_pmaxsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pmaxsb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1122. + // Rule at src/isa/x64/inst.isle line 1718. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxsb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1769,10 +2197,10 @@ pub fn constructor_pmaxsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term pmaxsw. -pub fn constructor_pmaxsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pmaxsw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1127. + // Rule at src/isa/x64/inst.isle line 1723. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxsw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1780,10 +2208,10 @@ pub fn constructor_pmaxsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term pmaxsd. -pub fn constructor_pmaxsd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pmaxsd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1132. + // Rule at src/isa/x64/inst.isle line 1728. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxsd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1791,10 +2219,10 @@ pub fn constructor_pmaxsd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term pminsb. -pub fn constructor_pminsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pminsb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1137. + // Rule at src/isa/x64/inst.isle line 1733. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pminsb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1802,10 +2230,10 @@ pub fn constructor_pminsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term pminsw. -pub fn constructor_pminsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pminsw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1142. + // Rule at src/isa/x64/inst.isle line 1738. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pminsw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1813,10 +2241,10 @@ pub fn constructor_pminsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term pminsd. -pub fn constructor_pminsd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pminsd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1147. + // Rule at src/isa/x64/inst.isle line 1743. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pminsd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1824,10 +2252,10 @@ pub fn constructor_pminsd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term pmaxub. -pub fn constructor_pmaxub(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pmaxub(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1152. + // Rule at src/isa/x64/inst.isle line 1748. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxub; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1835,10 +2263,10 @@ pub fn constructor_pmaxub(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term pmaxuw. -pub fn constructor_pmaxuw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pmaxuw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1157. + // Rule at src/isa/x64/inst.isle line 1753. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxuw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1846,10 +2274,10 @@ pub fn constructor_pmaxuw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term pmaxud. -pub fn constructor_pmaxud(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pmaxud(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1162. + // Rule at src/isa/x64/inst.isle line 1758. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxud; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1857,10 +2285,10 @@ pub fn constructor_pmaxud(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term pminub. -pub fn constructor_pminub(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pminub(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1167. + // Rule at src/isa/x64/inst.isle line 1763. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pminub; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1868,10 +2296,10 @@ pub fn constructor_pminub(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term pminuw. -pub fn constructor_pminuw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pminuw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1172. + // Rule at src/isa/x64/inst.isle line 1768. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pminuw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1879,10 +2307,10 @@ pub fn constructor_pminuw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term pminud. -pub fn constructor_pminud(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pminud(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1177. + // Rule at src/isa/x64/inst.isle line 1773. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pminud; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1890,10 +2318,10 @@ pub fn constructor_pminud(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term punpcklbw. -pub fn constructor_punpcklbw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_punpcklbw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1182. + // Rule at src/isa/x64/inst.isle line 1778. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Punpcklbw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1901,10 +2329,10 @@ pub fn constructor_punpcklbw(ctx: &mut C, arg0: Reg, arg1: &RegMem) } // Generated as internal constructor for term punpckhbw. -pub fn constructor_punpckhbw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_punpckhbw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1187. + // Rule at src/isa/x64/inst.isle line 1783. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Punpckhbw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1912,10 +2340,10 @@ pub fn constructor_punpckhbw(ctx: &mut C, arg0: Reg, arg1: &RegMem) } // Generated as internal constructor for term packsswb. -pub fn constructor_packsswb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_packsswb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1192. + // Rule at src/isa/x64/inst.isle line 1788. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Packsswb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1930,15 +2358,15 @@ pub fn constructor_xmm_rm_r_imm( arg2: &RegMem, arg3: u8, arg4: &OperandSize, -) -> Option { +) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; let pattern4_0 = arg4; - // Rule at src/isa/x64/inst.isle line 1197. - let expr0_0: Type = I8X16; - let expr1_0 = C::temp_writable_reg(ctx, expr0_0); + // Rule at src/isa/x64/inst.isle line 1793. + let expr0_0 = C::temp_writable_xmm(ctx); + let expr1_0 = C::writable_xmm_to_reg(ctx, expr0_0); let expr2_0 = MInst::XmmRmRImm { op: pattern0_0.clone(), src1: pattern1_0, @@ -1948,118 +2376,218 @@ pub fn constructor_xmm_rm_r_imm( size: pattern4_0.clone(), }; let expr3_0 = C::emit(ctx, &expr2_0); - let expr4_0 = C::writable_reg_to_reg(ctx, expr1_0); + let expr4_0 = C::writable_xmm_to_xmm(ctx, expr0_0); return Some(expr4_0); } // Generated as internal constructor for term palignr. pub fn constructor_palignr( ctx: &mut C, - arg0: Reg, - arg1: &RegMem, + arg0: Xmm, + arg1: &XmmMem, arg2: u8, arg3: &OperandSize, -) -> Option { +) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1209. + // Rule at src/isa/x64/inst.isle line 1805. let expr0_0 = SseOpcode::Palignr; - let expr1_0 = constructor_xmm_rm_r_imm( - ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0, pattern3_0, - )?; - return Some(expr1_0); + let expr1_0 = C::xmm_to_reg(ctx, pattern0_0); + let expr2_0 = C::xmm_mem_to_reg_mem(ctx, pattern1_0); + let expr3_0 = + constructor_xmm_rm_r_imm(ctx, &expr0_0, expr1_0, &expr2_0, pattern2_0, pattern3_0)?; + return Some(expr3_0); +} + +// Generated as internal constructor for term cmpps. +pub fn constructor_cmpps( + ctx: &mut C, + arg0: Xmm, + arg1: &XmmMem, + arg2: &FcmpImm, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + // Rule at src/isa/x64/inst.isle line 1814. + let expr0_0 = SseOpcode::Cmpps; + let expr1_0 = C::xmm_to_reg(ctx, pattern0_0); + let expr2_0 = C::xmm_mem_to_reg_mem(ctx, pattern1_0); + let expr3_0 = C::encode_fcmp_imm(ctx, pattern2_0); + let expr4_0 = OperandSize::Size32; + let expr5_0 = constructor_xmm_rm_r_imm(ctx, &expr0_0, expr1_0, &expr2_0, expr3_0, &expr4_0)?; + return Some(expr5_0); +} + +// Generated as internal constructor for term pinsrb. +pub fn constructor_pinsrb( + ctx: &mut C, + arg0: Xmm, + arg1: &GprMem, + arg2: u8, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + // Rule at src/isa/x64/inst.isle line 1823. + let expr0_0 = SseOpcode::Pinsrb; + let expr1_0 = C::xmm_to_reg(ctx, pattern0_0); + let expr2_0 = C::gpr_mem_to_reg_mem(ctx, pattern1_0); + let expr3_0 = OperandSize::Size32; + let expr4_0 = constructor_xmm_rm_r_imm(ctx, &expr0_0, expr1_0, &expr2_0, pattern2_0, &expr3_0)?; + return Some(expr4_0); +} + +// Generated as internal constructor for term pinsrw. +pub fn constructor_pinsrw( + ctx: &mut C, + arg0: Xmm, + arg1: &GprMem, + arg2: u8, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + // Rule at src/isa/x64/inst.isle line 1832. + let expr0_0 = SseOpcode::Pinsrw; + let expr1_0 = C::xmm_to_reg(ctx, pattern0_0); + let expr2_0 = C::gpr_mem_to_reg_mem(ctx, pattern1_0); + let expr3_0 = OperandSize::Size32; + let expr4_0 = constructor_xmm_rm_r_imm(ctx, &expr0_0, expr1_0, &expr2_0, pattern2_0, &expr3_0)?; + return Some(expr4_0); +} + +// Generated as internal constructor for term pinsrd. +pub fn constructor_pinsrd( + ctx: &mut C, + arg0: Xmm, + arg1: &GprMem, + arg2: u8, + arg3: &OperandSize, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + let pattern3_0 = arg3; + // Rule at src/isa/x64/inst.isle line 1841. + let expr0_0 = SseOpcode::Pinsrd; + let expr1_0 = C::xmm_to_reg(ctx, pattern0_0); + let expr2_0 = C::gpr_mem_to_reg_mem(ctx, pattern1_0); + let expr3_0 = + constructor_xmm_rm_r_imm(ctx, &expr0_0, expr1_0, &expr2_0, pattern2_0, pattern3_0)?; + return Some(expr3_0); +} + +// Generated as internal constructor for term insertps. +pub fn constructor_insertps( + ctx: &mut C, + arg0: Xmm, + arg1: &XmmMem, + arg2: u8, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + // Rule at src/isa/x64/inst.isle line 1850. + let expr0_0 = SseOpcode::Insertps; + let expr1_0 = C::xmm_to_reg(ctx, pattern0_0); + let expr2_0 = C::xmm_mem_to_reg_mem(ctx, pattern1_0); + let expr3_0 = OperandSize::Size32; + let expr4_0 = constructor_xmm_rm_r_imm(ctx, &expr0_0, expr1_0, &expr2_0, pattern2_0, &expr3_0)?; + return Some(expr4_0); } // Generated as internal constructor for term pshufd. pub fn constructor_pshufd( ctx: &mut C, - arg0: &RegMem, + arg0: &XmmMem, arg1: u8, arg2: &OperandSize, -) -> Option { +) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1218. - let expr0_0: Type = I8X16; - let expr1_0 = C::temp_writable_reg(ctx, expr0_0); - let expr2_0 = C::writable_reg_to_reg(ctx, expr1_0); - let expr3_0 = SseOpcode::Pshufd; - let expr4_0 = MInst::XmmRmRImm { - op: expr3_0, - src1: expr2_0, - src2: pattern0_0.clone(), - dst: expr1_0, + // Rule at src/isa/x64/inst.isle line 1859. + let expr0_0 = C::temp_writable_xmm(ctx); + let expr1_0 = C::writable_xmm_to_xmm(ctx, expr0_0); + let expr2_0 = SseOpcode::Pshufd; + let expr3_0 = C::xmm_to_reg(ctx, expr1_0); + let expr4_0 = C::xmm_mem_to_reg_mem(ctx, pattern0_0); + let expr5_0 = C::writable_xmm_to_reg(ctx, expr0_0); + let expr6_0 = MInst::XmmRmRImm { + op: expr2_0, + src1: expr3_0, + src2: expr4_0, + dst: expr5_0, imm: pattern1_0, size: pattern2_0.clone(), }; - let expr5_0 = C::emit(ctx, &expr4_0); - return Some(expr2_0); + let expr7_0 = C::emit(ctx, &expr6_0); + return Some(expr1_0); } // Generated as internal constructor for term xmm_unary_rm_r. pub fn constructor_xmm_unary_rm_r( ctx: &mut C, arg0: &SseOpcode, - arg1: &RegMem, -) -> Option { + arg1: &XmmMem, +) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1231. - let expr0_0: Type = I8X16; - let expr1_0 = C::temp_writable_reg(ctx, expr0_0); - let expr2_0 = MInst::XmmUnaryRmR { + // Rule at src/isa/x64/inst.isle line 1872. + let expr0_0 = C::temp_writable_xmm(ctx); + let expr1_0 = MInst::XmmUnaryRmR { op: pattern0_0.clone(), src: pattern1_0.clone(), - dst: expr1_0, + dst: expr0_0, }; - let expr3_0 = C::emit(ctx, &expr2_0); - let expr4_0 = C::writable_reg_to_reg(ctx, expr1_0); - return Some(expr4_0); + let expr2_0 = C::emit(ctx, &expr1_0); + let expr3_0 = C::writable_xmm_to_xmm(ctx, expr0_0); + return Some(expr3_0); } // Generated as internal constructor for term pmovsxbw. -pub fn constructor_pmovsxbw(ctx: &mut C, arg0: &RegMem) -> Option { +pub fn constructor_pmovsxbw(ctx: &mut C, arg0: &XmmMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1238. + // Rule at src/isa/x64/inst.isle line 1879. let expr0_0 = SseOpcode::Pmovsxbw; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); } // Generated as internal constructor for term pmovzxbw. -pub fn constructor_pmovzxbw(ctx: &mut C, arg0: &RegMem) -> Option { +pub fn constructor_pmovzxbw(ctx: &mut C, arg0: &XmmMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1243. + // Rule at src/isa/x64/inst.isle line 1884. let expr0_0 = SseOpcode::Pmovzxbw; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); } // Generated as internal constructor for term pabsb. -pub fn constructor_pabsb(ctx: &mut C, arg0: &RegMem) -> Option { +pub fn constructor_pabsb(ctx: &mut C, arg0: &XmmMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1248. + // Rule at src/isa/x64/inst.isle line 1889. let expr0_0 = SseOpcode::Pabsb; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); } // Generated as internal constructor for term pabsw. -pub fn constructor_pabsw(ctx: &mut C, arg0: &RegMem) -> Option { +pub fn constructor_pabsw(ctx: &mut C, arg0: &XmmMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1253. + // Rule at src/isa/x64/inst.isle line 1894. let expr0_0 = SseOpcode::Pabsw; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); } // Generated as internal constructor for term pabsd. -pub fn constructor_pabsd(ctx: &mut C, arg0: &RegMem) -> Option { +pub fn constructor_pabsd(ctx: &mut C, arg0: &XmmMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1258. + // Rule at src/isa/x64/inst.isle line 1899. let expr0_0 = SseOpcode::Pabsd; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -2069,27 +2597,26 @@ pub fn constructor_pabsd(ctx: &mut C, arg0: &RegMem) -> Option pub fn constructor_xmm_unary_rm_r_evex( ctx: &mut C, arg0: &Avx512Opcode, - arg1: &RegMem, -) -> Option { + arg1: &XmmMem, +) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1263. - let expr0_0: Type = I8X16; - let expr1_0 = C::temp_writable_reg(ctx, expr0_0); - let expr2_0 = MInst::XmmUnaryRmREvex { + // Rule at src/isa/x64/inst.isle line 1904. + let expr0_0 = C::temp_writable_xmm(ctx); + let expr1_0 = MInst::XmmUnaryRmREvex { op: pattern0_0.clone(), src: pattern1_0.clone(), - dst: expr1_0, + dst: expr0_0, }; - let expr3_0 = C::emit(ctx, &expr2_0); - let expr4_0 = C::writable_reg_to_reg(ctx, expr1_0); - return Some(expr4_0); + let expr2_0 = C::emit(ctx, &expr1_0); + let expr3_0 = C::writable_xmm_to_xmm(ctx, expr0_0); + return Some(expr3_0); } // Generated as internal constructor for term vpabsq. -pub fn constructor_vpabsq(ctx: &mut C, arg0: &RegMem) -> Option { +pub fn constructor_vpabsq(ctx: &mut C, arg0: &XmmMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1270. + // Rule at src/isa/x64/inst.isle line 1911. let expr0_0 = Avx512Opcode::Vpabsq; let expr1_0 = constructor_xmm_unary_rm_r_evex(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -2099,140 +2626,35 @@ pub fn constructor_vpabsq(ctx: &mut C, arg0: &RegMem) -> Option pub fn constructor_xmm_rm_r_evex( ctx: &mut C, arg0: &Avx512Opcode, - arg1: &RegMem, - arg2: Reg, -) -> Option { + arg1: &XmmMem, + arg2: Xmm, +) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1275. - let expr0_0: Type = I8X16; - let expr1_0 = C::temp_writable_reg(ctx, expr0_0); - let expr2_0 = MInst::XmmRmREvex { + // Rule at src/isa/x64/inst.isle line 1916. + let expr0_0 = C::temp_writable_xmm(ctx); + let expr1_0 = MInst::XmmRmREvex { op: pattern0_0.clone(), src1: pattern1_0.clone(), src2: pattern2_0, - dst: expr1_0, + dst: expr0_0, }; - let expr3_0 = C::emit(ctx, &expr2_0); - let expr4_0 = C::writable_reg_to_reg(ctx, expr1_0); - return Some(expr4_0); + let expr2_0 = C::emit(ctx, &expr1_0); + let expr3_0 = C::writable_xmm_to_xmm(ctx, expr0_0); + return Some(expr3_0); } // Generated as internal constructor for term vpmullq. -pub fn constructor_vpmullq(ctx: &mut C, arg0: &RegMem, arg1: Reg) -> Option { +pub fn constructor_vpmullq(ctx: &mut C, arg0: &XmmMem, arg1: Xmm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1287. + // Rule at src/isa/x64/inst.isle line 1928. let expr0_0 = Avx512Opcode::Vpmullq; let expr1_0 = constructor_xmm_rm_r_evex(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); } -// Generated as internal constructor for term xmm_rmi_reg. -pub fn constructor_xmm_rmi_reg( - ctx: &mut C, - arg0: &SseOpcode, - arg1: Reg, - arg2: &RegMemImm, -) -> Option { - let pattern0_0 = arg0; - let pattern1_0 = arg1; - let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1294. - let expr0_0: Type = I8X16; - let expr1_0 = C::temp_writable_reg(ctx, expr0_0); - let expr2_0 = MInst::XmmRmiReg { - opcode: pattern0_0.clone(), - src1: pattern1_0, - src2: pattern2_0.clone(), - dst: expr1_0, - }; - let expr3_0 = C::emit(ctx, &expr2_0); - let expr4_0 = C::writable_reg_to_reg(ctx, expr1_0); - return Some(expr4_0); -} - -// Generated as internal constructor for term psllw. -pub fn constructor_psllw(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option { - let pattern0_0 = arg0; - let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1304. - let expr0_0 = SseOpcode::Psllw; - let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; - return Some(expr1_0); -} - -// Generated as internal constructor for term pslld. -pub fn constructor_pslld(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option { - let pattern0_0 = arg0; - let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1309. - let expr0_0 = SseOpcode::Pslld; - let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; - return Some(expr1_0); -} - -// Generated as internal constructor for term psllq. -pub fn constructor_psllq(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option { - let pattern0_0 = arg0; - let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1314. - let expr0_0 = SseOpcode::Psllq; - let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; - return Some(expr1_0); -} - -// Generated as internal constructor for term psrlw. -pub fn constructor_psrlw(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option { - let pattern0_0 = arg0; - let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1319. - let expr0_0 = SseOpcode::Psrlw; - let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; - return Some(expr1_0); -} - -// Generated as internal constructor for term psrld. -pub fn constructor_psrld(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option { - let pattern0_0 = arg0; - let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1324. - let expr0_0 = SseOpcode::Psrld; - let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; - return Some(expr1_0); -} - -// Generated as internal constructor for term psrlq. -pub fn constructor_psrlq(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option { - let pattern0_0 = arg0; - let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1329. - let expr0_0 = SseOpcode::Psrlq; - let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; - return Some(expr1_0); -} - -// Generated as internal constructor for term psraw. -pub fn constructor_psraw(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option { - let pattern0_0 = arg0; - let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1334. - let expr0_0 = SseOpcode::Psraw; - let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; - return Some(expr1_0); -} - -// Generated as internal constructor for term psrad. -pub fn constructor_psrad(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option { - let pattern0_0 = arg0; - let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1339. - let expr0_0 = SseOpcode::Psrad; - let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; - return Some(expr1_0); -} - // Generated as internal constructor for term mul_hi. pub fn constructor_mul_hi( ctx: &mut C, @@ -2245,7 +2667,7 @@ pub fn constructor_mul_hi( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1346. + // Rule at src/isa/x64/inst.isle line 1937. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::temp_writable_reg(ctx, pattern0_0); let expr2_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); @@ -2274,179 +2696,192 @@ pub fn constructor_mulhi_u( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1362. + // Rule at src/isa/x64/inst.isle line 1953. let expr0_0: bool = false; let expr1_0 = constructor_mul_hi(ctx, pattern0_0, expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); } -// Generated as internal constructor for term cmpps. -pub fn constructor_cmpps( +// Generated as internal constructor for term xmm_rmi_xmm. +pub fn constructor_xmm_rmi_xmm( ctx: &mut C, - arg0: Reg, - arg1: &RegMem, - arg2: &FcmpImm, -) -> Option { + arg0: &SseOpcode, + arg1: Xmm, + arg2: &XmmMemImm, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + // Rule at src/isa/x64/inst.isle line 1958. + let expr0_0 = C::temp_writable_xmm(ctx); + let expr1_0 = MInst::XmmRmiReg { + opcode: pattern0_0.clone(), + src1: pattern1_0, + src2: pattern2_0.clone(), + dst: expr0_0, + }; + let expr2_0 = C::emit(ctx, &expr1_0); + let expr3_0 = C::writable_xmm_to_xmm(ctx, expr0_0); + return Some(expr3_0); +} + +// Generated as internal constructor for term psllw. +pub fn constructor_psllw(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 1968. + let expr0_0 = SseOpcode::Psllw; + let expr1_0 = constructor_xmm_rmi_xmm(ctx, &expr0_0, pattern0_0, pattern1_0)?; + return Some(expr1_0); +} + +// Generated as internal constructor for term pslld. +pub fn constructor_pslld(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 1973. + let expr0_0 = SseOpcode::Pslld; + let expr1_0 = constructor_xmm_rmi_xmm(ctx, &expr0_0, pattern0_0, pattern1_0)?; + return Some(expr1_0); +} + +// Generated as internal constructor for term psllq. +pub fn constructor_psllq(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 1978. + let expr0_0 = SseOpcode::Psllq; + let expr1_0 = constructor_xmm_rmi_xmm(ctx, &expr0_0, pattern0_0, pattern1_0)?; + return Some(expr1_0); +} + +// Generated as internal constructor for term psrlw. +pub fn constructor_psrlw(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1367. - let expr0_0 = SseOpcode::Cmpps; - let expr1_0 = C::encode_fcmp_imm(ctx, pattern2_0); - let expr2_0 = OperandSize::Size32; - let expr3_0 = - constructor_xmm_rm_r_imm(ctx, &expr0_0, pattern0_0, pattern1_0, expr1_0, &expr2_0)?; - return Some(expr3_0); + // Rule at src/isa/x64/inst.isle line 1983. + let expr0_0 = SseOpcode::Psrlw; + let expr1_0 = constructor_xmm_rmi_xmm(ctx, &expr0_0, pattern0_0, pattern1_0)?; + return Some(expr1_0); } -// Generated as internal constructor for term cmppd. -pub fn constructor_cmppd( - ctx: &mut C, - arg0: Reg, - arg1: &RegMem, - arg2: &FcmpImm, -) -> Option { +// Generated as internal constructor for term psrld. +pub fn constructor_psrld(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1380. - let expr0_0 = SseOpcode::Cmppd; - let expr1_0 = C::encode_fcmp_imm(ctx, pattern2_0); - let expr2_0 = OperandSize::Size32; - let expr3_0 = - constructor_xmm_rm_r_imm(ctx, &expr0_0, pattern0_0, pattern1_0, expr1_0, &expr2_0)?; - return Some(expr3_0); + // Rule at src/isa/x64/inst.isle line 1988. + let expr0_0 = SseOpcode::Psrld; + let expr1_0 = constructor_xmm_rmi_xmm(ctx, &expr0_0, pattern0_0, pattern1_0)?; + return Some(expr1_0); } -// Generated as internal constructor for term gpr_to_xmm. -pub fn constructor_gpr_to_xmm( - ctx: &mut C, - arg0: Type, - arg1: &SseOpcode, - arg2: &RegMem, - arg3: &OperandSize, -) -> Option { +// Generated as internal constructor for term psrlq. +pub fn constructor_psrlq(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - let pattern2_0 = arg2; - let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1389. - let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); - let expr1_0 = MInst::GprToXmm { - op: pattern1_0.clone(), - src: pattern2_0.clone(), - dst: expr0_0, - src_size: pattern3_0.clone(), - }; - let expr2_0 = C::emit(ctx, &expr1_0); - let expr3_0 = C::writable_reg_to_reg(ctx, expr0_0); - return Some(expr3_0); + // Rule at src/isa/x64/inst.isle line 1993. + let expr0_0 = SseOpcode::Psrlq; + let expr1_0 = constructor_xmm_rmi_xmm(ctx, &expr0_0, pattern0_0, pattern1_0)?; + return Some(expr1_0); } -// Generated as internal constructor for term pinsrb. -pub fn constructor_pinsrb( - ctx: &mut C, - arg0: Reg, - arg1: &RegMem, - arg2: u8, -) -> Option { +// Generated as internal constructor for term psraw. +pub fn constructor_psraw(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1396. - let expr0_0 = SseOpcode::Pinsrb; - let expr1_0 = OperandSize::Size32; - let expr2_0 = - constructor_xmm_rm_r_imm(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0, &expr1_0)?; - return Some(expr2_0); + // Rule at src/isa/x64/inst.isle line 1998. + let expr0_0 = SseOpcode::Psraw; + let expr1_0 = constructor_xmm_rmi_xmm(ctx, &expr0_0, pattern0_0, pattern1_0)?; + return Some(expr1_0); } -// Generated as internal constructor for term pinsrw. -pub fn constructor_pinsrw( - ctx: &mut C, - arg0: Reg, - arg1: &RegMem, - arg2: u8, -) -> Option { +// Generated as internal constructor for term psrad. +pub fn constructor_psrad(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1401. - let expr0_0 = SseOpcode::Pinsrw; - let expr1_0 = OperandSize::Size32; - let expr2_0 = - constructor_xmm_rm_r_imm(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0, &expr1_0)?; - return Some(expr2_0); + // Rule at src/isa/x64/inst.isle line 2003. + let expr0_0 = SseOpcode::Psrad; + let expr1_0 = constructor_xmm_rmi_xmm(ctx, &expr0_0, pattern0_0, pattern1_0)?; + return Some(expr1_0); } -// Generated as internal constructor for term pinsrd. -pub fn constructor_pinsrd( - ctx: &mut C, - arg0: Reg, - arg1: &RegMem, - arg2: u8, - arg3: &OperandSize, -) -> Option { +// Generated as internal constructor for term pextrd. +pub fn constructor_pextrd(ctx: &mut C, arg0: Type, arg1: Xmm, arg2: u8) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1406. - let expr0_0 = SseOpcode::Pinsrd; - let expr1_0 = constructor_xmm_rm_r_imm( - ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0, pattern3_0, - )?; + // Rule at src/isa/x64/inst.isle line 2008. + let expr0_0 = C::temp_writable_gpr(ctx); + let expr1_0 = C::writable_gpr_to_gpr(ctx, expr0_0); + let expr2_0 = SseOpcode::Pextrd; + let expr3_0 = C::gpr_to_reg(ctx, expr1_0); + let expr4_0 = C::xmm_to_reg(ctx, pattern1_0); + let expr5_0 = RegMem::Reg { reg: expr4_0 }; + let expr6_0 = C::writable_gpr_to_reg(ctx, expr0_0); + let expr7_0 = C::lane_type(ctx, pattern0_0); + let expr8_0 = C::operand_size_of_type_32_64(ctx, expr7_0); + let expr9_0 = MInst::XmmRmRImm { + op: expr2_0, + src1: expr3_0, + src2: expr5_0, + dst: expr6_0, + imm: pattern2_0, + size: expr8_0, + }; + let expr10_0 = C::emit(ctx, &expr9_0); return Some(expr1_0); } -// Generated as internal constructor for term insertps. -pub fn constructor_insertps( +// Generated as internal constructor for term cmppd. +pub fn constructor_cmppd( ctx: &mut C, - arg0: Reg, - arg1: &RegMem, - arg2: u8, -) -> Option { + arg0: Xmm, + arg1: &XmmMem, + arg2: &FcmpImm, +) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1411. - let expr0_0 = SseOpcode::Insertps; - let expr1_0 = OperandSize::Size32; - let expr2_0 = - constructor_xmm_rm_r_imm(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0, &expr1_0)?; - return Some(expr2_0); + // Rule at src/isa/x64/inst.isle line 2025. + let expr0_0 = SseOpcode::Cmppd; + let expr1_0 = C::xmm_to_reg(ctx, pattern0_0); + let expr2_0 = C::xmm_mem_to_reg_mem(ctx, pattern1_0); + let expr3_0 = C::encode_fcmp_imm(ctx, pattern2_0); + let expr4_0 = OperandSize::Size32; + let expr5_0 = constructor_xmm_rm_r_imm(ctx, &expr0_0, expr1_0, &expr2_0, expr3_0, &expr4_0)?; + return Some(expr5_0); } -// Generated as internal constructor for term pextrd. -pub fn constructor_pextrd(ctx: &mut C, arg0: Type, arg1: Reg, arg2: u8) -> Option { +// Generated as internal constructor for term gpr_to_xmm. +pub fn constructor_gpr_to_xmm( + ctx: &mut C, + arg0: &SseOpcode, + arg1: &GprMem, + arg2: &OperandSize, +) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1416. - let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); - let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); - let expr2_0 = SseOpcode::Pextrd; - let expr3_0 = RegMem::Reg { reg: pattern1_0 }; - let expr4_0 = C::lane_type(ctx, pattern0_0); - let expr5_0 = C::operand_size_of_type_32_64(ctx, expr4_0); - let expr6_0 = MInst::XmmRmRImm { - op: expr2_0, - src1: expr1_0, - src2: expr3_0, + // Rule at src/isa/x64/inst.isle line 2034. + let expr0_0 = C::temp_writable_xmm(ctx); + let expr1_0 = C::gpr_mem_to_reg_mem(ctx, pattern1_0); + let expr2_0 = MInst::GprToXmm { + op: pattern0_0.clone(), + src: expr1_0, dst: expr0_0, - imm: pattern2_0, - size: expr5_0, + src_size: pattern2_0.clone(), }; - let expr7_0 = C::emit(ctx, &expr6_0); - return Some(expr1_0); + let expr3_0 = C::emit(ctx, &expr2_0); + let expr4_0 = C::writable_xmm_to_xmm(ctx, expr0_0); + return Some(expr4_0); } // Generated as internal constructor for term not. -pub fn constructor_not(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { +pub fn constructor_not(ctx: &mut C, arg0: Type, arg1: Gpr) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1429. - let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); + // Rule at src/isa/x64/inst.isle line 2041. + let expr0_0 = C::temp_writable_gpr(ctx); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = MInst::Not { size: expr1_0, @@ -2454,16 +2889,16 @@ pub fn constructor_not(ctx: &mut C, arg0: Type, arg1: Reg) -> Option dst: expr0_0, }; let expr3_0 = C::emit(ctx, &expr2_0); - let expr4_0 = C::writable_reg_to_reg(ctx, expr0_0); + let expr4_0 = C::writable_gpr_to_gpr(ctx, expr0_0); return Some(expr4_0); } // Generated as internal constructor for term neg. -pub fn constructor_neg(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { +pub fn constructor_neg(ctx: &mut C, arg0: Type, arg1: Gpr) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1437. - let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); + // Rule at src/isa/x64/inst.isle line 2049. + let expr0_0 = C::temp_writable_gpr(ctx); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = MInst::Neg { size: expr1_0, @@ -2471,29 +2906,28 @@ pub fn constructor_neg(ctx: &mut C, arg0: Type, arg1: Reg) -> Option dst: expr0_0, }; let expr3_0 = C::emit(ctx, &expr2_0); - let expr4_0 = C::writable_reg_to_reg(ctx, expr0_0); + let expr4_0 = C::writable_gpr_to_gpr(ctx, expr0_0); return Some(expr4_0); } // Generated as internal constructor for term lea. -pub fn constructor_lea(ctx: &mut C, arg0: &SyntheticAmode) -> Option { +pub fn constructor_lea(ctx: &mut C, arg0: &SyntheticAmode) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1444. - let expr0_0: Type = I64; - let expr1_0 = C::temp_writable_reg(ctx, expr0_0); - let expr2_0 = MInst::LoadEffectiveAddress { + // Rule at src/isa/x64/inst.isle line 2056. + let expr0_0 = C::temp_writable_gpr(ctx); + let expr1_0 = MInst::LoadEffectiveAddress { addr: pattern0_0.clone(), - dst: expr1_0, + dst: expr0_0, }; - let expr3_0 = C::emit(ctx, &expr2_0); - let expr4_0 = C::writable_reg_to_reg(ctx, expr1_0); - return Some(expr4_0); + let expr2_0 = C::emit(ctx, &expr1_0); + let expr3_0 = C::writable_gpr_to_gpr(ctx, expr0_0); + return Some(expr3_0); } // Generated as internal constructor for term ud2. pub fn constructor_ud2(ctx: &mut C, arg0: &TrapCode) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1451. + // Rule at src/isa/x64/inst.isle line 2063. let expr0_0 = MInst::Ud2 { trap_code: pattern0_0.clone(), }; @@ -2538,13 +2972,13 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { match &pattern2_0 { &Opcode::Trap => { - // Rule at src/isa/x64/lower.isle line 1438. + // Rule at src/isa/x64/lower.isle line 1463. let expr0_0 = constructor_ud2(ctx, &pattern2_1)?; let expr1_0 = constructor_value_regs_none(ctx, &expr0_0)?; return Some(expr1_0); } &Opcode::ResumableTrap => { - // Rule at src/isa/x64/lower.isle line 1443. + // Rule at src/isa/x64/lower.isle line 1468. let expr0_0 = constructor_ud2(ctx, &pattern2_1)?; let expr1_0 = constructor_value_regs_none(ctx, &expr0_0)?; return Some(expr1_0); @@ -2561,12 +2995,12 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 384. + // Rule at src/isa/x64/lower.isle line 386. let expr0_0 = C::put_in_regs(ctx, pattern7_0); let expr1_0: usize = 0; let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); @@ -2630,7 +3064,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 458. + // Rule at src/isa/x64/lower.isle line 462. let expr0_0 = C::put_in_regs(ctx, pattern7_0); let expr1_0: usize = 0; let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); @@ -2646,7 +3080,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 524. + // Rule at src/isa/x64/lower.isle line 528. let expr0_0 = C::put_in_regs(ctx, pattern7_0); let expr1_0: usize = 0; let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); @@ -2667,7 +3101,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::Bnot = &pattern5_0 { - // Rule at src/isa/x64/lower.isle line 1304. + // Rule at src/isa/x64/lower.isle line 1321. let expr0_0 = constructor_i128_not(ctx, pattern5_1)?; return Some(expr0_0); } @@ -2750,7 +3184,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 992. + // Rule at src/isa/x64/lower.isle line 1007. let expr0_0 = C::put_in_regs(ctx, pattern7_0); let expr1_0: usize = 0; let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); @@ -2786,7 +3220,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 374. + // Rule at src/isa/x64/lower.isle line 376. let expr0_0 = C::put_in_regs(ctx, pattern7_0); let expr1_0: usize = 0; let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); @@ -2809,7 +3243,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 455. + // Rule at src/isa/x64/lower.isle line 459. let expr0_0 = C::put_in_regs(ctx, pattern7_0); let expr1_0 = C::put_in_regs(ctx, pattern7_1); let expr2_0 = constructor_or_i128(ctx, expr0_0, expr1_0)?; @@ -2818,7 +3252,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 514. + // Rule at src/isa/x64/lower.isle line 518. let expr0_0 = C::put_in_regs(ctx, pattern7_0); let expr1_0: usize = 0; let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); @@ -2841,7 +3275,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 860. + // Rule at src/isa/x64/lower.isle line 875. let expr0_0 = C::put_in_regs(ctx, pattern7_0); let expr1_0 = constructor_lo_reg(ctx, pattern7_1)?; let expr2_0 = constructor_shl_i128(ctx, expr0_0, expr1_0)?; @@ -2858,7 +3292,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 900. + // Rule at src/isa/x64/lower.isle line 915. let expr0_0 = C::put_in_regs(ctx, pattern7_0); let expr1_0 = constructor_lo_reg(ctx, pattern7_1)?; let expr2_0 = constructor_shr_i128(ctx, expr0_0, expr1_0)?; @@ -2875,7 +3309,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 568. + // Rule at src/isa/x64/lower.isle line 572. let expr0_0 = constructor_lo_reg(ctx, pattern7_1)?; let expr1_0 = C::put_in_regs(ctx, pattern7_0); let expr2_0 = constructor_shl_i128(ctx, expr1_0, expr0_0)?; @@ -2884,7 +3318,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 664. + // Rule at src/isa/x64/lower.isle line 671. let expr0_0 = constructor_lo_reg(ctx, pattern7_1)?; let expr1_0 = C::put_in_regs(ctx, pattern7_0); let expr2_0 = constructor_shr_i128(ctx, expr1_0, expr0_0)?; @@ -2893,7 +3327,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 761. + // Rule at src/isa/x64/lower.isle line 773. let expr0_0 = constructor_lo_reg(ctx, pattern7_1)?; let expr1_0 = C::put_in_regs(ctx, pattern7_0); let expr2_0 = constructor_sar_i128(ctx, expr1_0, expr0_0)?; @@ -2907,7 +3341,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::Bnot = &pattern5_0 { - // Rule at src/isa/x64/lower.isle line 1301. + // Rule at src/isa/x64/lower.isle line 1318. let expr0_0 = constructor_i128_not(ctx, pattern5_1)?; return Some(expr0_0); } @@ -2952,50 +3386,50 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1405. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); + // Rule at src/isa/x64/lower.isle line 1430. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_pminsb(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::Umin => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1427. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); + // Rule at src/isa/x64/lower.isle line 1452. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_pminub(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::Imax => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1394. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); + // Rule at src/isa/x64/lower.isle line 1419. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_pmaxsb(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::Umax => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1416. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); + // Rule at src/isa/x64/lower.isle line 1441. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_pmaxub(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::Ishl => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 580. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); + // Rule at src/isa/x64/lower.isle line 584. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = C::put_in_reg_mem_imm(ctx, pattern7_1); - let expr2_0 = constructor_reg_mem_imm_to_xmm(ctx, &expr1_0)?; + let expr2_0 = constructor_mov_rmi_to_xmm(ctx, &expr1_0)?; let expr3_0 = constructor_psllw(ctx, expr0_0, &expr2_0)?; let expr4_0 = constructor_ishl_i8x16_mask(ctx, &expr1_0)?; let expr5_0: Type = I8X16; @@ -3003,17 +3437,18 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 674. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); + // Rule at src/isa/x64/lower.isle line 681. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = C::put_in_reg_mem_imm(ctx, pattern7_1); - let expr2_0 = constructor_reg_mem_imm_to_xmm(ctx, &expr1_0)?; + let expr2_0 = constructor_mov_rmi_to_xmm(ctx, &expr1_0)?; let expr3_0 = constructor_psrlw(ctx, expr0_0, &expr2_0)?; let expr4_0 = constructor_ushr_i8x16_mask(ctx, &expr1_0)?; let expr5_0: Type = I8X16; @@ -3021,28 +3456,29 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); let pattern8_0 = C::value_type(ctx, pattern7_1); - // Rule at src/isa/x64/lower.isle line 782. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); - let expr1_0 = RegMem::Reg { reg: expr0_0 }; + // Rule at src/isa/x64/lower.isle line 794. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; + let expr1_0 = C::xmm_to_xmm_mem(ctx, expr0_0); let expr2_0 = constructor_punpcklbw(ctx, expr0_0, &expr1_0)?; - let expr3_0 = RegMem::Reg { reg: expr0_0 }; + let expr3_0 = C::xmm_to_xmm_mem(ctx, expr0_0); let expr4_0 = constructor_punpckhbw(ctx, expr0_0, &expr3_0)?; let expr5_0 = C::put_in_reg_mem_imm(ctx, pattern7_1); let expr6_0 = constructor_sshr_i8x16_bigger_shift(ctx, pattern8_0, &expr5_0)?; let expr7_0 = constructor_psraw(ctx, expr2_0, &expr6_0)?; let expr8_0 = constructor_psraw(ctx, expr4_0, &expr6_0)?; - let expr9_0 = RegMem::Reg { reg: expr8_0 }; + let expr9_0 = C::xmm_to_xmm_mem(ctx, expr8_0); let expr10_0 = constructor_packsswb(ctx, expr7_0, &expr9_0)?; - let expr11_0 = C::value_reg(ctx, expr10_0); + let expr11_0 = constructor_value_xmm(ctx, expr10_0)?; return Some(expr11_0); } _ => {} @@ -3054,20 +3490,21 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { match &pattern5_0 { &Opcode::Ineg => { - // Rule at src/isa/x64/lower.isle line 917. + // Rule at src/isa/x64/lower.isle line 932. let expr0_0: Type = I8X16; let expr1_0: u64 = 0; let expr2_0 = constructor_imm(ctx, expr0_0, expr1_0)?; - let expr3_0 = C::put_in_reg_mem(ctx, pattern5_1); - let expr4_0 = constructor_psubb(ctx, expr2_0, &expr3_0)?; - let expr5_0 = C::value_reg(ctx, expr4_0); - return Some(expr5_0); + let expr3_0 = C::xmm_new(ctx, expr2_0); + let expr4_0 = constructor_put_in_xmm_mem(ctx, pattern5_1)?; + let expr5_0 = constructor_psubb(ctx, expr3_0, &expr4_0)?; + let expr6_0 = constructor_value_xmm(ctx, expr5_0)?; + return Some(expr6_0); } &Opcode::Iabs => { - // Rule at src/isa/x64/lower.isle line 1247. - let expr0_0 = C::put_in_reg_mem(ctx, pattern5_1); + // Rule at src/isa/x64/lower.isle line 1262. + let expr0_0 = constructor_put_in_xmm_mem(ctx, pattern5_1)?; let expr1_0 = constructor_pabsb(ctx, &expr0_0)?; - let expr2_0 = C::value_reg(ctx, expr1_0); + let expr2_0 = constructor_value_xmm(ctx, expr1_0)?; return Some(expr2_0); } _ => {} @@ -3087,74 +3524,74 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1408. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); + // Rule at src/isa/x64/lower.isle line 1433. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_pminsw(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::Umin => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1430. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); + // Rule at src/isa/x64/lower.isle line 1455. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_pminuw(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::Imax => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1397. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); + // Rule at src/isa/x64/lower.isle line 1422. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_pmaxsw(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::Umax => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1419. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); + // Rule at src/isa/x64/lower.isle line 1444. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_pmaxuw(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::Ishl => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 621. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); + // Rule at src/isa/x64/lower.isle line 626. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = C::put_in_reg_mem_imm(ctx, pattern7_1); - let expr2_0 = constructor_reg_mem_imm_to_xmm(ctx, &expr1_0)?; + let expr2_0 = constructor_mov_rmi_to_xmm(ctx, &expr1_0)?; let expr3_0 = constructor_psllw(ctx, expr0_0, &expr2_0)?; - let expr4_0 = C::value_reg(ctx, expr3_0); + let expr4_0 = constructor_value_xmm(ctx, expr3_0)?; return Some(expr4_0); } &Opcode::Ushr => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 715. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); + // Rule at src/isa/x64/lower.isle line 725. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = C::put_in_reg_mem_imm(ctx, pattern7_1); - let expr2_0 = constructor_reg_mem_imm_to_xmm(ctx, &expr1_0)?; + let expr2_0 = constructor_mov_rmi_to_xmm(ctx, &expr1_0)?; let expr3_0 = constructor_psrlw(ctx, expr0_0, &expr2_0)?; - let expr4_0 = C::value_reg(ctx, expr3_0); + let expr4_0 = constructor_value_xmm(ctx, expr3_0)?; return Some(expr4_0); } &Opcode::Sshr => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 804. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); + // Rule at src/isa/x64/lower.isle line 817. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = C::put_in_reg_mem_imm(ctx, pattern7_1); - let expr2_0 = constructor_reg_mem_imm_to_xmm(ctx, &expr1_0)?; + let expr2_0 = constructor_mov_rmi_to_xmm(ctx, &expr1_0)?; let expr3_0 = constructor_psraw(ctx, expr0_0, &expr2_0)?; - let expr4_0 = C::value_reg(ctx, expr3_0); + let expr4_0 = constructor_value_xmm(ctx, expr3_0)?; return Some(expr4_0); } _ => {} @@ -3166,20 +3603,21 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { match &pattern5_0 { &Opcode::Ineg => { - // Rule at src/isa/x64/lower.isle line 921. + // Rule at src/isa/x64/lower.isle line 936. let expr0_0: Type = I16X8; let expr1_0: u64 = 0; let expr2_0 = constructor_imm(ctx, expr0_0, expr1_0)?; - let expr3_0 = C::put_in_reg_mem(ctx, pattern5_1); - let expr4_0 = constructor_psubw(ctx, expr2_0, &expr3_0)?; - let expr5_0 = C::value_reg(ctx, expr4_0); - return Some(expr5_0); + let expr3_0 = C::xmm_new(ctx, expr2_0); + let expr4_0 = constructor_put_in_xmm_mem(ctx, pattern5_1)?; + let expr5_0 = constructor_psubw(ctx, expr3_0, &expr4_0)?; + let expr6_0 = constructor_value_xmm(ctx, expr5_0)?; + return Some(expr6_0); } &Opcode::Iabs => { - // Rule at src/isa/x64/lower.isle line 1250. - let expr0_0 = C::put_in_reg_mem(ctx, pattern5_1); + // Rule at src/isa/x64/lower.isle line 1265. + let expr0_0 = constructor_put_in_xmm_mem(ctx, pattern5_1)?; let expr1_0 = constructor_pabsw(ctx, &expr0_0)?; - let expr2_0 = C::value_reg(ctx, expr1_0); + let expr2_0 = constructor_value_xmm(ctx, expr1_0)?; return Some(expr2_0); } _ => {} @@ -3199,74 +3637,74 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1411. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); + // Rule at src/isa/x64/lower.isle line 1436. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_pminsd(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::Umin => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1433. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); + // Rule at src/isa/x64/lower.isle line 1458. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_pminud(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::Imax => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1400. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); + // Rule at src/isa/x64/lower.isle line 1425. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_pmaxsd(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::Umax => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1422. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); + // Rule at src/isa/x64/lower.isle line 1447. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_pmaxud(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::Ishl => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 624. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); + // Rule at src/isa/x64/lower.isle line 630. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = C::put_in_reg_mem_imm(ctx, pattern7_1); - let expr2_0 = constructor_reg_mem_imm_to_xmm(ctx, &expr1_0)?; + let expr2_0 = constructor_mov_rmi_to_xmm(ctx, &expr1_0)?; let expr3_0 = constructor_pslld(ctx, expr0_0, &expr2_0)?; - let expr4_0 = C::value_reg(ctx, expr3_0); + let expr4_0 = constructor_value_xmm(ctx, expr3_0)?; return Some(expr4_0); } &Opcode::Ushr => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 718. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); + // Rule at src/isa/x64/lower.isle line 729. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = C::put_in_reg_mem_imm(ctx, pattern7_1); - let expr2_0 = constructor_reg_mem_imm_to_xmm(ctx, &expr1_0)?; + let expr2_0 = constructor_mov_rmi_to_xmm(ctx, &expr1_0)?; let expr3_0 = constructor_psrld(ctx, expr0_0, &expr2_0)?; - let expr4_0 = C::value_reg(ctx, expr3_0); + let expr4_0 = constructor_value_xmm(ctx, expr3_0)?; return Some(expr4_0); } &Opcode::Sshr => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 807. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); + // Rule at src/isa/x64/lower.isle line 821. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = C::put_in_reg_mem_imm(ctx, pattern7_1); - let expr2_0 = constructor_reg_mem_imm_to_xmm(ctx, &expr1_0)?; + let expr2_0 = constructor_mov_rmi_to_xmm(ctx, &expr1_0)?; let expr3_0 = constructor_psrad(ctx, expr0_0, &expr2_0)?; - let expr4_0 = C::value_reg(ctx, expr3_0); + let expr4_0 = constructor_value_xmm(ctx, expr3_0)?; return Some(expr4_0); } _ => {} @@ -3278,20 +3716,21 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { match &pattern5_0 { &Opcode::Ineg => { - // Rule at src/isa/x64/lower.isle line 925. + // Rule at src/isa/x64/lower.isle line 940. let expr0_0: Type = I32X4; let expr1_0: u64 = 0; let expr2_0 = constructor_imm(ctx, expr0_0, expr1_0)?; - let expr3_0 = C::put_in_reg_mem(ctx, pattern5_1); - let expr4_0 = constructor_psubd(ctx, expr2_0, &expr3_0)?; - let expr5_0 = C::value_reg(ctx, expr4_0); - return Some(expr5_0); + let expr3_0 = C::xmm_new(ctx, expr2_0); + let expr4_0 = constructor_put_in_xmm_mem(ctx, pattern5_1)?; + let expr5_0 = constructor_psubd(ctx, expr3_0, &expr4_0)?; + let expr6_0 = constructor_value_xmm(ctx, expr5_0)?; + return Some(expr6_0); } &Opcode::Iabs => { - // Rule at src/isa/x64/lower.isle line 1253. - let expr0_0 = C::put_in_reg_mem(ctx, pattern5_1); + // Rule at src/isa/x64/lower.isle line 1268. + let expr0_0 = constructor_put_in_xmm_mem(ctx, pattern5_1)?; let expr1_0 = constructor_pabsd(ctx, &expr0_0)?; - let expr2_0 = C::value_reg(ctx, expr1_0); + let expr2_0 = constructor_value_xmm(ctx, expr1_0)?; return Some(expr2_0); } _ => {} @@ -3311,30 +3750,30 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 627. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); + // Rule at src/isa/x64/lower.isle line 634. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = C::put_in_reg_mem_imm(ctx, pattern7_1); - let expr2_0 = constructor_reg_mem_imm_to_xmm(ctx, &expr1_0)?; + let expr2_0 = constructor_mov_rmi_to_xmm(ctx, &expr1_0)?; let expr3_0 = constructor_psllq(ctx, expr0_0, &expr2_0)?; - let expr4_0 = C::value_reg(ctx, expr3_0); + let expr4_0 = constructor_value_xmm(ctx, expr3_0)?; return Some(expr4_0); } &Opcode::Ushr => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 721. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); + // Rule at src/isa/x64/lower.isle line 733. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = C::put_in_reg_mem_imm(ctx, pattern7_1); - let expr2_0 = constructor_reg_mem_imm_to_xmm(ctx, &expr1_0)?; + let expr2_0 = constructor_mov_rmi_to_xmm(ctx, &expr1_0)?; let expr3_0 = constructor_psrlq(ctx, expr0_0, &expr2_0)?; - let expr4_0 = C::value_reg(ctx, expr3_0); + let expr4_0 = constructor_value_xmm(ctx, expr3_0)?; return Some(expr4_0); } &Opcode::Sshr => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 819. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); + // Rule at src/isa/x64/lower.isle line 833. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0: Type = I64; let expr2_0: u8 = 0; let expr3_0 = constructor_pextrd(ctx, expr1_0, expr0_0, expr2_0)?; @@ -3344,15 +3783,19 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option {} } @@ -3363,27 +3806,29 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { match &pattern5_0 { &Opcode::Ineg => { - // Rule at src/isa/x64/lower.isle line 929. + // Rule at src/isa/x64/lower.isle line 944. let expr0_0: Type = I64X2; let expr1_0: u64 = 0; let expr2_0 = constructor_imm(ctx, expr0_0, expr1_0)?; - let expr3_0 = C::put_in_reg_mem(ctx, pattern5_1); - let expr4_0 = constructor_psubq(ctx, expr2_0, &expr3_0)?; - let expr5_0 = C::value_reg(ctx, expr4_0); - return Some(expr5_0); + let expr3_0 = C::xmm_new(ctx, expr2_0); + let expr4_0 = constructor_put_in_xmm_mem(ctx, pattern5_1)?; + let expr5_0 = constructor_psubq(ctx, expr3_0, &expr4_0)?; + let expr6_0 = constructor_value_xmm(ctx, expr5_0)?; + return Some(expr6_0); } &Opcode::Iabs => { - // Rule at src/isa/x64/lower.isle line 1267. - let expr0_0 = C::put_in_reg(ctx, pattern5_1); + // Rule at src/isa/x64/lower.isle line 1282. + let expr0_0 = constructor_put_in_xmm(ctx, pattern5_1)?; let expr1_0: Type = I64X2; let expr2_0: u64 = 0; let expr3_0 = constructor_imm(ctx, expr1_0, expr2_0)?; - let expr4_0 = RegMem::Reg { reg: expr0_0 }; - let expr5_0 = constructor_psubq(ctx, expr3_0, &expr4_0)?; - let expr6_0 = RegMem::Reg { reg: expr0_0 }; - let expr7_0 = constructor_blendvpd(ctx, expr5_0, &expr6_0, expr5_0)?; - let expr8_0 = C::value_reg(ctx, expr7_0); - return Some(expr8_0); + let expr4_0 = C::xmm_new(ctx, expr3_0); + let expr5_0 = C::xmm_to_xmm_mem(ctx, expr0_0); + let expr6_0 = constructor_psubq(ctx, expr4_0, &expr5_0)?; + let expr7_0 = C::xmm_to_xmm_mem(ctx, expr0_0); + let expr8_0 = constructor_blendvpd(ctx, expr6_0, &expr7_0, expr6_0)?; + let expr9_0 = constructor_value_xmm(ctx, expr8_0)?; + return Some(expr9_0); } _ => {} } @@ -3399,17 +3844,18 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::BandNot = &pattern4_0 { let (pattern6_0, pattern6_1) = C::unpack_value_array_2(ctx, &pattern4_1); - // Rule at src/isa/x64/lower.isle line 1240. - let expr0_0 = C::put_in_reg(ctx, pattern6_1); - let expr1_0 = C::put_in_reg_mem(ctx, pattern6_0); + // Rule at src/isa/x64/lower.isle line 1255. + let expr0_0 = constructor_put_in_xmm(ctx, pattern6_1)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern6_0)?; let expr2_0 = constructor_sse_and_not(ctx, pattern2_0, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } } @@ -3478,11 +3925,11 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { let (pattern9_0, pattern9_1) = C::unpack_value_array_2(ctx, &pattern7_1); - // Rule at src/isa/x64/lower.isle line 935. - let expr0_0 = C::put_in_reg(ctx, pattern9_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); + // Rule at src/isa/x64/lower.isle line 950. + let expr0_0 = constructor_put_in_xmm(ctx, pattern9_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern9_1)?; let expr2_0 = constructor_pavgb(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::UaddSat => { let (pattern9_0, pattern9_1) = C::unpack_value_array_2(ctx, &pattern7_1); // Rule at src/isa/x64/lower.isle line 144. - let expr0_0 = C::put_in_reg(ctx, pattern9_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); + let expr0_0 = constructor_put_in_xmm(ctx, pattern9_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern9_1)?; let expr2_0 = constructor_paddusb(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::SaddSat => { let (pattern9_0, pattern9_1) = C::unpack_value_array_2(ctx, &pattern7_1); // Rule at src/isa/x64/lower.isle line 132. - let expr0_0 = C::put_in_reg(ctx, pattern9_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); + let expr0_0 = constructor_put_in_xmm(ctx, pattern9_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern9_1)?; let expr2_0 = constructor_paddsb(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::UsubSat => { let (pattern9_0, pattern9_1) = C::unpack_value_array_2(ctx, &pattern7_1); // Rule at src/isa/x64/lower.isle line 313. - let expr0_0 = C::put_in_reg(ctx, pattern9_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); + let expr0_0 = constructor_put_in_xmm(ctx, pattern9_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern9_1)?; let expr2_0 = constructor_psubusb(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::SsubSat => { let (pattern9_0, pattern9_1) = C::unpack_value_array_2(ctx, &pattern7_1); // Rule at src/isa/x64/lower.isle line 301. - let expr0_0 = C::put_in_reg(ctx, pattern9_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); + let expr0_0 = constructor_put_in_xmm(ctx, pattern9_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern9_1)?; let expr2_0 = constructor_psubsb(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::Iadd => { let (pattern9_0, pattern9_1) = C::unpack_value_array_2(ctx, &pattern7_1); // Rule at src/isa/x64/lower.isle line 96. - let expr0_0 = C::put_in_reg(ctx, pattern9_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); + let expr0_0 = constructor_put_in_xmm(ctx, pattern9_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern9_1)?; let expr2_0 = constructor_paddb(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::Isub => { let (pattern9_0, pattern9_1) = C::unpack_value_array_2(ctx, &pattern7_1); // Rule at src/isa/x64/lower.isle line 265. - let expr0_0 = C::put_in_reg(ctx, pattern9_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); + let expr0_0 = constructor_put_in_xmm(ctx, pattern9_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern9_1)?; let expr2_0 = constructor_psubb(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } _ => {} @@ -3606,71 +4053,71 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern9_0, pattern9_1) = C::unpack_value_array_2(ctx, &pattern7_1); - // Rule at src/isa/x64/lower.isle line 939. - let expr0_0 = C::put_in_reg(ctx, pattern9_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); + // Rule at src/isa/x64/lower.isle line 954. + let expr0_0 = constructor_put_in_xmm(ctx, pattern9_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern9_1)?; let expr2_0 = constructor_pavgw(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::UaddSat => { let (pattern9_0, pattern9_1) = C::unpack_value_array_2(ctx, &pattern7_1); // Rule at src/isa/x64/lower.isle line 149. - let expr0_0 = C::put_in_reg(ctx, pattern9_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); + let expr0_0 = constructor_put_in_xmm(ctx, pattern9_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern9_1)?; let expr2_0 = constructor_paddusw(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::SaddSat => { let (pattern9_0, pattern9_1) = C::unpack_value_array_2(ctx, &pattern7_1); // Rule at src/isa/x64/lower.isle line 137. - let expr0_0 = C::put_in_reg(ctx, pattern9_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); + let expr0_0 = constructor_put_in_xmm(ctx, pattern9_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern9_1)?; let expr2_0 = constructor_paddsw(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::UsubSat => { let (pattern9_0, pattern9_1) = C::unpack_value_array_2(ctx, &pattern7_1); // Rule at src/isa/x64/lower.isle line 318. - let expr0_0 = C::put_in_reg(ctx, pattern9_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); + let expr0_0 = constructor_put_in_xmm(ctx, pattern9_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern9_1)?; let expr2_0 = constructor_psubusw(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::SsubSat => { let (pattern9_0, pattern9_1) = C::unpack_value_array_2(ctx, &pattern7_1); // Rule at src/isa/x64/lower.isle line 306. - let expr0_0 = C::put_in_reg(ctx, pattern9_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); + let expr0_0 = constructor_put_in_xmm(ctx, pattern9_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern9_1)?; let expr2_0 = constructor_psubsw(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::Iadd => { let (pattern9_0, pattern9_1) = C::unpack_value_array_2(ctx, &pattern7_1); // Rule at src/isa/x64/lower.isle line 101. - let expr0_0 = C::put_in_reg(ctx, pattern9_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); + let expr0_0 = constructor_put_in_xmm(ctx, pattern9_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern9_1)?; let expr2_0 = constructor_paddw(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::Isub => { let (pattern9_0, pattern9_1) = C::unpack_value_array_2(ctx, &pattern7_1); // Rule at src/isa/x64/lower.isle line 270. - let expr0_0 = C::put_in_reg(ctx, pattern9_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); + let expr0_0 = constructor_put_in_xmm(ctx, pattern9_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern9_1)?; let expr2_0 = constructor_psubw(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::Imul => { @@ -3719,16 +4166,14 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option {} @@ -3981,20 +4404,20 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern9_0, pattern9_1) = C::unpack_value_array_2(ctx, &pattern7_1); // Rule at src/isa/x64/lower.isle line 275. - let expr0_0 = C::put_in_reg(ctx, pattern9_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); + let expr0_0 = constructor_put_in_xmm(ctx, pattern9_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern9_1)?; let expr2_0 = constructor_psubd(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::Imul => { @@ -4043,22 +4466,16 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option {} @@ -4293,20 +4692,20 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern9_0, pattern9_1) = C::unpack_value_array_2(ctx, &pattern7_1); // Rule at src/isa/x64/lower.isle line 280. - let expr0_0 = C::put_in_reg(ctx, pattern9_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); + let expr0_0 = constructor_put_in_xmm(ctx, pattern9_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern9_1)?; let expr2_0 = constructor_psubq(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::Imul => { @@ -4355,20 +4754,18 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option {} } @@ -4613,30 +5007,30 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 440. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); + // Rule at src/isa/x64/lower.isle line 442. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_sse_or(ctx, pattern2_0, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::Bxor => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 509. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); + // Rule at src/isa/x64/lower.isle line 513. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_sse_xor(ctx, pattern2_0, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } _ => {} @@ -4650,30 +5044,30 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1, pattern7_2) = C::unpack_value_array_3(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1314. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); - let expr1_0 = C::put_in_reg(ctx, pattern7_1); - let expr2_0 = RegMem::Reg { reg: expr0_0 }; + // Rule at src/isa/x64/lower.isle line 1331. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; + let expr1_0 = constructor_put_in_xmm(ctx, pattern7_1)?; + let expr2_0 = C::xmm_to_xmm_mem(ctx, expr0_0); let expr3_0 = constructor_sse_and(ctx, pattern2_0, expr1_0, &expr2_0)?; - let expr4_0 = C::put_in_reg_mem(ctx, pattern7_2); + let expr4_0 = constructor_put_in_xmm_mem(ctx, pattern7_2)?; let expr5_0 = constructor_sse_and_not(ctx, pattern2_0, expr0_0, &expr4_0)?; - let expr6_0 = RegMem::Reg { reg: expr3_0 }; + let expr6_0 = C::xmm_to_xmm_mem(ctx, expr3_0); let expr7_0 = constructor_sse_or(ctx, pattern2_0, expr5_0, &expr6_0)?; - let expr8_0 = C::value_reg(ctx, expr7_0); + let expr8_0 = constructor_value_xmm(ctx, expr7_0)?; return Some(expr8_0); } &Opcode::Vselect => { let (pattern7_0, pattern7_1, pattern7_2) = C::unpack_value_array_3(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1328. - let expr0_0 = C::put_in_reg_mem(ctx, pattern7_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); - let expr2_0 = C::put_in_reg(ctx, pattern7_2); + // Rule at src/isa/x64/lower.isle line 1345. + let expr0_0 = constructor_put_in_xmm_mem(ctx, pattern7_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; + let expr2_0 = constructor_put_in_xmm(ctx, pattern7_2)?; let expr3_0 = constructor_sse_blend( ctx, pattern2_0, &expr0_0, &expr1_0, expr2_0, )?; - let expr4_0 = C::value_reg(ctx, expr3_0); + let expr4_0 = constructor_value_xmm(ctx, expr3_0)?; return Some(expr4_0); } _ => {} @@ -4684,12 +5078,12 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::Bnot = &pattern5_0 { - // Rule at src/isa/x64/lower.isle line 1309. - let expr0_0 = C::put_in_reg(ctx, pattern5_1); + // Rule at src/isa/x64/lower.isle line 1326. + let expr0_0 = constructor_put_in_xmm(ctx, pattern5_1)?; let expr1_0 = constructor_vector_all_ones(ctx, pattern2_0)?; - let expr2_0 = RegMem::Reg { reg: expr1_0 }; + let expr2_0 = C::xmm_to_xmm_mem(ctx, expr1_0); let expr3_0 = constructor_sse_xor(ctx, pattern2_0, expr0_0, &expr2_0)?; - let expr4_0 = C::value_reg(ctx, expr3_0); + let expr4_0 = constructor_value_xmm(ctx, expr3_0)?; return Some(expr4_0); } } @@ -4811,7 +5205,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 539. + // Rule at src/isa/x64/lower.isle line 543. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_masked_in_imm8_reg(ctx, pattern7_1, pattern3_0); let expr2_0 = constructor_shl(ctx, pattern3_0, expr0_0, &expr1_0)?; @@ -5052,7 +5446,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 635. + // Rule at src/isa/x64/lower.isle line 642. let expr0_0 = ExtendKind::Zero; let expr1_0 = constructor_extend_to_reg(ctx, pattern7_0, pattern3_0, &expr0_0)?; @@ -5064,7 +5458,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 729. + // Rule at src/isa/x64/lower.isle line 741. let expr0_0 = ExtendKind::Sign; let expr1_0 = constructor_extend_to_reg(ctx, pattern7_0, pattern3_0, &expr0_0)?; @@ -5082,17 +5476,17 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { match &pattern5_0 { &Opcode::Ineg => { - // Rule at src/isa/x64/lower.isle line 912. - let expr0_0 = C::put_in_reg(ctx, pattern5_1); + // Rule at src/isa/x64/lower.isle line 927. + let expr0_0 = constructor_put_in_gpr(ctx, pattern5_1)?; let expr1_0 = constructor_neg(ctx, pattern3_0, expr0_0)?; - let expr2_0 = C::value_reg(ctx, expr1_0); + let expr2_0 = constructor_value_gpr(ctx, expr1_0)?; return Some(expr2_0); } &Opcode::Bnot => { - // Rule at src/isa/x64/lower.isle line 1288. - let expr0_0 = C::put_in_reg(ctx, pattern5_1); + // Rule at src/isa/x64/lower.isle line 1305. + let expr0_0 = constructor_put_in_gpr(ctx, pattern5_1)?; let expr1_0 = constructor_not(ctx, pattern3_0, expr0_0)?; - let expr2_0 = C::value_reg(ctx, expr1_0); + let expr2_0 = constructor_value_gpr(ctx, expr1_0)?; return Some(expr2_0); } _ => {} @@ -5136,7 +5530,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option( ctx: &mut C, arg0: Type, - arg1: Reg, - arg2: &RegMem, -) -> Option { + arg1: Xmm, + arg2: &XmmMem, +) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32X4 { let pattern2_0 = arg1; @@ -5307,28 +5701,28 @@ pub fn constructor_sse_and( pub fn constructor_sse_or( ctx: &mut C, arg0: Type, - arg1: Reg, - arg2: &RegMem, -) -> Option { + arg1: Xmm, + arg2: &XmmMem, +) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32X4 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/lower.isle line 436. + // Rule at src/isa/x64/lower.isle line 438. let expr0_0 = constructor_orps(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if pattern0_0 == F64X2 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/lower.isle line 437. + // Rule at src/isa/x64/lower.isle line 439. let expr0_0 = constructor_orpd(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/lower.isle line 438. + // Rule at src/isa/x64/lower.isle line 440. let expr0_0 = constructor_por(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } @@ -5343,7 +5737,7 @@ pub fn constructor_or_i128( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/lower.isle line 447. + // Rule at src/isa/x64/lower.isle line 451. let expr0_0: usize = 0; let expr1_0 = C::value_regs_get(ctx, pattern0_0, expr0_0); let expr2_0: usize = 1; @@ -5370,7 +5764,7 @@ pub fn constructor_shl_i128( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/lower.isle line 545. + // Rule at src/isa/x64/lower.isle line 549. let expr0_0: usize = 0; let expr1_0 = C::value_regs_get(ctx, pattern0_0, expr0_0); let expr2_0: usize = 1; @@ -5429,12 +5823,12 @@ pub fn constructor_ishl_i8x16_mask( let pattern0_0 = arg0; match pattern0_0 { &RegMemImm::Imm { simm32: pattern1_0 } => { - // Rule at src/isa/x64/lower.isle line 600. + // Rule at src/isa/x64/lower.isle line 604. let expr0_0 = C::ishl_i8x16_mask_for_const(ctx, pattern1_0); return Some(expr0_0); } &RegMemImm::Reg { reg: pattern1_0 } => { - // Rule at src/isa/x64/lower.isle line 609. + // Rule at src/isa/x64/lower.isle line 613. let expr0_0 = C::ishl_i8x16_mask_table(ctx); let expr1_0 = constructor_lea(ctx, &expr0_0)?; let expr2_0: Type = I64; @@ -5442,15 +5836,16 @@ pub fn constructor_ishl_i8x16_mask( let expr4_0 = Imm8Reg::Imm8 { imm: expr3_0 }; let expr5_0 = constructor_shl(ctx, expr2_0, pattern1_0, &expr4_0)?; let expr6_0: u32 = 0; - let expr7_0: u8 = 0; - let expr8_0 = C::amode_imm_reg_reg_shift(ctx, expr6_0, expr1_0, expr5_0, expr7_0); - let expr9_0 = C::amode_to_synthetic_amode(ctx, &expr8_0); - return Some(expr9_0); + let expr7_0 = C::gpr_new(ctx, expr5_0); + let expr8_0: u8 = 0; + let expr9_0 = C::amode_imm_reg_reg_shift(ctx, expr6_0, expr1_0, expr7_0, expr8_0); + let expr10_0 = C::amode_to_synthetic_amode(ctx, &expr9_0); + return Some(expr10_0); } &RegMemImm::Mem { addr: ref pattern1_0, } => { - // Rule at src/isa/x64/lower.isle line 617. + // Rule at src/isa/x64/lower.isle line 621. let expr0_0: Type = I64; let expr1_0 = ExtKind::None; let expr2_0 = constructor_x64_load(ctx, expr0_0, &pattern1_0, &expr1_0)?; @@ -5471,7 +5866,7 @@ pub fn constructor_shr_i128( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/lower.isle line 642. + // Rule at src/isa/x64/lower.isle line 649. let expr0_0: usize = 0; let expr1_0 = C::value_regs_get(ctx, pattern0_0, expr0_0); let expr2_0: usize = 1; @@ -5533,12 +5928,12 @@ pub fn constructor_ushr_i8x16_mask( let pattern0_0 = arg0; match pattern0_0 { &RegMemImm::Imm { simm32: pattern1_0 } => { - // Rule at src/isa/x64/lower.isle line 694. + // Rule at src/isa/x64/lower.isle line 703. let expr0_0 = C::ushr_i8x16_mask_for_const(ctx, pattern1_0); return Some(expr0_0); } &RegMemImm::Reg { reg: pattern1_0 } => { - // Rule at src/isa/x64/lower.isle line 703. + // Rule at src/isa/x64/lower.isle line 712. let expr0_0 = C::ushr_i8x16_mask_table(ctx); let expr1_0 = constructor_lea(ctx, &expr0_0)?; let expr2_0: Type = I64; @@ -5546,15 +5941,16 @@ pub fn constructor_ushr_i8x16_mask( let expr4_0 = Imm8Reg::Imm8 { imm: expr3_0 }; let expr5_0 = constructor_shl(ctx, expr2_0, pattern1_0, &expr4_0)?; let expr6_0: u32 = 0; - let expr7_0: u8 = 0; - let expr8_0 = C::amode_imm_reg_reg_shift(ctx, expr6_0, expr1_0, expr5_0, expr7_0); - let expr9_0 = C::amode_to_synthetic_amode(ctx, &expr8_0); - return Some(expr9_0); + let expr7_0 = C::gpr_new(ctx, expr5_0); + let expr8_0: u8 = 0; + let expr9_0 = C::amode_imm_reg_reg_shift(ctx, expr6_0, expr1_0, expr7_0, expr8_0); + let expr10_0 = C::amode_to_synthetic_amode(ctx, &expr9_0); + return Some(expr10_0); } &RegMemImm::Mem { addr: ref pattern1_0, } => { - // Rule at src/isa/x64/lower.isle line 711. + // Rule at src/isa/x64/lower.isle line 720. let expr0_0: Type = I64; let expr1_0 = ExtKind::None; let expr2_0 = constructor_x64_load(ctx, expr0_0, &pattern1_0, &expr1_0)?; @@ -5575,7 +5971,7 @@ pub fn constructor_sar_i128( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/lower.isle line 736. + // Rule at src/isa/x64/lower.isle line 748. let expr0_0: usize = 0; let expr1_0 = C::value_regs_get(ctx, pattern0_0, expr0_0); let expr2_0: usize = 1; @@ -5635,35 +6031,36 @@ pub fn constructor_sshr_i8x16_bigger_shift( ctx: &mut C, arg0: Type, arg1: &RegMemImm, -) -> Option { +) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; match pattern1_0 { &RegMemImm::Imm { simm32: pattern2_0 } => { - // Rule at src/isa/x64/lower.isle line 795. + // Rule at src/isa/x64/lower.isle line 807. let expr0_0: u32 = 8; let expr1_0 = C::u32_add(ctx, pattern2_0, expr0_0); let expr2_0 = RegMemImm::Imm { simm32: expr1_0 }; - return Some(expr2_0); + let expr3_0 = C::xmm_mem_imm_new(ctx, &expr2_0); + return Some(expr3_0); } &RegMemImm::Reg { reg: pattern2_0 } => { - // Rule at src/isa/x64/lower.isle line 797. + // Rule at src/isa/x64/lower.isle line 809. let expr0_0: u32 = 8; let expr1_0 = RegMemImm::Imm { simm32: expr0_0 }; let expr2_0 = constructor_add(ctx, pattern0_0, pattern2_0, &expr1_0)?; let expr3_0 = RegMemImm::Reg { reg: expr2_0 }; - let expr4_0 = constructor_reg_mem_imm_to_xmm(ctx, &expr3_0)?; + let expr4_0 = constructor_mov_rmi_to_xmm(ctx, &expr3_0)?; return Some(expr4_0); } &RegMemImm::Mem { addr: ref pattern2_0, } => { - // Rule at src/isa/x64/lower.isle line 799. + // Rule at src/isa/x64/lower.isle line 811. let expr0_0: u64 = 8; let expr1_0 = constructor_imm(ctx, pattern0_0, expr0_0)?; let expr2_0 = constructor_add(ctx, pattern0_0, expr1_0, pattern1_0)?; let expr3_0 = RegMemImm::Reg { reg: expr2_0 }; - let expr4_0 = constructor_reg_mem_imm_to_xmm(ctx, &expr3_0)?; + let expr4_0 = constructor_mov_rmi_to_xmm(ctx, &expr3_0)?; return Some(expr4_0); } _ => {} @@ -5675,28 +6072,28 @@ pub fn constructor_sshr_i8x16_bigger_shift( pub fn constructor_sse_and_not( ctx: &mut C, arg0: Type, - arg1: Reg, - arg2: &RegMem, -) -> Option { + arg1: Xmm, + arg2: &XmmMem, +) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32X4 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/lower.isle line 1229. + // Rule at src/isa/x64/lower.isle line 1244. let expr0_0 = constructor_andnps(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if pattern0_0 == F64X2 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/lower.isle line 1230. + // Rule at src/isa/x64/lower.isle line 1245. let expr0_0 = constructor_andnpd(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/lower.isle line 1231. + // Rule at src/isa/x64/lower.isle line 1246. let expr0_0 = constructor_pandn(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } @@ -5706,71 +6103,78 @@ pub fn constructor_sse_and_not( // Generated as internal constructor for term i128_not. pub fn constructor_i128_not(ctx: &mut C, arg0: Value) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/lower.isle line 1294. + // Rule at src/isa/x64/lower.isle line 1311. let expr0_0 = C::put_in_regs(ctx, pattern0_0); let expr1_0: usize = 0; let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); - let expr3_0: usize = 1; - let expr4_0 = C::value_regs_get(ctx, expr0_0, expr3_0); - let expr5_0: Type = I64; - let expr6_0 = constructor_not(ctx, expr5_0, expr2_0)?; + let expr3_0 = C::gpr_new(ctx, expr2_0); + let expr4_0: usize = 1; + let expr5_0 = C::value_regs_get(ctx, expr0_0, expr4_0); + let expr6_0 = C::gpr_new(ctx, expr5_0); let expr7_0: Type = I64; - let expr8_0 = constructor_not(ctx, expr7_0, expr4_0)?; - let expr9_0 = C::value_regs(ctx, expr6_0, expr8_0); - return Some(expr9_0); + let expr8_0 = constructor_not(ctx, expr7_0, expr3_0)?; + let expr9_0: Type = I64; + let expr10_0 = constructor_not(ctx, expr9_0, expr6_0)?; + let expr11_0 = constructor_value_gprs(ctx, expr8_0, expr10_0)?; + return Some(expr11_0); } // Generated as internal constructor for term vec_insert_lane. pub fn constructor_vec_insert_lane( ctx: &mut C, arg0: Type, - arg1: Reg, + arg1: Xmm, arg2: &RegMem, arg3: u8, -) -> Option { +) -> Option { let pattern0_0 = arg0; if pattern0_0 == I8X16 { let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/lower.isle line 1348. - let expr0_0 = constructor_pinsrb(ctx, pattern2_0, pattern3_0, pattern4_0)?; - return Some(expr0_0); + // Rule at src/isa/x64/lower.isle line 1365. + let expr0_0 = C::gpr_mem_new(ctx, pattern3_0); + let expr1_0 = constructor_pinsrb(ctx, pattern2_0, &expr0_0, pattern4_0)?; + return Some(expr1_0); } if pattern0_0 == I16X8 { let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/lower.isle line 1351. - let expr0_0 = constructor_pinsrw(ctx, pattern2_0, pattern3_0, pattern4_0)?; - return Some(expr0_0); + // Rule at src/isa/x64/lower.isle line 1369. + let expr0_0 = C::gpr_mem_new(ctx, pattern3_0); + let expr1_0 = constructor_pinsrw(ctx, pattern2_0, &expr0_0, pattern4_0)?; + return Some(expr1_0); } if pattern0_0 == I32X4 { let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/lower.isle line 1354. - let expr0_0 = OperandSize::Size32; - let expr1_0 = constructor_pinsrd(ctx, pattern2_0, pattern3_0, pattern4_0, &expr0_0)?; - return Some(expr1_0); + // Rule at src/isa/x64/lower.isle line 1373. + let expr0_0 = C::gpr_mem_new(ctx, pattern3_0); + let expr1_0 = OperandSize::Size32; + let expr2_0 = constructor_pinsrd(ctx, pattern2_0, &expr0_0, pattern4_0, &expr1_0)?; + return Some(expr2_0); } if pattern0_0 == I64X2 { let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/lower.isle line 1357. - let expr0_0 = OperandSize::Size64; - let expr1_0 = constructor_pinsrd(ctx, pattern2_0, pattern3_0, pattern4_0, &expr0_0)?; - return Some(expr1_0); + // Rule at src/isa/x64/lower.isle line 1377. + let expr0_0 = C::gpr_mem_new(ctx, pattern3_0); + let expr1_0 = OperandSize::Size64; + let expr2_0 = constructor_pinsrd(ctx, pattern2_0, &expr0_0, pattern4_0, &expr1_0)?; + return Some(expr2_0); } if pattern0_0 == F32X4 { let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/lower.isle line 1360. - let expr0_0 = C::sse_insertps_lane_imm(ctx, pattern4_0); - let expr1_0 = constructor_insertps(ctx, pattern2_0, pattern3_0, expr0_0)?; - return Some(expr1_0); + // Rule at src/isa/x64/lower.isle line 1381. + let expr0_0 = C::xmm_mem_new(ctx, pattern3_0); + let expr1_0 = C::sse_insertps_lane_imm(ctx, pattern4_0); + let expr2_0 = constructor_insertps(ctx, pattern2_0, &expr0_0, expr1_0)?; + return Some(expr2_0); } if pattern0_0 == F64X2 { let pattern2_0 = arg1; @@ -5778,25 +6182,28 @@ pub fn constructor_vec_insert_lane( if let &RegMem::Reg { reg: pattern4_0 } = pattern3_0 { let pattern5_0 = arg3; if pattern5_0 == 0 { - // Rule at src/isa/x64/lower.isle line 1381. + // Rule at src/isa/x64/lower.isle line 1403. let expr0_0 = RegMem::Reg { reg: pattern4_0 }; - let expr1_0 = constructor_movsd(ctx, pattern2_0, &expr0_0)?; - return Some(expr1_0); + let expr1_0 = C::xmm_mem_new(ctx, &expr0_0); + let expr2_0 = constructor_movsd(ctx, pattern2_0, &expr1_0)?; + return Some(expr2_0); } } let pattern4_0 = arg3; if pattern4_0 == 0 { - // Rule at src/isa/x64/lower.isle line 1382. + // Rule at src/isa/x64/lower.isle line 1405. let expr0_0 = SseOpcode::Movsd; - let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern3_0)?; - let expr2_0 = RegMem::Reg { reg: expr1_0 }; - let expr3_0 = constructor_movsd(ctx, pattern2_0, &expr2_0)?; - return Some(expr3_0); + let expr1_0 = C::xmm_mem_new(ctx, pattern3_0); + let expr2_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, &expr1_0)?; + let expr3_0 = C::xmm_to_xmm_mem(ctx, expr2_0); + let expr4_0 = constructor_movsd(ctx, pattern2_0, &expr3_0)?; + return Some(expr4_0); } if pattern4_0 == 1 { - // Rule at src/isa/x64/lower.isle line 1390. - let expr0_0 = constructor_movlhps(ctx, pattern2_0, pattern3_0)?; - return Some(expr0_0); + // Rule at src/isa/x64/lower.isle line 1414. + let expr0_0 = C::xmm_mem_new(ctx, pattern3_0); + let expr1_0 = constructor_movlhps(ctx, pattern2_0, &expr0_0)?; + return Some(expr1_0); } } return None; diff --git a/cranelift/codegen/src/machinst/isle.rs b/cranelift/codegen/src/machinst/isle.rs index 369f3d1f6616..e33cbb4f084e 100644 --- a/cranelift/codegen/src/machinst/isle.rs +++ b/cranelift/codegen/src/machinst/isle.rs @@ -1,14 +1,25 @@ use crate::ir::{Inst, Value}; use crate::machinst::{get_output_reg, InsnOutput, LowerCtx, MachInst, RegRenamer}; +use alloc::boxed::Box; +use alloc::vec::Vec; use regalloc::{Reg, Writable}; use smallvec::SmallVec; +pub use super::MachLabel; +pub use crate::ir::ExternalName; +pub use crate::isa::unwind::UnwindInst; + pub type Unit = (); pub type ValueSlice<'a> = &'a [Value]; pub type ValueArray2 = [Value; 2]; pub type ValueArray3 = [Value; 3]; pub type WritableReg = Writable; +pub type OptionWritableReg = Option; +pub type VecReg = Vec; +pub type VecWritableReg = Vec; pub type ValueRegs = crate::machinst::ValueRegs; +pub type VecMachLabel = Vec; +pub type BoxExternalName = Box; /// Helper macro to define methods in `prelude.isle` within `impl Context for /// ...` for each backend. These methods are shared amongst all backends. diff --git a/cranelift/codegen/src/prelude.isle b/cranelift/codegen/src/prelude.isle index c5a037b6579c..8dfeaefb571c 100644 --- a/cranelift/codegen/src/prelude.isle +++ b/cranelift/codegen/src/prelude.isle @@ -48,6 +48,9 @@ (type Reg (primitive Reg)) (type WritableReg (primitive WritableReg)) +(type OptionWritableReg (primitive OptionWritableReg)) +(type VecReg extern (enum)) +(type VecWritableReg extern (enum)) ;; Construct a `ValueRegs` of one register. (decl value_reg (Reg) ValueRegs) @@ -106,6 +109,15 @@ (let ((regs ValueRegs (put_in_regs val))) (value_regs_get regs 0))) +;;;; Common Mach Types ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(type MachLabel (primitive MachLabel)) +(type VecMachLabel extern (enum)) +(type ValueLabel (primitive ValueLabel)) +(type UnwindInst (primitive UnwindInst)) +(type ExternalName (primitive ExternalName)) +(type BoxExternalName (primitive BoxExternalName)) + ;;;; Primitive Type Conversions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (decl u8_as_u64 (u8) u64) @@ -349,4 +361,3 @@ (decl avoid_div_traps () Type) (extern extractor avoid_div_traps avoid_div_traps) -