From e1f4e29efefa3b2140fcc101d5e1ad21ad76b3ff Mon Sep 17 00:00:00 2001 From: Nick Fitzgerald Date: Tue, 25 Jan 2022 14:38:23 -0800 Subject: [PATCH 1/3] ISLE: Add a `nodebug` type attribute to disable `derive(Debug)` I need this to move the x64 `Inst` definition into ISLE without losing its custom `Debug` implementation that prints the assembly for the `Inst`. --- cranelift/isle/isle/src/ast.rs | 1 + cranelift/isle/isle/src/codegen.rs | 14 ++++++-- cranelift/isle/isle/src/parser.rs | 20 +++++++++-- cranelift/isle/isle/src/sema.rs | 43 +++++++++++++++-------- cranelift/isle/isle_examples/nodebug.isle | 4 +++ 5 files changed, 63 insertions(+), 19 deletions(-) create mode 100644 cranelift/isle/isle_examples/nodebug.isle diff --git a/cranelift/isle/isle/src/ast.rs b/cranelift/isle/isle/src/ast.rs index 57f257a89293..dd69628eede2 100644 --- a/cranelift/isle/isle/src/ast.rs +++ b/cranelift/isle/isle/src/ast.rs @@ -32,6 +32,7 @@ pub struct Ident(pub String, pub Pos); pub struct Type { pub name: Ident, pub is_extern: bool, + pub is_nodebug: bool, pub ty: TypeValue, pub pos: Pos, } diff --git a/cranelift/isle/isle/src/codegen.rs b/cranelift/isle/isle/src/codegen.rs index 23a76d39e743..c5de4deb6692 100644 --- a/cranelift/isle/isle/src/codegen.rs +++ b/cranelift/isle/isle/src/codegen.rs @@ -138,6 +138,7 @@ impl<'a> Codegen<'a> { &Type::Enum { name, is_extern, + is_nodebug, ref variants, pos, .. @@ -150,11 +151,20 @@ impl<'a> Codegen<'a> { pos.pretty_print_line(&self.typeenv.filenames[..]) ) .unwrap(); + + // Generate the `derive`s. + let debug_derive = if is_nodebug { "" } else { ", Debug" }; if variants.iter().all(|v| v.fields.is_empty()) { - writeln!(code, "#[derive(Copy, Clone, Debug, PartialEq, Eq)]").unwrap(); + writeln!( + code, + "#[derive(Copy, Clone, PartialEq, Eq{})]", + debug_derive + ) + .unwrap(); } else { - writeln!(code, "#[derive(Clone, Debug)]").unwrap(); + writeln!(code, "#[derive(Clone{})]", debug_derive).unwrap(); } + writeln!(code, "pub enum {} {{", name).unwrap(); for variant in variants { let name = &self.typeenv.syms[variant.name.index()]; diff --git a/cranelift/isle/isle/src/parser.rs b/cranelift/isle/isle/src/parser.rs index 8d8bbad5d8f7..1f8de5ccff98 100644 --- a/cranelift/isle/isle/src/parser.rs +++ b/cranelift/isle/isle/src/parser.rs @@ -197,15 +197,29 @@ impl<'a> Parser<'a> { fn parse_type(&mut self) -> Result { let pos = self.pos(); let name = self.parse_ident()?; + let mut is_extern = false; - if self.is_sym_str("extern") { - self.symbol()?; - is_extern = true; + let mut is_nodebug = false; + + while self.lexer.peek().map_or(false, |(_pos, tok)| tok.is_sym()) { + let sym = self.symbol()?; + if sym == "extern" { + is_extern = true; + } else if sym == "nodebug" { + is_nodebug = true; + } else { + return Err(self.error( + self.pos(), + format!("unknown type declaration modifier: {}", sym), + )); + } } + let ty = self.parse_typevalue()?; Ok(Type { name, is_extern, + is_nodebug, ty, pos, }) diff --git a/cranelift/isle/isle/src/sema.rs b/cranelift/isle/isle/src/sema.rs index fab78e6b7c12..e10a5a5e7135 100644 --- a/cranelift/isle/isle/src/sema.rs +++ b/cranelift/isle/isle/src/sema.rs @@ -109,6 +109,10 @@ pub enum Type { /// If so, ISLE will not emit a definition for it. If not, then it will /// emit a Rust definition for it. is_extern: bool, + /// Whether this type should *not* derive `Debug`. + /// + /// Incompatible with `is_extern`. + is_nodebug: bool, /// The different variants for this enum. variants: Vec, /// The ISLE source position where this `enum` is defined. @@ -607,7 +611,7 @@ impl TypeEnv { let ty = match tyenv.type_map.get(&ty) { Some(ty) => *ty, None => { - tyenv.report_error(pos, "Unknown type for constant".into()); + tyenv.report_error(pos, "Unknown type for constant"); continue; } }; @@ -635,9 +639,22 @@ impl TypeEnv { let name = self.intern(&ty.name).unwrap(); match &ty.ty { &ast::TypeValue::Primitive(ref id, ..) => { + if ty.is_nodebug { + self.report_error(ty.pos, "primitive types cannot be marked `nodebug`"); + return None; + } + if ty.is_extern { + self.report_error(ty.pos, "primitive types cannot be marked `extern`"); + return None; + } Some(Type::Primitive(tid, self.intern_mut(id), ty.pos)) } &ast::TypeValue::Enum(ref ty_variants, ..) => { + if ty.is_extern && ty.is_nodebug { + self.report_error(ty.pos, "external types cannot be marked `nodebug`"); + return None; + } + let mut variants = vec![]; for variant in ty_variants { let combined_ident = @@ -696,6 +713,7 @@ impl TypeEnv { name, id: tid, is_extern: ty.is_extern, + is_nodebug: ty.is_nodebug, variants, pos: ty.pos, }) @@ -703,9 +721,9 @@ impl TypeEnv { } } - fn error(&self, pos: Pos, msg: String) -> Error { + fn error(&self, pos: Pos, msg: impl Into) -> Error { let e = Error::TypeError { - msg, + msg: msg.into(), src: Source::new( self.filenames[pos.file].clone(), self.file_texts[pos.file].clone(), @@ -716,7 +734,7 @@ impl TypeEnv { e } - fn report_error(&mut self, pos: Pos, msg: String) { + fn report_error(&mut self, pos: Pos, msg: impl Into) { let err = self.error(pos, msg); self.errors.push(err); } @@ -987,8 +1005,7 @@ impl TermEnv { tyenv.report_error( ext.pos, "Extractor macro body defined on term of incorrect kind; cannot be an \ - enum variant" - .into(), + enum variant", ); continue; } @@ -1329,10 +1346,7 @@ impl TermEnv { let ty = match expected_ty { Some(t) => t, None => { - tyenv.report_error( - pos, - "Need an implied type for an integer constant".into(), - ); + tyenv.report_error(pos, "Need an implied type for an integer constant"); return None; } }; @@ -1353,12 +1367,12 @@ impl TermEnv { let const_ty = match tyenv.const_types.get(&val) { Some(ty) => *ty, None => { - tyenv.report_error(pos, "Unknown constant".into()); + tyenv.report_error(pos, "Unknown constant"); return None; } }; if expected_ty.is_some() && expected_ty != Some(const_ty) { - tyenv.report_error(pos, "Type mismatch for constant".into()); + tyenv.report_error(pos, "Type mismatch for constant"); } Some((Pattern::ConstPrim(const_ty, val), const_ty)) } @@ -1366,7 +1380,7 @@ impl TermEnv { let ty = match expected_ty { Some(t) => t, None => { - tyenv.report_error(pos, "Need an implied type for a wildcard".into()); + tyenv.report_error(pos, "Need an implied type for a wildcard"); return None; } }; @@ -1775,7 +1789,7 @@ impl TermEnv { let const_ty = match tyenv.const_types.get(&val) { Some(ty) => *ty, None => { - tyenv.report_error(pos, "Unknown constant".into()); + tyenv.report_error(pos, "Unknown constant"); return None; } }; @@ -1920,6 +1934,7 @@ mod test { name: sym_a, id: TypeId(1), is_extern: true, + is_nodebug: false, variants: vec![ Variant { name: sym_b, diff --git a/cranelift/isle/isle_examples/nodebug.isle b/cranelift/isle/isle_examples/nodebug.isle new file mode 100644 index 000000000000..71accb034f85 --- /dev/null +++ b/cranelift/isle/isle_examples/nodebug.isle @@ -0,0 +1,4 @@ +(type DoesNotDeriveDebug nodebug + (enum A + B + C)) From 795b0aaf9ac5b9005c6f2d41ee76f71f1caad853 Mon Sep 17 00:00:00 2001 From: Nick Fitzgerald Date: Fri, 21 Jan 2022 14:10:40 -0800 Subject: [PATCH 2/3] cranelift: Add newtype wrappers for x64 register classes This primary motivation of this large commit (apologies for its size!) is to introduce `Gpr` and `Xmm` newtypes over `Reg`. This should help catch difficult-to-diagnose register class mixup bugs in x64 lowerings. But having a newtype for `Gpr` and `Xmm` themselves isn't enough to catch all of our operand-with-wrong-register-class bugs, because about 50% of operands on x64 aren't just a register, but a register or memory address or even an immediate! So we have `{Gpr,Xmm}Mem[Imm]` newtypes as well. Unfortunately, `GprMem` et al can't be `enum`s and are therefore a little bit noisier to work with from ISLE. They need to maintain the invariant that their registers really are of the claimed register class, so they need to encapsulate the inner data. If they exposed the underlying `enum` variants, then anyone could just change register classes or construct a `GprMem` that holds an XMM register, defeating the whole point of these newtypes. So when working with these newtypes from ISLE, we rely on external constructors like `(gpr_to_gpr_mem my_gpr)` instead of `(GprMem.Gpr my_gpr)`. A bit of extra lines of code are included to add support for register mapping for all of these newtypes as well. Ultimately this is all a bit wordier than I'd hoped it would be when I first started authoring this commit, but I think it is all worth it nonetheless! In the process of adding these newtypes, I didn't want to have to update both the ISLE `extern` type definition of `MInst` and the Rust definition, so I move the definition fully into ISLE, similar as aarch64. Finally, this process isn't complete. I've introduced the newtypes here, and I've made most XMM-using instructions switch from `Reg` to `Xmm`, as well as register class-converting instructions, but I haven't moved all of the GPR-using instructions over to the newtypes yet. I figured this commit was big enough as it was, and I can continue the adoption of these newtypes in follow up commits. Part of #3685. --- Cargo.lock | 4 +- cranelift/codegen/Cargo.toml | 2 +- cranelift/codegen/build.rs | 2 + cranelift/codegen/src/isa/aarch64/inst.isle | 5 - .../lower/isle/generated_code.manifest | 4 +- .../isa/aarch64/lower/isle/generated_code.rs | 434 +-- cranelift/codegen/src/isa/s390x/inst.isle | 10 - .../s390x/lower/isle/generated_code.manifest | 4 +- .../isa/s390x/lower/isle/generated_code.rs | 932 +++--- cranelift/codegen/src/isa/x64/encoding/rex.rs | 6 +- cranelift/codegen/src/isa/x64/inst.isle | 1246 ++++++-- cranelift/codegen/src/isa/x64/inst/args.rs | 315 +- cranelift/codegen/src/isa/x64/inst/emit.rs | 240 +- .../codegen/src/isa/x64/inst/emit_tests.rs | 370 ++- cranelift/codegen/src/isa/x64/inst/mod.rs | 769 +---- cranelift/codegen/src/isa/x64/lower.isle | 449 +-- cranelift/codegen/src/isa/x64/lower.rs | 26 +- cranelift/codegen/src/isa/x64/lower/isle.rs | 138 +- .../x64/lower/isle/generated_code.manifest | 6 +- .../src/isa/x64/lower/isle/generated_code.rs | 2751 ++++++++++------- cranelift/codegen/src/machinst/isle.rs | 11 + cranelift/codegen/src/prelude.isle | 13 +- 22 files changed, 4560 insertions(+), 3177 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d95912d78449..5a01aaacc735 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2381,9 +2381,9 @@ dependencies = [ [[package]] name = "regalloc" -version = "0.0.33" +version = "0.0.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d808cff91dfca7b239d40b972ba628add94892b1d9e19a842aedc5cfae8ab1a" +checksum = "62446b1d3ebf980bdc68837700af1d77b37bc430e524bf95319c6eada2a4cc02" dependencies = [ "log", "rustc-hash", diff --git a/cranelift/codegen/Cargo.toml b/cranelift/codegen/Cargo.toml index 3e20edcc8d56..27979797bf9d 100644 --- a/cranelift/codegen/Cargo.toml +++ b/cranelift/codegen/Cargo.toml @@ -23,7 +23,7 @@ serde = { version = "1.0.94", features = ["derive"], optional = true } bincode = { version = "1.2.1", optional = true } gimli = { version = "0.26.0", default-features = false, features = ["write"], optional = true } smallvec = { version = "1.6.1" } -regalloc = { version = "0.0.33" } +regalloc = "0.0.34" souper-ir = { version = "2.1.0", optional = true } # It is a goal of the cranelift-codegen crate to have minimal external dependencies. # Please don't add any unless they are essential to the task of creating binary diff --git a/cranelift/codegen/build.rs b/cranelift/codegen/build.rs index cb704cb0abbb..1b95bf02a834 100644 --- a/cranelift/codegen/build.rs +++ b/cranelift/codegen/build.rs @@ -406,6 +406,8 @@ fn rebuild_isle( ) -> Result<(), Box> { use cranelift_isle as isle; + println!("Rebuilding {}", compilation.output.display()); + // First, remove the manifest, if any; we will recreate it // below if the compilation is successful. Ignore error if no // manifest was present. diff --git a/cranelift/codegen/src/isa/aarch64/inst.isle b/cranelift/codegen/src/isa/aarch64/inst.isle index 4b6e305ba3d1..778098d81160 100644 --- a/cranelift/codegen/src/isa/aarch64/inst.isle +++ b/cranelift/codegen/src/isa/aarch64/inst.isle @@ -871,15 +871,10 @@ (type BoxCallInfo (primitive BoxCallInfo)) (type BoxCallIndInfo (primitive BoxCallIndInfo)) -(type VecMachLabel (primitive VecMachLabel)) (type CondBrKind (primitive CondBrKind)) (type BranchTarget (primitive BranchTarget)) (type BoxJTSequenceInfo (primitive BoxJTSequenceInfo)) -(type BoxExternalName (primitive BoxExternalName)) (type CodeOffset (primitive CodeOffset)) -(type ExternalName (primitive ExternalName)) -(type ValueLabel (primitive ValueLabel)) -(type UnwindInst (primitive UnwindInst)) (type ExtendOp extern (enum diff --git a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest index db1f12e7e61b..209a864f892b 100644 --- a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest +++ b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest @@ -1,4 +1,4 @@ src/clif.isle 9ea75a6f790b5c03 -src/prelude.isle 2bfcafbef6b29358 -src/isa/aarch64/inst.isle 944323ff7d6db098 +src/prelude.isle 6aaf8ce0f5a5c2ec +src/isa/aarch64/inst.isle dafd813ba278ce19 src/isa/aarch64/lower.isle 2d2e1e076a0c8a23 diff --git a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs index a5a717b448fc..54b56919b72d 100644 --- a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs +++ b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs @@ -96,19 +96,19 @@ pub trait Context { fn rotr_opposite_amount(&mut self, arg0: Type, arg1: ImmShift) -> ImmShift; } -/// Internal type SideEffectNoResult: defined at src/prelude.isle line 295. +/// Internal type SideEffectNoResult: defined at src/prelude.isle line 307. #[derive(Clone, Debug)] pub enum SideEffectNoResult { Inst { inst: MInst }, } -/// Internal type ProducesFlags: defined at src/prelude.isle line 314. +/// Internal type ProducesFlags: defined at src/prelude.isle line 326. #[derive(Clone, Debug)] pub enum ProducesFlags { ProducesFlags { inst: MInst, result: Reg }, } -/// Internal type ConsumesFlags: defined at src/prelude.isle line 317. +/// Internal type ConsumesFlags: defined at src/prelude.isle line 329. #[derive(Clone, Debug)] pub enum ConsumesFlags { ConsumesFlags { inst: MInst, result: Reg }, @@ -683,7 +683,7 @@ pub enum MInst { } /// Internal type ALUOp: defined at src/isa/aarch64/inst.isle line 789. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum ALUOp { Add32, Add64, @@ -730,7 +730,7 @@ pub enum ALUOp { } /// Internal type ALUOp3: defined at src/isa/aarch64/inst.isle line 850. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum ALUOp3 { MAdd32, MAdd64, @@ -738,8 +738,8 @@ pub enum ALUOp3 { MSub64, } -/// Internal type BitOp: defined at src/isa/aarch64/inst.isle line 898. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type BitOp: defined at src/isa/aarch64/inst.isle line 893. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum BitOp { RBit32, RBit64, @@ -749,8 +749,8 @@ pub enum BitOp { Cls64, } -/// Internal type FPUOp1: defined at src/isa/aarch64/inst.isle line 964. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type FPUOp1: defined at src/isa/aarch64/inst.isle line 959. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum FPUOp1 { Abs32, Abs64, @@ -762,8 +762,8 @@ pub enum FPUOp1 { Cvt64To32, } -/// Internal type FPUOp2: defined at src/isa/aarch64/inst.isle line 977. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type FPUOp2: defined at src/isa/aarch64/inst.isle line 972. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum FPUOp2 { Add32, Add64, @@ -783,15 +783,15 @@ pub enum FPUOp2 { Uqsub64, } -/// Internal type FPUOp3: defined at src/isa/aarch64/inst.isle line 1002. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type FPUOp3: defined at src/isa/aarch64/inst.isle line 997. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum FPUOp3 { MAdd32, MAdd64, } -/// Internal type FpuToIntOp: defined at src/isa/aarch64/inst.isle line 1009. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type FpuToIntOp: defined at src/isa/aarch64/inst.isle line 1004. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum FpuToIntOp { F32ToU32, F32ToI32, @@ -803,8 +803,8 @@ pub enum FpuToIntOp { F64ToI64, } -/// Internal type IntToFpuOp: defined at src/isa/aarch64/inst.isle line 1022. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type IntToFpuOp: defined at src/isa/aarch64/inst.isle line 1017. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum IntToFpuOp { U32ToF32, I32ToF32, @@ -816,8 +816,8 @@ pub enum IntToFpuOp { I64ToF64, } -/// Internal type FpuRoundMode: defined at src/isa/aarch64/inst.isle line 1036. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type FpuRoundMode: defined at src/isa/aarch64/inst.isle line 1031. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum FpuRoundMode { Minus32, Minus64, @@ -829,8 +829,8 @@ pub enum FpuRoundMode { Nearest64, } -/// Internal type VecExtendOp: defined at src/isa/aarch64/inst.isle line 1049. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type VecExtendOp: defined at src/isa/aarch64/inst.isle line 1044. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum VecExtendOp { Sxtl8, Sxtl16, @@ -840,8 +840,8 @@ pub enum VecExtendOp { Uxtl32, } -/// Internal type VecALUOp: defined at src/isa/aarch64/inst.isle line 1066. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type VecALUOp: defined at src/isa/aarch64/inst.isle line 1061. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum VecALUOp { Sqadd, Uqadd, @@ -882,8 +882,8 @@ pub enum VecALUOp { Sqrdmulh, } -/// Internal type VecMisc2: defined at src/isa/aarch64/inst.isle line 1145. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type VecMisc2: defined at src/isa/aarch64/inst.isle line 1140. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum VecMisc2 { Not, Neg, @@ -904,8 +904,8 @@ pub enum VecMisc2 { Cmeq0, } -/// Internal type VecRRLongOp: defined at src/isa/aarch64/inst.isle line 1184. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type VecRRLongOp: defined at src/isa/aarch64/inst.isle line 1179. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum VecRRLongOp { Fcvtl16, Fcvtl32, @@ -914,8 +914,8 @@ pub enum VecRRLongOp { Shll32, } -/// Internal type VecRRNarrowOp: defined at src/isa/aarch64/inst.isle line 1199. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type VecRRNarrowOp: defined at src/isa/aarch64/inst.isle line 1194. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum VecRRNarrowOp { Xtn16, Xtn32, @@ -933,8 +933,8 @@ pub enum VecRRNarrowOp { Fcvtn64, } -/// Internal type VecRRRLongOp: defined at src/isa/aarch64/inst.isle line 1231. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type VecRRRLongOp: defined at src/isa/aarch64/inst.isle line 1226. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum VecRRRLongOp { Smull8, Smull16, @@ -947,14 +947,14 @@ pub enum VecRRRLongOp { Umlal32, } -/// Internal type VecPairOp: defined at src/isa/aarch64/inst.isle line 1248. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type VecPairOp: defined at src/isa/aarch64/inst.isle line 1243. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum VecPairOp { Addp, } -/// Internal type VecRRPairLongOp: defined at src/isa/aarch64/inst.isle line 1256. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type VecRRPairLongOp: defined at src/isa/aarch64/inst.isle line 1251. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum VecRRPairLongOp { Saddlp8, Saddlp16, @@ -962,23 +962,23 @@ pub enum VecRRPairLongOp { Uaddlp16, } -/// Internal type VecLanesOp: defined at src/isa/aarch64/inst.isle line 1267. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type VecLanesOp: defined at src/isa/aarch64/inst.isle line 1262. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum VecLanesOp { Addv, Uminv, } -/// Internal type VecShiftImmOp: defined at src/isa/aarch64/inst.isle line 1276. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type VecShiftImmOp: defined at src/isa/aarch64/inst.isle line 1271. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum VecShiftImmOp { Shl, Ushr, Sshr, } -/// Internal type AtomicRMWOp: defined at src/isa/aarch64/inst.isle line 1287. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type AtomicRMWOp: defined at src/isa/aarch64/inst.isle line 1282. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum AtomicRMWOp { Add, Clr, @@ -993,7 +993,7 @@ pub enum AtomicRMWOp { // Generated as internal constructor for term temp_reg. pub fn constructor_temp_reg(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; - // Rule at src/prelude.isle line 70. + // Rule at src/prelude.isle line 73. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); return Some(expr1_0); @@ -1002,7 +1002,7 @@ pub fn constructor_temp_reg(ctx: &mut C, arg0: Type) -> Option // Generated as internal constructor for term lo_reg. pub fn constructor_lo_reg(ctx: &mut C, arg0: Value) -> Option { let pattern0_0 = arg0; - // Rule at src/prelude.isle line 105. + // Rule at src/prelude.isle line 108. let expr0_0 = C::put_in_regs(ctx, pattern0_0); let expr1_0: usize = 0; let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); @@ -1019,7 +1019,7 @@ pub fn constructor_value_regs_none( inst: ref pattern1_0, } = pattern0_0 { - // Rule at src/prelude.isle line 300. + // Rule at src/prelude.isle line 312. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::value_regs_invalid(ctx); return Some(expr1_0); @@ -1037,7 +1037,7 @@ pub fn constructor_safepoint( inst: ref pattern1_0, } = pattern0_0 { - // Rule at src/prelude.isle line 306. + // Rule at src/prelude.isle line 318. let expr0_0 = C::emit_safepoint(ctx, &pattern1_0); let expr1_0 = C::value_regs_invalid(ctx); return Some(expr1_0); @@ -1063,7 +1063,7 @@ pub fn constructor_with_flags( result: pattern3_1, } = pattern2_0 { - // Rule at src/prelude.isle line 327. + // Rule at src/prelude.isle line 339. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern3_0); let expr2_0 = C::value_regs(ctx, pattern1_1, pattern3_1); @@ -1091,7 +1091,7 @@ pub fn constructor_with_flags_1( result: pattern3_1, } = pattern2_0 { - // Rule at src/prelude.isle line 335. + // Rule at src/prelude.isle line 347. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern3_0); return Some(pattern3_1); @@ -1125,7 +1125,7 @@ pub fn constructor_with_flags_2( result: pattern5_1, } = pattern4_0 { - // Rule at src/prelude.isle line 345. + // Rule at src/prelude.isle line 357. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern5_0); let expr2_0 = C::emit(ctx, &pattern3_0); @@ -1143,28 +1143,28 @@ pub fn constructor_vector_size(ctx: &mut C, arg0: Type) -> Option( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1383. + // Rule at src/isa/aarch64/inst.isle line 1378. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::MovZ { @@ -1202,7 +1202,7 @@ pub fn constructor_movn( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1390. + // Rule at src/isa/aarch64/inst.isle line 1385. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::MovN { @@ -1225,7 +1225,7 @@ pub fn constructor_alu_rr_imm_logic( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1397. + // Rule at src/isa/aarch64/inst.isle line 1392. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::AluRRImmLogic { @@ -1249,7 +1249,7 @@ pub fn constructor_alu_rr_imm_shift( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1404. + // Rule at src/isa/aarch64/inst.isle line 1399. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::AluRRImmShift { @@ -1273,7 +1273,7 @@ pub fn constructor_alu_rrr( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1411. + // Rule at src/isa/aarch64/inst.isle line 1406. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::AluRRR { @@ -1299,7 +1299,7 @@ pub fn constructor_vec_rrr( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1418. + // Rule at src/isa/aarch64/inst.isle line 1413. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::VecRRR { @@ -1324,7 +1324,7 @@ pub fn constructor_vec_lanes( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1425. + // Rule at src/isa/aarch64/inst.isle line 1420. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::VecLanes { @@ -1342,7 +1342,7 @@ pub fn constructor_vec_lanes( pub fn constructor_vec_dup(ctx: &mut C, arg0: Reg, arg1: &VectorSize) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1432. + // Rule at src/isa/aarch64/inst.isle line 1427. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::VecDup { @@ -1365,7 +1365,7 @@ pub fn constructor_alu_rr_imm12( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1439. + // Rule at src/isa/aarch64/inst.isle line 1434. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::AluRRImm12 { @@ -1391,7 +1391,7 @@ pub fn constructor_alu_rrr_shift( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1446. + // Rule at src/isa/aarch64/inst.isle line 1441. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::AluRRRShift { @@ -1418,7 +1418,7 @@ pub fn constructor_alu_rrr_extend( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1453. + // Rule at src/isa/aarch64/inst.isle line 1448. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::AluRRRExtend { @@ -1443,7 +1443,7 @@ pub fn constructor_alu_rr_extend_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1461. + // Rule at src/isa/aarch64/inst.isle line 1456. let expr0_0 = C::put_extended_in_reg(ctx, pattern2_0); let expr1_0 = C::get_extended_op(ctx, pattern2_0); let expr2_0 = constructor_alu_rrr_extend(ctx, pattern0_0, pattern1_0, expr0_0, &expr1_0)?; @@ -1462,7 +1462,7 @@ pub fn constructor_alu_rrrr( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1468. + // Rule at src/isa/aarch64/inst.isle line 1463. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::AluRRRR { @@ -1481,7 +1481,7 @@ pub fn constructor_alu_rrrr( pub fn constructor_bit_rr(ctx: &mut C, arg0: &BitOp, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1475. + // Rule at src/isa/aarch64/inst.isle line 1470. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::BitRR { @@ -1502,7 +1502,7 @@ pub fn constructor_add64_with_flags( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1482. + // Rule at src/isa/aarch64/inst.isle line 1477. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = ALUOp::AddS64; @@ -1524,7 +1524,7 @@ pub fn constructor_add64_with_flags( pub fn constructor_adc64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1489. + // Rule at src/isa/aarch64/inst.isle line 1484. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = ALUOp::Adc64; @@ -1550,7 +1550,7 @@ pub fn constructor_sub64_with_flags( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1496. + // Rule at src/isa/aarch64/inst.isle line 1491. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = ALUOp::SubS64; @@ -1576,7 +1576,7 @@ pub fn constructor_cmp64_imm( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1502. + // Rule at src/isa/aarch64/inst.isle line 1497. let expr0_0 = ALUOp::SubS64; let expr1_0 = C::writable_zero_reg(ctx); let expr2_0 = MInst::AluRRImm12 { @@ -1597,7 +1597,7 @@ pub fn constructor_cmp64_imm( pub fn constructor_sbc64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1508. + // Rule at src/isa/aarch64/inst.isle line 1503. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = ALUOp::Sbc64; @@ -1625,7 +1625,7 @@ pub fn constructor_vec_misc( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1515. + // Rule at src/isa/aarch64/inst.isle line 1510. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::VecMisc { @@ -1651,7 +1651,7 @@ pub fn constructor_vec_rrr_long( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1522. + // Rule at src/isa/aarch64/inst.isle line 1517. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::VecRRRLong { @@ -1680,7 +1680,7 @@ pub fn constructor_vec_rrrr_long( let pattern2_0 = arg2; let pattern3_0 = arg3; let pattern4_0 = arg4; - // Rule at src/isa/aarch64/inst.isle line 1532. + // Rule at src/isa/aarch64/inst.isle line 1527. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::FpuMove128 { @@ -1710,7 +1710,7 @@ pub fn constructor_vec_rr_narrow( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1540. + // Rule at src/isa/aarch64/inst.isle line 1535. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::VecRRNarrow { @@ -1734,7 +1734,7 @@ pub fn constructor_vec_rr_long( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1547. + // Rule at src/isa/aarch64/inst.isle line 1542. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::VecRRLong { @@ -1756,7 +1756,7 @@ pub fn constructor_mov_to_fpu( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1554. + // Rule at src/isa/aarch64/inst.isle line 1549. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::MovToFpu { @@ -1781,7 +1781,7 @@ pub fn constructor_mov_to_vec( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1561. + // Rule at src/isa/aarch64/inst.isle line 1556. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::FpuMove128 { @@ -1810,7 +1810,7 @@ pub fn constructor_mov_from_vec( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1569. + // Rule at src/isa/aarch64/inst.isle line 1564. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::MovFromVec { @@ -1836,7 +1836,7 @@ pub fn constructor_mov_from_vec_signed( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1576. + // Rule at src/isa/aarch64/inst.isle line 1571. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::MovFromVecSigned { @@ -1863,7 +1863,7 @@ pub fn constructor_extend( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1583. + // Rule at src/isa/aarch64/inst.isle line 1578. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::Extend { @@ -1882,7 +1882,7 @@ pub fn constructor_extend( pub fn constructor_load_acquire(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1590. + // Rule at src/isa/aarch64/inst.isle line 1585. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::LoadAcquire { @@ -1903,7 +1903,7 @@ pub fn constructor_tst64_imm( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1600. + // Rule at src/isa/aarch64/inst.isle line 1595. let expr0_0 = ALUOp::AndS64; let expr1_0 = C::writable_zero_reg(ctx); let expr2_0 = MInst::AluRRImmLogic { @@ -1930,7 +1930,7 @@ pub fn constructor_csel( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1613. + // Rule at src/isa/aarch64/inst.isle line 1608. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::CSel { @@ -1953,14 +1953,14 @@ pub fn constructor_add(ctx: &mut C, arg0: Type, arg1: Reg, arg2: Reg if pattern0_0 == I64 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1622. + // Rule at src/isa/aarch64/inst.isle line 1617. let expr0_0 = constructor_add64(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if let Some(pattern1_0) = C::fits_in_32(ctx, pattern0_0) { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1621. + // Rule at src/isa/aarch64/inst.isle line 1616. let expr0_0 = constructor_add32(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } @@ -1971,7 +1971,7 @@ pub fn constructor_add(ctx: &mut C, arg0: Type, arg1: Reg, arg2: Reg pub fn constructor_add32(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1625. + // Rule at src/isa/aarch64/inst.isle line 1620. let expr0_0 = ALUOp::Add32; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -1981,7 +1981,7 @@ pub fn constructor_add32(ctx: &mut C, arg0: Reg, arg1: Reg) -> Optio pub fn constructor_add64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1628. + // Rule at src/isa/aarch64/inst.isle line 1623. let expr0_0 = ALUOp::Add64; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -1998,14 +1998,14 @@ pub fn constructor_add_imm( if pattern0_0 == I64 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1632. + // Rule at src/isa/aarch64/inst.isle line 1627. let expr0_0 = constructor_add64_imm(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if let Some(pattern1_0) = C::fits_in_32(ctx, pattern0_0) { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1631. + // Rule at src/isa/aarch64/inst.isle line 1626. let expr0_0 = constructor_add32_imm(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } @@ -2016,7 +2016,7 @@ pub fn constructor_add_imm( pub fn constructor_add32_imm(ctx: &mut C, arg0: Reg, arg1: Imm12) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1635. + // Rule at src/isa/aarch64/inst.isle line 1630. let expr0_0 = ALUOp::Add32; let expr1_0 = constructor_alu_rr_imm12(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2026,7 +2026,7 @@ pub fn constructor_add32_imm(ctx: &mut C, arg0: Reg, arg1: Imm12) -> pub fn constructor_add64_imm(ctx: &mut C, arg0: Reg, arg1: Imm12) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1638. + // Rule at src/isa/aarch64/inst.isle line 1633. let expr0_0 = ALUOp::Add64; let expr1_0 = constructor_alu_rr_imm12(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2043,14 +2043,14 @@ pub fn constructor_add_extend( if pattern0_0 == I64 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1642. + // Rule at src/isa/aarch64/inst.isle line 1637. let expr0_0 = constructor_add64_extend(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if let Some(pattern1_0) = C::fits_in_32(ctx, pattern0_0) { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1641. + // Rule at src/isa/aarch64/inst.isle line 1636. let expr0_0 = constructor_add32_extend(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } @@ -2065,7 +2065,7 @@ pub fn constructor_add32_extend( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1645. + // Rule at src/isa/aarch64/inst.isle line 1640. let expr0_0 = ALUOp::Add32; let expr1_0 = constructor_alu_rr_extend_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2079,7 +2079,7 @@ pub fn constructor_add64_extend( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1648. + // Rule at src/isa/aarch64/inst.isle line 1643. let expr0_0 = ALUOp::Add64; let expr1_0 = constructor_alu_rr_extend_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2098,7 +2098,7 @@ pub fn constructor_add_shift( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1652. + // Rule at src/isa/aarch64/inst.isle line 1647. let expr0_0 = constructor_add64_shift(ctx, pattern2_0, pattern3_0, pattern4_0)?; return Some(expr0_0); } @@ -2106,7 +2106,7 @@ pub fn constructor_add_shift( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1651. + // Rule at src/isa/aarch64/inst.isle line 1646. let expr0_0 = constructor_add32_shift(ctx, pattern2_0, pattern3_0, pattern4_0)?; return Some(expr0_0); } @@ -2123,7 +2123,7 @@ pub fn constructor_add32_shift( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1655. + // Rule at src/isa/aarch64/inst.isle line 1650. let expr0_0 = ALUOp::Add32; let expr1_0 = constructor_alu_rrr_shift(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2139,7 +2139,7 @@ pub fn constructor_add64_shift( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1658. + // Rule at src/isa/aarch64/inst.isle line 1653. let expr0_0 = ALUOp::Add64; let expr1_0 = constructor_alu_rrr_shift(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2155,7 +2155,7 @@ pub fn constructor_add_vec( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1661. + // Rule at src/isa/aarch64/inst.isle line 1656. let expr0_0 = VecALUOp::Add; let expr1_0 = constructor_vec_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2167,14 +2167,14 @@ pub fn constructor_sub(ctx: &mut C, arg0: Type, arg1: Reg, arg2: Reg if pattern0_0 == I64 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1667. + // Rule at src/isa/aarch64/inst.isle line 1662. let expr0_0 = constructor_sub64(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if let Some(pattern1_0) = C::fits_in_32(ctx, pattern0_0) { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1666. + // Rule at src/isa/aarch64/inst.isle line 1661. let expr0_0 = constructor_sub32(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } @@ -2185,7 +2185,7 @@ pub fn constructor_sub(ctx: &mut C, arg0: Type, arg1: Reg, arg2: Reg pub fn constructor_sub32(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1670. + // Rule at src/isa/aarch64/inst.isle line 1665. let expr0_0 = ALUOp::Sub32; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2195,7 +2195,7 @@ pub fn constructor_sub32(ctx: &mut C, arg0: Reg, arg1: Reg) -> Optio pub fn constructor_sub64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1673. + // Rule at src/isa/aarch64/inst.isle line 1668. let expr0_0 = ALUOp::Sub64; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2212,14 +2212,14 @@ pub fn constructor_sub_imm( if pattern0_0 == I64 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1677. + // Rule at src/isa/aarch64/inst.isle line 1672. let expr0_0 = constructor_sub64_imm(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if let Some(pattern1_0) = C::fits_in_32(ctx, pattern0_0) { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1676. + // Rule at src/isa/aarch64/inst.isle line 1671. let expr0_0 = constructor_sub32_imm(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } @@ -2230,7 +2230,7 @@ pub fn constructor_sub_imm( pub fn constructor_sub32_imm(ctx: &mut C, arg0: Reg, arg1: Imm12) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1680. + // Rule at src/isa/aarch64/inst.isle line 1675. let expr0_0 = ALUOp::Sub32; let expr1_0 = constructor_alu_rr_imm12(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2240,7 +2240,7 @@ pub fn constructor_sub32_imm(ctx: &mut C, arg0: Reg, arg1: Imm12) -> pub fn constructor_sub64_imm(ctx: &mut C, arg0: Reg, arg1: Imm12) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1683. + // Rule at src/isa/aarch64/inst.isle line 1678. let expr0_0 = ALUOp::Sub64; let expr1_0 = constructor_alu_rr_imm12(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2257,14 +2257,14 @@ pub fn constructor_sub_extend( if pattern0_0 == I64 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1687. + // Rule at src/isa/aarch64/inst.isle line 1682. let expr0_0 = constructor_sub64_extend(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if let Some(pattern1_0) = C::fits_in_32(ctx, pattern0_0) { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1686. + // Rule at src/isa/aarch64/inst.isle line 1681. let expr0_0 = constructor_sub32_extend(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } @@ -2279,7 +2279,7 @@ pub fn constructor_sub32_extend( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1690. + // Rule at src/isa/aarch64/inst.isle line 1685. let expr0_0 = ALUOp::Sub32; let expr1_0 = constructor_alu_rr_extend_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2293,7 +2293,7 @@ pub fn constructor_sub64_extend( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1693. + // Rule at src/isa/aarch64/inst.isle line 1688. let expr0_0 = ALUOp::Sub64; let expr1_0 = constructor_alu_rr_extend_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2312,7 +2312,7 @@ pub fn constructor_sub_shift( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1697. + // Rule at src/isa/aarch64/inst.isle line 1692. let expr0_0 = constructor_sub64_shift(ctx, pattern2_0, pattern3_0, pattern4_0)?; return Some(expr0_0); } @@ -2320,7 +2320,7 @@ pub fn constructor_sub_shift( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1696. + // Rule at src/isa/aarch64/inst.isle line 1691. let expr0_0 = constructor_sub32_shift(ctx, pattern2_0, pattern3_0, pattern4_0)?; return Some(expr0_0); } @@ -2337,7 +2337,7 @@ pub fn constructor_sub32_shift( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1700. + // Rule at src/isa/aarch64/inst.isle line 1695. let expr0_0 = ALUOp::Sub32; let expr1_0 = constructor_alu_rrr_shift(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2353,7 +2353,7 @@ pub fn constructor_sub64_shift( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1703. + // Rule at src/isa/aarch64/inst.isle line 1698. let expr0_0 = ALUOp::Sub64; let expr1_0 = constructor_alu_rrr_shift(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2369,7 +2369,7 @@ pub fn constructor_sub_vec( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1706. + // Rule at src/isa/aarch64/inst.isle line 1701. let expr0_0 = VecALUOp::Sub; let expr1_0 = constructor_vec_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2388,7 +2388,7 @@ pub fn constructor_madd( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1712. + // Rule at src/isa/aarch64/inst.isle line 1707. let expr0_0 = constructor_madd64(ctx, pattern2_0, pattern3_0, pattern4_0)?; return Some(expr0_0); } @@ -2396,7 +2396,7 @@ pub fn constructor_madd( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1711. + // Rule at src/isa/aarch64/inst.isle line 1706. let expr0_0 = constructor_madd32(ctx, pattern2_0, pattern3_0, pattern4_0)?; return Some(expr0_0); } @@ -2408,7 +2408,7 @@ pub fn constructor_madd32(ctx: &mut C, arg0: Reg, arg1: Reg, arg2: R let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1715. + // Rule at src/isa/aarch64/inst.isle line 1710. let expr0_0 = ALUOp3::MAdd32; let expr1_0 = constructor_alu_rrrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2419,7 +2419,7 @@ pub fn constructor_madd64(ctx: &mut C, arg0: Reg, arg1: Reg, arg2: R let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1718. + // Rule at src/isa/aarch64/inst.isle line 1713. let expr0_0 = ALUOp3::MAdd64; let expr1_0 = constructor_alu_rrrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2430,7 +2430,7 @@ pub fn constructor_msub64(ctx: &mut C, arg0: Reg, arg1: Reg, arg2: R let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1723. + // Rule at src/isa/aarch64/inst.isle line 1718. let expr0_0 = ALUOp3::MSub64; let expr1_0 = constructor_alu_rrrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2446,7 +2446,7 @@ pub fn constructor_uqadd( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1727. + // Rule at src/isa/aarch64/inst.isle line 1722. let expr0_0 = VecALUOp::Uqadd; let expr1_0 = constructor_vec_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2462,7 +2462,7 @@ pub fn constructor_sqadd( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1731. + // Rule at src/isa/aarch64/inst.isle line 1726. let expr0_0 = VecALUOp::Sqadd; let expr1_0 = constructor_vec_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2478,7 +2478,7 @@ pub fn constructor_uqsub( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1735. + // Rule at src/isa/aarch64/inst.isle line 1730. let expr0_0 = VecALUOp::Uqsub; let expr1_0 = constructor_vec_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2494,7 +2494,7 @@ pub fn constructor_sqsub( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1739. + // Rule at src/isa/aarch64/inst.isle line 1734. let expr0_0 = VecALUOp::Sqsub; let expr1_0 = constructor_vec_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2504,7 +2504,7 @@ pub fn constructor_sqsub( pub fn constructor_umulh(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1743. + // Rule at src/isa/aarch64/inst.isle line 1738. let expr0_0 = ALUOp::UMulH; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2514,7 +2514,7 @@ pub fn constructor_umulh(ctx: &mut C, arg0: Reg, arg1: Reg) -> Optio pub fn constructor_smulh(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1747. + // Rule at src/isa/aarch64/inst.isle line 1742. let expr0_0 = ALUOp::SMulH; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2530,7 +2530,7 @@ pub fn constructor_mul( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1751. + // Rule at src/isa/aarch64/inst.isle line 1746. let expr0_0 = VecALUOp::Mul; let expr1_0 = constructor_vec_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2540,7 +2540,7 @@ pub fn constructor_mul( pub fn constructor_neg(ctx: &mut C, arg0: Reg, arg1: &VectorSize) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1755. + // Rule at src/isa/aarch64/inst.isle line 1750. let expr0_0 = VecMisc2::Neg; let expr1_0 = constructor_vec_misc(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2550,7 +2550,7 @@ pub fn constructor_neg(ctx: &mut C, arg0: Reg, arg1: &VectorSize) -> pub fn constructor_rev64(ctx: &mut C, arg0: Reg, arg1: &VectorSize) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1759. + // Rule at src/isa/aarch64/inst.isle line 1754. let expr0_0 = VecMisc2::Rev64; let expr1_0 = constructor_vec_misc(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2560,7 +2560,7 @@ pub fn constructor_rev64(ctx: &mut C, arg0: Reg, arg1: &VectorSize) pub fn constructor_xtn64(ctx: &mut C, arg0: Reg, arg1: bool) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1763. + // Rule at src/isa/aarch64/inst.isle line 1758. let expr0_0 = VecRRNarrowOp::Xtn64; let expr1_0 = constructor_vec_rr_narrow(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2576,7 +2576,7 @@ pub fn constructor_addp( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1767. + // Rule at src/isa/aarch64/inst.isle line 1762. let expr0_0 = VecALUOp::Addp; let expr1_0 = constructor_vec_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2586,7 +2586,7 @@ pub fn constructor_addp( pub fn constructor_addv(ctx: &mut C, arg0: Reg, arg1: &VectorSize) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1771. + // Rule at src/isa/aarch64/inst.isle line 1766. let expr0_0 = VecLanesOp::Addv; let expr1_0 = constructor_vec_lanes(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2596,7 +2596,7 @@ pub fn constructor_addv(ctx: &mut C, arg0: Reg, arg1: &VectorSize) - pub fn constructor_shll32(ctx: &mut C, arg0: Reg, arg1: bool) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1775. + // Rule at src/isa/aarch64/inst.isle line 1770. let expr0_0 = VecRRLongOp::Shll32; let expr1_0 = constructor_vec_rr_long(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2614,7 +2614,7 @@ pub fn constructor_umlal32( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1779. + // Rule at src/isa/aarch64/inst.isle line 1774. let expr0_0 = VecRRRLongOp::Umlal32; let expr1_0 = constructor_vec_rrrr_long( ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0, pattern3_0, @@ -2632,7 +2632,7 @@ pub fn constructor_smull8( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1783. + // Rule at src/isa/aarch64/inst.isle line 1778. let expr0_0 = VecRRRLongOp::Smull8; let expr1_0 = constructor_vec_rrr_long(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2648,7 +2648,7 @@ pub fn constructor_umull8( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1787. + // Rule at src/isa/aarch64/inst.isle line 1782. let expr0_0 = VecRRRLongOp::Umull8; let expr1_0 = constructor_vec_rrr_long(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2664,7 +2664,7 @@ pub fn constructor_smull16( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1791. + // Rule at src/isa/aarch64/inst.isle line 1786. let expr0_0 = VecRRRLongOp::Smull16; let expr1_0 = constructor_vec_rrr_long(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2680,7 +2680,7 @@ pub fn constructor_umull16( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1795. + // Rule at src/isa/aarch64/inst.isle line 1790. let expr0_0 = VecRRRLongOp::Umull16; let expr1_0 = constructor_vec_rrr_long(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2696,7 +2696,7 @@ pub fn constructor_smull32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1799. + // Rule at src/isa/aarch64/inst.isle line 1794. let expr0_0 = VecRRRLongOp::Smull32; let expr1_0 = constructor_vec_rrr_long(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2712,7 +2712,7 @@ pub fn constructor_umull32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1803. + // Rule at src/isa/aarch64/inst.isle line 1798. let expr0_0 = VecRRRLongOp::Umull32; let expr1_0 = constructor_vec_rrr_long(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2722,7 +2722,7 @@ pub fn constructor_umull32( pub fn constructor_asr64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1807. + // Rule at src/isa/aarch64/inst.isle line 1802. let expr0_0 = ALUOp::Asr64; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2732,7 +2732,7 @@ pub fn constructor_asr64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Optio pub fn constructor_asr64_imm(ctx: &mut C, arg0: Reg, arg1: ImmShift) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1810. + // Rule at src/isa/aarch64/inst.isle line 1805. let expr0_0 = ALUOp::Asr64; let expr1_0 = constructor_alu_rr_imm_shift(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2742,7 +2742,7 @@ pub fn constructor_asr64_imm(ctx: &mut C, arg0: Reg, arg1: ImmShift) pub fn constructor_lsr32(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1814. + // Rule at src/isa/aarch64/inst.isle line 1809. let expr0_0 = ALUOp::Lsr32; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2752,7 +2752,7 @@ pub fn constructor_lsr32(ctx: &mut C, arg0: Reg, arg1: Reg) -> Optio pub fn constructor_lsr32_imm(ctx: &mut C, arg0: Reg, arg1: ImmShift) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1817. + // Rule at src/isa/aarch64/inst.isle line 1812. let expr0_0 = ALUOp::Lsr32; let expr1_0 = constructor_alu_rr_imm_shift(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2762,7 +2762,7 @@ pub fn constructor_lsr32_imm(ctx: &mut C, arg0: Reg, arg1: ImmShift) pub fn constructor_lsr64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1820. + // Rule at src/isa/aarch64/inst.isle line 1815. let expr0_0 = ALUOp::Lsr64; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2772,7 +2772,7 @@ pub fn constructor_lsr64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Optio pub fn constructor_lsr64_imm(ctx: &mut C, arg0: Reg, arg1: ImmShift) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1823. + // Rule at src/isa/aarch64/inst.isle line 1818. let expr0_0 = ALUOp::Lsr64; let expr1_0 = constructor_alu_rr_imm_shift(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2782,7 +2782,7 @@ pub fn constructor_lsr64_imm(ctx: &mut C, arg0: Reg, arg1: ImmShift) pub fn constructor_lsl32(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1827. + // Rule at src/isa/aarch64/inst.isle line 1822. let expr0_0 = ALUOp::Lsl32; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2792,7 +2792,7 @@ pub fn constructor_lsl32(ctx: &mut C, arg0: Reg, arg1: Reg) -> Optio pub fn constructor_lsl32_imm(ctx: &mut C, arg0: Reg, arg1: ImmShift) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1830. + // Rule at src/isa/aarch64/inst.isle line 1825. let expr0_0 = ALUOp::Lsl32; let expr1_0 = constructor_alu_rr_imm_shift(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2802,7 +2802,7 @@ pub fn constructor_lsl32_imm(ctx: &mut C, arg0: Reg, arg1: ImmShift) pub fn constructor_lsl64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1833. + // Rule at src/isa/aarch64/inst.isle line 1828. let expr0_0 = ALUOp::Lsl64; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2812,7 +2812,7 @@ pub fn constructor_lsl64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Optio pub fn constructor_lsl64_imm(ctx: &mut C, arg0: Reg, arg1: ImmShift) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1836. + // Rule at src/isa/aarch64/inst.isle line 1831. let expr0_0 = ALUOp::Lsl64; let expr1_0 = constructor_alu_rr_imm_shift(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2822,7 +2822,7 @@ pub fn constructor_lsl64_imm(ctx: &mut C, arg0: Reg, arg1: ImmShift) pub fn constructor_udiv64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1840. + // Rule at src/isa/aarch64/inst.isle line 1835. let expr0_0 = ALUOp::UDiv64; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2832,7 +2832,7 @@ pub fn constructor_udiv64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Opti pub fn constructor_sdiv64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1844. + // Rule at src/isa/aarch64/inst.isle line 1839. let expr0_0 = ALUOp::SDiv64; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2842,7 +2842,7 @@ pub fn constructor_sdiv64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Opti pub fn constructor_not(ctx: &mut C, arg0: Reg, arg1: &VectorSize) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1848. + // Rule at src/isa/aarch64/inst.isle line 1843. let expr0_0 = VecMisc2::Not; let expr1_0 = constructor_vec_misc(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2859,14 +2859,14 @@ pub fn constructor_orr_not( if pattern0_0 == I64 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1854. + // Rule at src/isa/aarch64/inst.isle line 1849. let expr0_0 = constructor_orr_not64(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if let Some(pattern1_0) = C::fits_in_32(ctx, pattern0_0) { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1853. + // Rule at src/isa/aarch64/inst.isle line 1848. let expr0_0 = constructor_orr_not32(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } @@ -2877,7 +2877,7 @@ pub fn constructor_orr_not( pub fn constructor_orr_not32(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1857. + // Rule at src/isa/aarch64/inst.isle line 1852. let expr0_0 = ALUOp::OrrNot32; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2887,7 +2887,7 @@ pub fn constructor_orr_not32(ctx: &mut C, arg0: Reg, arg1: Reg) -> O pub fn constructor_orr_not64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1860. + // Rule at src/isa/aarch64/inst.isle line 1855. let expr0_0 = ALUOp::OrrNot64; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2906,7 +2906,7 @@ pub fn constructor_orr_not_shift( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1864. + // Rule at src/isa/aarch64/inst.isle line 1859. let expr0_0 = constructor_orr_not_shift64(ctx, pattern2_0, pattern3_0, pattern4_0)?; return Some(expr0_0); } @@ -2914,7 +2914,7 @@ pub fn constructor_orr_not_shift( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1863. + // Rule at src/isa/aarch64/inst.isle line 1858. let expr0_0 = constructor_orr_not_shift32(ctx, pattern2_0, pattern3_0, pattern4_0)?; return Some(expr0_0); } @@ -2931,7 +2931,7 @@ pub fn constructor_orr_not_shift32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1867. + // Rule at src/isa/aarch64/inst.isle line 1862. let expr0_0 = ALUOp::OrrNot32; let expr1_0 = constructor_alu_rrr_shift(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2947,7 +2947,7 @@ pub fn constructor_orr_not_shift64( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1870. + // Rule at src/isa/aarch64/inst.isle line 1865. let expr0_0 = ALUOp::OrrNot64; let expr1_0 = constructor_alu_rrr_shift(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2957,7 +2957,7 @@ pub fn constructor_orr_not_shift64( pub fn constructor_orr32(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1875. + // Rule at src/isa/aarch64/inst.isle line 1870. let expr0_0 = ALUOp::Orr32; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2967,7 +2967,7 @@ pub fn constructor_orr32(ctx: &mut C, arg0: Reg, arg1: Reg) -> Optio pub fn constructor_orr32_imm(ctx: &mut C, arg0: Reg, arg1: ImmLogic) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1878. + // Rule at src/isa/aarch64/inst.isle line 1873. let expr0_0 = ALUOp::Orr32; let expr1_0 = constructor_alu_rr_imm_logic(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2977,7 +2977,7 @@ pub fn constructor_orr32_imm(ctx: &mut C, arg0: Reg, arg1: ImmLogic) pub fn constructor_orr64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1881. + // Rule at src/isa/aarch64/inst.isle line 1876. let expr0_0 = ALUOp::Orr64; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2987,7 +2987,7 @@ pub fn constructor_orr64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Optio pub fn constructor_orr64_imm(ctx: &mut C, arg0: Reg, arg1: ImmLogic) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1884. + // Rule at src/isa/aarch64/inst.isle line 1879. let expr0_0 = ALUOp::Orr64; let expr1_0 = constructor_alu_rr_imm_logic(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -3003,7 +3003,7 @@ pub fn constructor_orr_vec( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1887. + // Rule at src/isa/aarch64/inst.isle line 1882. let expr0_0 = VecALUOp::Orr; let expr1_0 = constructor_vec_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -3013,7 +3013,7 @@ pub fn constructor_orr_vec( pub fn constructor_and32_imm(ctx: &mut C, arg0: Reg, arg1: ImmLogic) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1892. + // Rule at src/isa/aarch64/inst.isle line 1887. let expr0_0 = ALUOp::And32; let expr1_0 = constructor_alu_rr_imm_logic(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -3029,7 +3029,7 @@ pub fn constructor_and_vec( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1895. + // Rule at src/isa/aarch64/inst.isle line 1890. let expr0_0 = VecALUOp::And; let expr1_0 = constructor_vec_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -3045,7 +3045,7 @@ pub fn constructor_eor_vec( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1899. + // Rule at src/isa/aarch64/inst.isle line 1894. let expr0_0 = VecALUOp::Eor; let expr1_0 = constructor_vec_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -3061,7 +3061,7 @@ pub fn constructor_bic_vec( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1903. + // Rule at src/isa/aarch64/inst.isle line 1898. let expr0_0 = VecALUOp::Bic; let expr1_0 = constructor_vec_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -3077,7 +3077,7 @@ pub fn constructor_sshl( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1907. + // Rule at src/isa/aarch64/inst.isle line 1902. let expr0_0 = VecALUOp::Sshl; let expr1_0 = constructor_vec_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -3093,7 +3093,7 @@ pub fn constructor_ushl( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1911. + // Rule at src/isa/aarch64/inst.isle line 1906. let expr0_0 = VecALUOp::Ushl; let expr1_0 = constructor_vec_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -3103,7 +3103,7 @@ pub fn constructor_ushl( pub fn constructor_rotr32(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1916. + // Rule at src/isa/aarch64/inst.isle line 1911. let expr0_0 = ALUOp::RotR32; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -3113,7 +3113,7 @@ pub fn constructor_rotr32(ctx: &mut C, arg0: Reg, arg1: Reg) -> Opti pub fn constructor_rotr32_imm(ctx: &mut C, arg0: Reg, arg1: ImmShift) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1919. + // Rule at src/isa/aarch64/inst.isle line 1914. let expr0_0 = ALUOp::RotR32; let expr1_0 = constructor_alu_rr_imm_shift(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -3123,7 +3123,7 @@ pub fn constructor_rotr32_imm(ctx: &mut C, arg0: Reg, arg1: ImmShift pub fn constructor_rotr64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1922. + // Rule at src/isa/aarch64/inst.isle line 1917. let expr0_0 = ALUOp::RotR64; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -3133,7 +3133,7 @@ pub fn constructor_rotr64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Opti pub fn constructor_rotr64_imm(ctx: &mut C, arg0: Reg, arg1: ImmShift) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1925. + // Rule at src/isa/aarch64/inst.isle line 1920. let expr0_0 = ALUOp::RotR64; let expr1_0 = constructor_alu_rr_imm_shift(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -3142,7 +3142,7 @@ pub fn constructor_rotr64_imm(ctx: &mut C, arg0: Reg, arg1: ImmShift // Generated as internal constructor for term rbit32. pub fn constructor_rbit32(ctx: &mut C, arg0: Reg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/aarch64/inst.isle line 1930. + // Rule at src/isa/aarch64/inst.isle line 1925. let expr0_0 = BitOp::RBit32; let expr1_0 = constructor_bit_rr(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -3151,7 +3151,7 @@ pub fn constructor_rbit32(ctx: &mut C, arg0: Reg) -> Option { // Generated as internal constructor for term rbit64. pub fn constructor_rbit64(ctx: &mut C, arg0: Reg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/aarch64/inst.isle line 1933. + // Rule at src/isa/aarch64/inst.isle line 1928. let expr0_0 = BitOp::RBit64; let expr1_0 = constructor_bit_rr(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -3160,7 +3160,7 @@ pub fn constructor_rbit64(ctx: &mut C, arg0: Reg) -> Option { // Generated as internal constructor for term clz32. pub fn constructor_clz32(ctx: &mut C, arg0: Reg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/aarch64/inst.isle line 1938. + // Rule at src/isa/aarch64/inst.isle line 1933. let expr0_0 = BitOp::Clz32; let expr1_0 = constructor_bit_rr(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -3169,7 +3169,7 @@ pub fn constructor_clz32(ctx: &mut C, arg0: Reg) -> Option { // Generated as internal constructor for term clz64. pub fn constructor_clz64(ctx: &mut C, arg0: Reg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/aarch64/inst.isle line 1941. + // Rule at src/isa/aarch64/inst.isle line 1936. let expr0_0 = BitOp::Clz64; let expr1_0 = constructor_bit_rr(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -3178,7 +3178,7 @@ pub fn constructor_clz64(ctx: &mut C, arg0: Reg) -> Option { // Generated as internal constructor for term cls32. pub fn constructor_cls32(ctx: &mut C, arg0: Reg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/aarch64/inst.isle line 1946. + // Rule at src/isa/aarch64/inst.isle line 1941. let expr0_0 = BitOp::Cls32; let expr1_0 = constructor_bit_rr(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -3187,7 +3187,7 @@ pub fn constructor_cls32(ctx: &mut C, arg0: Reg) -> Option { // Generated as internal constructor for term cls64. pub fn constructor_cls64(ctx: &mut C, arg0: Reg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/aarch64/inst.isle line 1949. + // Rule at src/isa/aarch64/inst.isle line 1944. let expr0_0 = BitOp::Cls64; let expr1_0 = constructor_bit_rr(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -3197,7 +3197,7 @@ pub fn constructor_cls64(ctx: &mut C, arg0: Reg) -> Option { pub fn constructor_eon32(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1954. + // Rule at src/isa/aarch64/inst.isle line 1949. let expr0_0 = ALUOp::EorNot32; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -3207,7 +3207,7 @@ pub fn constructor_eon32(ctx: &mut C, arg0: Reg, arg1: Reg) -> Optio pub fn constructor_eon64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1957. + // Rule at src/isa/aarch64/inst.isle line 1952. let expr0_0 = ALUOp::EorNot64; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -3217,7 +3217,7 @@ pub fn constructor_eon64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Optio pub fn constructor_vec_cnt(ctx: &mut C, arg0: Reg, arg1: &VectorSize) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1962. + // Rule at src/isa/aarch64/inst.isle line 1957. let expr0_0 = VecMisc2::Cnt; let expr1_0 = constructor_vec_misc(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -3234,25 +3234,25 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option }; if let Some(pattern3_0) = closure3() { if let Some(pattern4_0) = C::imm_logic_from_u64(ctx, pattern2_0, pattern3_0) { - // Rule at src/isa/aarch64/inst.isle line 1977. + // Rule at src/isa/aarch64/inst.isle line 1972. let expr0_0 = C::zero_reg(ctx); let expr1_0 = constructor_orr64_imm(ctx, expr0_0, pattern4_0)?; return Some(expr1_0); } } if let Some(pattern3_0) = C::move_wide_const_from_u64(ctx, pattern2_0) { - // Rule at src/isa/aarch64/inst.isle line 1969. + // Rule at src/isa/aarch64/inst.isle line 1964. let expr0_0 = OperandSize::Size64; let expr1_0 = constructor_movz(ctx, pattern3_0, &expr0_0)?; return Some(expr1_0); } if let Some(pattern3_0) = C::move_wide_const_from_negated_u64(ctx, pattern2_0) { - // Rule at src/isa/aarch64/inst.isle line 1973. + // Rule at src/isa/aarch64/inst.isle line 1968. let expr0_0 = OperandSize::Size64; let expr1_0 = constructor_movn(ctx, pattern3_0, &expr0_0)?; return Some(expr1_0); } - // Rule at src/isa/aarch64/inst.isle line 1984. + // Rule at src/isa/aarch64/inst.isle line 1979. let expr0_0 = C::load_constant64_full(ctx, pattern2_0); return Some(expr0_0); } @@ -3264,17 +3264,17 @@ pub fn constructor_put_in_reg_sext32(ctx: &mut C, arg0: Value) -> Op let pattern0_0 = arg0; let pattern1_0 = C::value_type(ctx, pattern0_0); if pattern1_0 == I32 { - // Rule at src/isa/aarch64/inst.isle line 1995. + // Rule at src/isa/aarch64/inst.isle line 1990. let expr0_0 = C::put_in_reg(ctx, pattern0_0); return Some(expr0_0); } if pattern1_0 == I64 { - // Rule at src/isa/aarch64/inst.isle line 1996. + // Rule at src/isa/aarch64/inst.isle line 1991. let expr0_0 = C::put_in_reg(ctx, pattern0_0); return Some(expr0_0); } if let Some(pattern2_0) = C::fits_in_32(ctx, pattern1_0) { - // Rule at src/isa/aarch64/inst.isle line 1991. + // Rule at src/isa/aarch64/inst.isle line 1986. let expr0_0 = C::put_in_reg(ctx, pattern0_0); let expr1_0: bool = true; let expr2_0 = C::ty_bits(ctx, pattern2_0); @@ -3290,17 +3290,17 @@ pub fn constructor_put_in_reg_zext32(ctx: &mut C, arg0: Value) -> Op let pattern0_0 = arg0; let pattern1_0 = C::value_type(ctx, pattern0_0); if pattern1_0 == I32 { - // Rule at src/isa/aarch64/inst.isle line 2004. + // Rule at src/isa/aarch64/inst.isle line 1999. let expr0_0 = C::put_in_reg(ctx, pattern0_0); return Some(expr0_0); } if pattern1_0 == I64 { - // Rule at src/isa/aarch64/inst.isle line 2005. + // Rule at src/isa/aarch64/inst.isle line 2000. let expr0_0 = C::put_in_reg(ctx, pattern0_0); return Some(expr0_0); } if let Some(pattern2_0) = C::fits_in_32(ctx, pattern1_0) { - // Rule at src/isa/aarch64/inst.isle line 2000. + // Rule at src/isa/aarch64/inst.isle line 1995. let expr0_0 = C::put_in_reg(ctx, pattern0_0); let expr1_0: bool = false; let expr2_0 = C::ty_bits(ctx, pattern2_0); @@ -3316,12 +3316,12 @@ pub fn constructor_put_in_reg_sext64(ctx: &mut C, arg0: Value) -> Op let pattern0_0 = arg0; let pattern1_0 = C::value_type(ctx, pattern0_0); if pattern1_0 == I64 { - // Rule at src/isa/aarch64/inst.isle line 2013. + // Rule at src/isa/aarch64/inst.isle line 2008. let expr0_0 = C::put_in_reg(ctx, pattern0_0); return Some(expr0_0); } if let Some(pattern2_0) = C::fits_in_32(ctx, pattern1_0) { - // Rule at src/isa/aarch64/inst.isle line 2009. + // Rule at src/isa/aarch64/inst.isle line 2004. let expr0_0 = C::put_in_reg(ctx, pattern0_0); let expr1_0: bool = true; let expr2_0 = C::ty_bits(ctx, pattern2_0); @@ -3337,12 +3337,12 @@ pub fn constructor_put_in_reg_zext64(ctx: &mut C, arg0: Value) -> Op let pattern0_0 = arg0; let pattern1_0 = C::value_type(ctx, pattern0_0); if pattern1_0 == I64 { - // Rule at src/isa/aarch64/inst.isle line 2021. + // Rule at src/isa/aarch64/inst.isle line 2016. let expr0_0 = C::put_in_reg(ctx, pattern0_0); return Some(expr0_0); } if let Some(pattern2_0) = C::fits_in_32(ctx, pattern1_0) { - // Rule at src/isa/aarch64/inst.isle line 2017. + // Rule at src/isa/aarch64/inst.isle line 2012. let expr0_0 = C::put_in_reg(ctx, pattern0_0); let expr1_0: bool = false; let expr2_0 = C::ty_bits(ctx, pattern2_0); @@ -3356,7 +3356,7 @@ pub fn constructor_put_in_reg_zext64(ctx: &mut C, arg0: Value) -> Op // Generated as internal constructor for term trap_if_zero_divisor. pub fn constructor_trap_if_zero_divisor(ctx: &mut C, arg0: Reg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/aarch64/inst.isle line 2026. + // Rule at src/isa/aarch64/inst.isle line 2021. let expr0_0 = C::cond_br_zero(ctx, pattern0_0); let expr1_0 = C::trap_code_division_by_zero(ctx); let expr2_0 = MInst::TrapIf { @@ -3371,12 +3371,12 @@ pub fn constructor_trap_if_zero_divisor(ctx: &mut C, arg0: Reg) -> O pub fn constructor_size_from_ty(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I64 { - // Rule at src/isa/aarch64/inst.isle line 2032. + // Rule at src/isa/aarch64/inst.isle line 2027. let expr0_0 = OperandSize::Size64; return Some(expr0_0); } if let Some(pattern1_0) = C::fits_in_32(ctx, pattern0_0) { - // Rule at src/isa/aarch64/inst.isle line 2031. + // Rule at src/isa/aarch64/inst.isle line 2026. let expr0_0 = OperandSize::Size32; return Some(expr0_0); } @@ -3393,7 +3393,7 @@ pub fn constructor_trap_if_div_overflow( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 2038. + // Rule at src/isa/aarch64/inst.isle line 2033. let expr0_0 = constructor_adds_op(ctx, pattern0_0)?; let expr1_0 = C::writable_zero_reg(ctx); let expr2_0: u8 = 1; @@ -3437,12 +3437,12 @@ pub fn constructor_trap_if_div_overflow( pub fn constructor_adds_op(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I64 { - // Rule at src/isa/aarch64/inst.isle line 2058. + // Rule at src/isa/aarch64/inst.isle line 2053. let expr0_0 = ALUOp::AddS64; return Some(expr0_0); } if let Some(pattern1_0) = C::fits_in_32(ctx, pattern0_0) { - // Rule at src/isa/aarch64/inst.isle line 2057. + // Rule at src/isa/aarch64/inst.isle line 2052. let expr0_0 = ALUOp::AddS32; return Some(expr0_0); } @@ -3476,7 +3476,7 @@ pub fn constructor_alu_rs_imm_logic_commutative( C::imm_logic_from_imm64(ctx, pattern5_1, pattern7_0) { let pattern9_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 2088. + // Rule at src/isa/aarch64/inst.isle line 2083. let expr0_0 = C::put_in_reg(ctx, pattern9_0); let expr1_0 = constructor_alu_rr_imm_logic(ctx, pattern0_0, expr0_0, pattern8_0)?; @@ -3507,7 +3507,7 @@ pub fn constructor_alu_rs_imm_logic_commutative( C::lshl_from_imm64(ctx, pattern10_1, pattern12_0) { let pattern14_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 2094. + // Rule at src/isa/aarch64/inst.isle line 2089. let expr0_0 = C::put_in_reg(ctx, pattern14_0); let expr1_0 = C::put_in_reg(ctx, pattern7_0); let expr2_0 = constructor_alu_rrr_shift( @@ -3544,7 +3544,7 @@ pub fn constructor_alu_rs_imm_logic_commutative( if let Some(pattern9_0) = C::imm_logic_from_imm64(ctx, pattern6_1, pattern8_0) { - // Rule at src/isa/aarch64/inst.isle line 2086. + // Rule at src/isa/aarch64/inst.isle line 2081. let expr0_0 = C::put_in_reg(ctx, pattern2_0); let expr1_0 = constructor_alu_rr_imm_logic(ctx, pattern0_0, expr0_0, pattern9_0)?; @@ -3574,7 +3574,7 @@ pub fn constructor_alu_rs_imm_logic_commutative( if let Some(pattern14_0) = C::lshl_from_imm64(ctx, pattern11_1, pattern13_0) { - // Rule at src/isa/aarch64/inst.isle line 2092. + // Rule at src/isa/aarch64/inst.isle line 2087. let expr0_0 = C::put_in_reg(ctx, pattern2_0); let expr1_0 = C::put_in_reg(ctx, pattern8_0); let expr2_0 = constructor_alu_rrr_shift( @@ -3595,7 +3595,7 @@ pub fn constructor_alu_rs_imm_logic_commutative( _ => {} } } - // Rule at src/isa/aarch64/inst.isle line 2082. + // Rule at src/isa/aarch64/inst.isle line 2077. let expr0_0 = C::put_in_reg(ctx, pattern2_0); let expr1_0 = C::put_in_reg(ctx, pattern3_0); let expr2_0 = constructor_alu_rrr(ctx, pattern0_0, expr0_0, expr1_0)?; @@ -3629,7 +3629,7 @@ pub fn constructor_alu_rs_imm_logic( if let Some(pattern9_0) = C::imm_logic_from_imm64(ctx, pattern6_1, pattern8_0) { - // Rule at src/isa/aarch64/inst.isle line 2102. + // Rule at src/isa/aarch64/inst.isle line 2097. let expr0_0 = C::put_in_reg(ctx, pattern2_0); let expr1_0 = constructor_alu_rr_imm_logic(ctx, pattern0_0, expr0_0, pattern9_0)?; @@ -3659,7 +3659,7 @@ pub fn constructor_alu_rs_imm_logic( if let Some(pattern14_0) = C::lshl_from_imm64(ctx, pattern11_1, pattern13_0) { - // Rule at src/isa/aarch64/inst.isle line 2104. + // Rule at src/isa/aarch64/inst.isle line 2099. let expr0_0 = C::put_in_reg(ctx, pattern2_0); let expr1_0 = C::put_in_reg(ctx, pattern8_0); let expr2_0 = constructor_alu_rrr_shift( @@ -3680,7 +3680,7 @@ pub fn constructor_alu_rs_imm_logic( _ => {} } } - // Rule at src/isa/aarch64/inst.isle line 2100. + // Rule at src/isa/aarch64/inst.isle line 2095. let expr0_0 = C::put_in_reg(ctx, pattern2_0); let expr1_0 = C::put_in_reg(ctx, pattern3_0); let expr2_0 = constructor_alu_rrr(ctx, pattern0_0, expr0_0, expr1_0)?; @@ -3697,7 +3697,7 @@ pub fn constructor_i128_alu_bitop( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 2112. + // Rule at src/isa/aarch64/inst.isle line 2107. let expr0_0 = C::put_in_regs(ctx, pattern1_0); let expr1_0: usize = 0; let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); diff --git a/cranelift/codegen/src/isa/s390x/inst.isle b/cranelift/codegen/src/isa/s390x/inst.isle index 0e96092ef1bb..f5122148bc0a 100644 --- a/cranelift/codegen/src/isa/s390x/inst.isle +++ b/cranelift/codegen/src/isa/s390x/inst.isle @@ -678,11 +678,7 @@ (type BoxCallInfo (primitive BoxCallInfo)) (type BoxCallIndInfo (primitive BoxCallIndInfo)) -(type MachLabel (primitive MachLabel)) (type BoxJTSequenceInfo (primitive BoxJTSequenceInfo)) -(type BoxExternalName (primitive BoxExternalName)) -(type ValueLabel (primitive ValueLabel)) -(type UnwindInst (primitive UnwindInst)) ;; An ALU operation. (type ALUOp @@ -1041,10 +1037,6 @@ ;; Helpers for machine label vectors ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;; VecMachLabel needs to be passed by reference, so it cannot be -;; declared as primitive type. Declare as extern enum instead. -(type VecMachLabel extern (enum)) - (decl vec_length_minus1 (VecMachLabel) u32) (extern constructor vec_length_minus1 vec_length_minus1) @@ -2963,5 +2955,3 @@ (decl fcmp_reg (Type Reg Reg) ProducesFlags) (rule (fcmp_reg $F32 src1 src2) (fpu_cmp32 src1 src2)) (rule (fcmp_reg $F64 src1 src2) (fpu_cmp64 src1 src2)) - - diff --git a/cranelift/codegen/src/isa/s390x/lower/isle/generated_code.manifest b/cranelift/codegen/src/isa/s390x/lower/isle/generated_code.manifest index 9cbb4eaa96fa..eee9fc5608a3 100644 --- a/cranelift/codegen/src/isa/s390x/lower/isle/generated_code.manifest +++ b/cranelift/codegen/src/isa/s390x/lower/isle/generated_code.manifest @@ -1,4 +1,4 @@ src/clif.isle 9ea75a6f790b5c03 -src/prelude.isle 2bfcafbef6b29358 -src/isa/s390x/inst.isle 1d525c87f7c77c26 +src/prelude.isle 6aaf8ce0f5a5c2ec +src/isa/s390x/inst.isle f5af3708848ef1aa src/isa/s390x/lower.isle 57dcc39cbab2d1c6 diff --git a/cranelift/codegen/src/isa/s390x/lower/isle/generated_code.rs b/cranelift/codegen/src/isa/s390x/lower/isle/generated_code.rs index 96b59f69b2d5..2c52f5f2b96f 100644 --- a/cranelift/codegen/src/isa/s390x/lower/isle/generated_code.rs +++ b/cranelift/codegen/src/isa/s390x/lower/isle/generated_code.rs @@ -127,19 +127,19 @@ pub trait Context { fn sink_inst(&mut self, arg0: Inst) -> Unit; } -/// Internal type SideEffectNoResult: defined at src/prelude.isle line 295. +/// Internal type SideEffectNoResult: defined at src/prelude.isle line 307. #[derive(Clone, Debug)] pub enum SideEffectNoResult { Inst { inst: MInst }, } -/// Internal type ProducesFlags: defined at src/prelude.isle line 314. +/// Internal type ProducesFlags: defined at src/prelude.isle line 326. #[derive(Clone, Debug)] pub enum ProducesFlags { ProducesFlags { inst: MInst, result: Reg }, } -/// Internal type ConsumesFlags: defined at src/prelude.isle line 317. +/// Internal type ConsumesFlags: defined at src/prelude.isle line 329. #[derive(Clone, Debug)] pub enum ConsumesFlags { ConsumesFlags { inst: MInst, result: Reg }, @@ -652,8 +652,8 @@ pub enum MInst { }, } -/// Internal type ALUOp: defined at src/isa/s390x/inst.isle line 688. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type ALUOp: defined at src/isa/s390x/inst.isle line 684. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum ALUOp { Add32, Add32Ext16, @@ -690,8 +690,8 @@ pub enum ALUOp { XorNot64, } -/// Internal type UnaryOp: defined at src/isa/s390x/inst.isle line 729. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type UnaryOp: defined at src/isa/s390x/inst.isle line 725. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum UnaryOp { Abs32, Abs64, @@ -703,8 +703,8 @@ pub enum UnaryOp { PopcntReg, } -/// Internal type ShiftOp: defined at src/isa/s390x/inst.isle line 742. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type ShiftOp: defined at src/isa/s390x/inst.isle line 738. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum ShiftOp { RotL32, RotL64, @@ -716,8 +716,8 @@ pub enum ShiftOp { AShR64, } -/// Internal type CmpOp: defined at src/isa/s390x/inst.isle line 755. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type CmpOp: defined at src/isa/s390x/inst.isle line 751. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum CmpOp { CmpS32, CmpS32Ext16, @@ -731,8 +731,8 @@ pub enum CmpOp { CmpL64Ext32, } -/// Internal type FPUOp1: defined at src/isa/s390x/inst.isle line 770. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type FPUOp1: defined at src/isa/s390x/inst.isle line 766. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum FPUOp1 { Abs32, Abs64, @@ -746,8 +746,8 @@ pub enum FPUOp1 { Cvt64To32, } -/// Internal type FPUOp2: defined at src/isa/s390x/inst.isle line 785. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type FPUOp2: defined at src/isa/s390x/inst.isle line 781. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum FPUOp2 { Add32, Add64, @@ -763,8 +763,8 @@ pub enum FPUOp2 { Min64, } -/// Internal type FPUOp3: defined at src/isa/s390x/inst.isle line 802. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type FPUOp3: defined at src/isa/s390x/inst.isle line 798. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum FPUOp3 { MAdd32, MAdd64, @@ -772,8 +772,8 @@ pub enum FPUOp3 { MSub64, } -/// Internal type FpuToIntOp: defined at src/isa/s390x/inst.isle line 811. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type FpuToIntOp: defined at src/isa/s390x/inst.isle line 807. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum FpuToIntOp { F32ToU32, F32ToI32, @@ -785,8 +785,8 @@ pub enum FpuToIntOp { F64ToI64, } -/// Internal type IntToFpuOp: defined at src/isa/s390x/inst.isle line 824. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type IntToFpuOp: defined at src/isa/s390x/inst.isle line 820. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum IntToFpuOp { U32ToF32, I32ToF32, @@ -798,8 +798,8 @@ pub enum IntToFpuOp { I64ToF64, } -/// Internal type FpuRoundMode: defined at src/isa/s390x/inst.isle line 838. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type FpuRoundMode: defined at src/isa/s390x/inst.isle line 834. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum FpuRoundMode { Minus32, Minus64, @@ -811,19 +811,19 @@ pub enum FpuRoundMode { Nearest64, } -/// Internal type WritableRegPair: defined at src/isa/s390x/inst.isle line 1248. +/// Internal type WritableRegPair: defined at src/isa/s390x/inst.isle line 1240. #[derive(Clone, Debug)] pub enum WritableRegPair { WritableRegPair { hi: WritableReg, lo: WritableReg }, } -/// Internal type RegPair: defined at src/isa/s390x/inst.isle line 1270. +/// Internal type RegPair: defined at src/isa/s390x/inst.isle line 1262. #[derive(Clone, Debug)] pub enum RegPair { RegPair { hi: Reg, lo: Reg }, } -/// Internal type ProducesBool: defined at src/isa/s390x/inst.isle line 2195. +/// Internal type ProducesBool: defined at src/isa/s390x/inst.isle line 2187. #[derive(Clone, Debug)] pub enum ProducesBool { ProducesBool { producer: ProducesFlags, cond: Cond }, @@ -832,7 +832,7 @@ pub enum ProducesBool { // Generated as internal constructor for term temp_reg. pub fn constructor_temp_reg(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; - // Rule at src/prelude.isle line 70. + // Rule at src/prelude.isle line 73. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); return Some(expr1_0); @@ -841,7 +841,7 @@ pub fn constructor_temp_reg(ctx: &mut C, arg0: Type) -> Option // Generated as internal constructor for term lo_reg. pub fn constructor_lo_reg(ctx: &mut C, arg0: Value) -> Option { let pattern0_0 = arg0; - // Rule at src/prelude.isle line 105. + // Rule at src/prelude.isle line 108. let expr0_0 = C::put_in_regs(ctx, pattern0_0); let expr1_0: usize = 0; let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); @@ -858,7 +858,7 @@ pub fn constructor_value_regs_none( inst: ref pattern1_0, } = pattern0_0 { - // Rule at src/prelude.isle line 300. + // Rule at src/prelude.isle line 312. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::value_regs_invalid(ctx); return Some(expr1_0); @@ -876,7 +876,7 @@ pub fn constructor_safepoint( inst: ref pattern1_0, } = pattern0_0 { - // Rule at src/prelude.isle line 306. + // Rule at src/prelude.isle line 318. let expr0_0 = C::emit_safepoint(ctx, &pattern1_0); let expr1_0 = C::value_regs_invalid(ctx); return Some(expr1_0); @@ -902,7 +902,7 @@ pub fn constructor_with_flags( result: pattern3_1, } = pattern2_0 { - // Rule at src/prelude.isle line 327. + // Rule at src/prelude.isle line 339. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern3_0); let expr2_0 = C::value_regs(ctx, pattern1_1, pattern3_1); @@ -930,7 +930,7 @@ pub fn constructor_with_flags_1( result: pattern3_1, } = pattern2_0 { - // Rule at src/prelude.isle line 335. + // Rule at src/prelude.isle line 347. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern3_0); return Some(pattern3_1); @@ -964,7 +964,7 @@ pub fn constructor_with_flags_2( result: pattern5_1, } = pattern4_0 { - // Rule at src/prelude.isle line 345. + // Rule at src/prelude.isle line 357. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern5_0); let expr2_0 = C::emit(ctx, &pattern3_0); @@ -981,7 +981,7 @@ pub fn constructor_mask_amt_reg(ctx: &mut C, arg0: Type, arg1: Reg) let pattern0_0 = arg0; if let Some(pattern1_0) = C::gpr32_ty(ctx, pattern0_0) { let pattern2_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1013. + // Rule at src/isa/s390x/inst.isle line 1009. let expr0_0: i64 = -1; let expr1_0 = C::mask_amt_imm(ctx, pattern1_0, expr0_0); let expr2_0 = C::u8_as_u16(ctx, expr1_0); @@ -992,7 +992,7 @@ pub fn constructor_mask_amt_reg(ctx: &mut C, arg0: Type, arg1: Reg) } if let Some(pattern1_0) = C::gpr64_ty(ctx, pattern0_0) { let pattern2_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1016. + // Rule at src/isa/s390x/inst.isle line 1012. return Some(pattern2_0); } return None; @@ -1019,7 +1019,7 @@ pub fn constructor_lower_address( let pattern7_0 = arg2; let pattern8_0 = C::i64_from_offset(ctx, pattern7_0); if pattern8_0 == 0 { - // Rule at src/isa/s390x/inst.isle line 1112. + // Rule at src/isa/s390x/inst.isle line 1104. let expr0_0 = C::put_in_reg(ctx, pattern6_0); let expr1_0 = C::put_in_reg(ctx, pattern6_1); let expr2_0 = C::memarg_reg_plus_reg(ctx, expr0_0, expr1_0, pattern0_0); @@ -1038,7 +1038,7 @@ pub fn constructor_lower_address( if let Some(pattern8_0) = C::memarg_symbol_offset_sum(ctx, pattern6_0, pattern7_0) { - // Rule at src/isa/s390x/inst.isle line 1115. + // Rule at src/isa/s390x/inst.isle line 1107. let expr0_0 = C::memarg_symbol(ctx, pattern3_0, pattern8_0, pattern0_0); return Some(expr0_0); } @@ -1048,7 +1048,7 @@ pub fn constructor_lower_address( } let pattern2_0 = arg2; let pattern3_0 = C::i64_from_offset(ctx, pattern2_0); - // Rule at src/isa/s390x/inst.isle line 1109. + // Rule at src/isa/s390x/inst.isle line 1101. let expr0_0 = C::put_in_reg(ctx, pattern1_0); let expr1_0 = C::memarg_reg_plus_off(ctx, expr0_0, pattern3_0, pattern0_0); return Some(expr1_0); @@ -1064,7 +1064,7 @@ pub fn constructor_stack_addr_impl( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1142. + // Rule at src/isa/s390x/inst.isle line 1134. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::abi_stackslot_addr(ctx, expr0_0, pattern1_0, pattern2_0); let expr2_0 = C::emit(ctx, &expr1_0); @@ -1084,7 +1084,7 @@ pub fn constructor_sink_load(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C) -> Option { - // Rule at src/isa/s390x/inst.isle line 1253. + // Rule at src/isa/s390x/inst.isle line 1245. let expr0_0: u8 = 0; let expr1_0 = C::writable_gpr(ctx, expr0_0); let expr2_0: u8 = 1; @@ -1197,7 +1197,7 @@ pub fn constructor_copy_writable_regpair( arg0: &RegPair, ) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/s390x/inst.isle line 1259. + // Rule at src/isa/s390x/inst.isle line 1251. let expr0_0 = constructor_temp_writable_regpair(ctx)?; return Some(expr0_0); } @@ -1213,7 +1213,7 @@ pub fn constructor_writable_regpair_hi( lo: pattern1_1, } = pattern0_0 { - // Rule at src/isa/s390x/inst.isle line 1263. + // Rule at src/isa/s390x/inst.isle line 1255. return Some(pattern1_0); } return None; @@ -1230,7 +1230,7 @@ pub fn constructor_writable_regpair_lo( lo: pattern1_1, } = pattern0_0 { - // Rule at src/isa/s390x/inst.isle line 1267. + // Rule at src/isa/s390x/inst.isle line 1259. return Some(pattern1_1); } return None; @@ -1247,7 +1247,7 @@ pub fn constructor_writable_regpair_to_regpair( lo: pattern1_1, } = pattern0_0 { - // Rule at src/isa/s390x/inst.isle line 1274. + // Rule at src/isa/s390x/inst.isle line 1266. let expr0_0 = C::writable_reg_to_reg(ctx, pattern1_0); let expr1_0 = C::writable_reg_to_reg(ctx, pattern1_1); let expr2_0 = RegPair::RegPair { @@ -1261,7 +1261,7 @@ pub fn constructor_writable_regpair_to_regpair( // Generated as internal constructor for term uninitialized_regpair. pub fn constructor_uninitialized_regpair(ctx: &mut C) -> Option { - // Rule at src/isa/s390x/inst.isle line 1279. + // Rule at src/isa/s390x/inst.isle line 1271. let expr0_0 = constructor_temp_writable_regpair(ctx)?; let expr1_0 = constructor_writable_regpair_to_regpair(ctx, &expr0_0)?; return Some(expr1_0); @@ -1275,7 +1275,7 @@ pub fn constructor_regpair_hi(ctx: &mut C, arg0: &RegPair) -> Option lo: pattern1_1, } = pattern0_0 { - // Rule at src/isa/s390x/inst.isle line 1284. + // Rule at src/isa/s390x/inst.isle line 1276. return Some(pattern1_0); } return None; @@ -1289,7 +1289,7 @@ pub fn constructor_regpair_lo(ctx: &mut C, arg0: &RegPair) -> Option lo: pattern1_1, } = pattern0_0 { - // Rule at src/isa/s390x/inst.isle line 1288. + // Rule at src/isa/s390x/inst.isle line 1280. return Some(pattern1_1); } return None; @@ -1307,7 +1307,7 @@ pub fn constructor_alu_rrr( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 1295. + // Rule at src/isa/s390x/inst.isle line 1287. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::AluRRR { alu_op: pattern1_0.clone(), @@ -1332,7 +1332,7 @@ pub fn constructor_alu_rrsimm16( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 1302. + // Rule at src/isa/s390x/inst.isle line 1294. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::AluRRSImm16 { alu_op: pattern1_0.clone(), @@ -1357,7 +1357,7 @@ pub fn constructor_alu_rr( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 1309. + // Rule at src/isa/s390x/inst.isle line 1301. let expr0_0 = constructor_copy_writable_reg(ctx, pattern0_0, pattern2_0)?; let expr1_0 = MInst::AluRR { alu_op: pattern1_0.clone(), @@ -1381,7 +1381,7 @@ pub fn constructor_alu_rx( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 1316. + // Rule at src/isa/s390x/inst.isle line 1308. let expr0_0 = constructor_copy_writable_reg(ctx, pattern0_0, pattern2_0)?; let expr1_0 = MInst::AluRX { alu_op: pattern1_0.clone(), @@ -1405,7 +1405,7 @@ pub fn constructor_alu_rsimm16( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 1323. + // Rule at src/isa/s390x/inst.isle line 1315. let expr0_0 = constructor_copy_writable_reg(ctx, pattern0_0, pattern2_0)?; let expr1_0 = MInst::AluRSImm16 { alu_op: pattern1_0.clone(), @@ -1429,7 +1429,7 @@ pub fn constructor_alu_rsimm32( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 1330. + // Rule at src/isa/s390x/inst.isle line 1322. let expr0_0 = constructor_copy_writable_reg(ctx, pattern0_0, pattern2_0)?; let expr1_0 = MInst::AluRSImm32 { alu_op: pattern1_0.clone(), @@ -1453,7 +1453,7 @@ pub fn constructor_alu_ruimm32( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 1337. + // Rule at src/isa/s390x/inst.isle line 1329. let expr0_0 = constructor_copy_writable_reg(ctx, pattern0_0, pattern2_0)?; let expr1_0 = MInst::AluRUImm32 { alu_op: pattern1_0.clone(), @@ -1477,7 +1477,7 @@ pub fn constructor_alu_ruimm16shifted( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 1344. + // Rule at src/isa/s390x/inst.isle line 1336. let expr0_0 = constructor_copy_writable_reg(ctx, pattern0_0, pattern2_0)?; let expr1_0 = MInst::AluRUImm16Shifted { alu_op: pattern1_0.clone(), @@ -1501,7 +1501,7 @@ pub fn constructor_alu_ruimm32shifted( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 1351. + // Rule at src/isa/s390x/inst.isle line 1343. let expr0_0 = constructor_copy_writable_reg(ctx, pattern0_0, pattern2_0)?; let expr1_0 = MInst::AluRUImm32Shifted { alu_op: pattern1_0.clone(), @@ -1517,7 +1517,7 @@ pub fn constructor_alu_ruimm32shifted( pub fn constructor_smul_wide(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1358. + // Rule at src/isa/s390x/inst.isle line 1350. let expr0_0 = constructor_temp_writable_regpair(ctx)?; let expr1_0 = MInst::SMulWide { rn: pattern0_0, @@ -1532,7 +1532,7 @@ pub fn constructor_smul_wide(ctx: &mut C, arg0: Reg, arg1: Reg) -> O pub fn constructor_umul_wide(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1365. + // Rule at src/isa/s390x/inst.isle line 1357. let expr0_0 = constructor_temp_writable_regpair(ctx)?; let expr1_0 = constructor_writable_regpair_lo(ctx, &expr0_0)?; let expr2_0 = MInst::Mov64 { @@ -1554,7 +1554,7 @@ pub fn constructor_sdivmod32( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1373. + // Rule at src/isa/s390x/inst.isle line 1365. let expr0_0 = constructor_copy_writable_regpair(ctx, pattern0_0)?; let expr1_0 = MInst::SDivMod32 { rn: pattern1_0 }; let expr2_0 = C::emit(ctx, &expr1_0); @@ -1570,7 +1570,7 @@ pub fn constructor_sdivmod64( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1380. + // Rule at src/isa/s390x/inst.isle line 1372. let expr0_0 = constructor_copy_writable_regpair(ctx, pattern0_0)?; let expr1_0 = MInst::SDivMod64 { rn: pattern1_0 }; let expr2_0 = C::emit(ctx, &expr1_0); @@ -1586,7 +1586,7 @@ pub fn constructor_udivmod32( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1387. + // Rule at src/isa/s390x/inst.isle line 1379. let expr0_0 = constructor_copy_writable_regpair(ctx, pattern0_0)?; let expr1_0 = MInst::UDivMod32 { rn: pattern1_0 }; let expr2_0 = C::emit(ctx, &expr1_0); @@ -1602,7 +1602,7 @@ pub fn constructor_udivmod64( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1394. + // Rule at src/isa/s390x/inst.isle line 1386. let expr0_0 = constructor_copy_writable_regpair(ctx, pattern0_0)?; let expr1_0 = MInst::UDivMod64 { rn: pattern1_0 }; let expr2_0 = C::emit(ctx, &expr1_0); @@ -1624,7 +1624,7 @@ pub fn constructor_shift_rr( let pattern2_0 = arg2; let pattern3_0 = arg3; let pattern4_0 = arg4; - // Rule at src/isa/s390x/inst.isle line 1401. + // Rule at src/isa/s390x/inst.isle line 1393. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::ShiftRR { shift_op: pattern1_0.clone(), @@ -1648,7 +1648,7 @@ pub fn constructor_unary_rr( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1408. + // Rule at src/isa/s390x/inst.isle line 1400. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::UnaryRR { op: pattern1_0.clone(), @@ -1670,7 +1670,7 @@ pub fn constructor_cmp_rr( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1415. + // Rule at src/isa/s390x/inst.isle line 1407. let expr0_0 = MInst::CmpRR { op: pattern0_0.clone(), rn: pattern1_0, @@ -1694,7 +1694,7 @@ pub fn constructor_cmp_rx( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1421. + // Rule at src/isa/s390x/inst.isle line 1413. let expr0_0 = MInst::CmpRX { op: pattern0_0.clone(), rn: pattern1_0, @@ -1718,7 +1718,7 @@ pub fn constructor_cmp_rsimm16( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1427. + // Rule at src/isa/s390x/inst.isle line 1419. let expr0_0 = MInst::CmpRSImm16 { op: pattern0_0.clone(), rn: pattern1_0, @@ -1742,7 +1742,7 @@ pub fn constructor_cmp_rsimm32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1433. + // Rule at src/isa/s390x/inst.isle line 1425. let expr0_0 = MInst::CmpRSImm32 { op: pattern0_0.clone(), rn: pattern1_0, @@ -1766,7 +1766,7 @@ pub fn constructor_cmp_ruimm32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1439. + // Rule at src/isa/s390x/inst.isle line 1431. let expr0_0 = MInst::CmpRUImm32 { op: pattern0_0.clone(), rn: pattern1_0, @@ -1792,7 +1792,7 @@ pub fn constructor_atomic_rmw_impl( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 1445. + // Rule at src/isa/s390x/inst.isle line 1437. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::AtomicRmw { alu_op: pattern1_0.clone(), @@ -1815,7 +1815,7 @@ pub fn constructor_atomic_cas32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1452. + // Rule at src/isa/s390x/inst.isle line 1444. let expr0_0: Type = I32; let expr1_0 = constructor_copy_writable_reg(ctx, expr0_0, pattern0_0)?; let expr2_0 = MInst::AtomicCas32 { @@ -1838,7 +1838,7 @@ pub fn constructor_atomic_cas64( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1459. + // Rule at src/isa/s390x/inst.isle line 1451. let expr0_0: Type = I64; let expr1_0 = constructor_copy_writable_reg(ctx, expr0_0, pattern0_0)?; let expr2_0 = MInst::AtomicCas64 { @@ -1853,7 +1853,7 @@ pub fn constructor_atomic_cas64( // Generated as internal constructor for term fence_impl. pub fn constructor_fence_impl(ctx: &mut C) -> Option { - // Rule at src/isa/s390x/inst.isle line 1466. + // Rule at src/isa/s390x/inst.isle line 1458. let expr0_0 = MInst::Fence; let expr1_0 = SideEffectNoResult::Inst { inst: expr0_0 }; return Some(expr1_0); @@ -1862,7 +1862,7 @@ pub fn constructor_fence_impl(ctx: &mut C) -> Option(ctx: &mut C, arg0: &MemArg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/s390x/inst.isle line 1471. + // Rule at src/isa/s390x/inst.isle line 1463. let expr0_0: Type = I32; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::Load32 { @@ -1877,7 +1877,7 @@ pub fn constructor_load32(ctx: &mut C, arg0: &MemArg) -> Option // Generated as internal constructor for term load64. pub fn constructor_load64(ctx: &mut C, arg0: &MemArg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/s390x/inst.isle line 1478. + // Rule at src/isa/s390x/inst.isle line 1470. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::Load64 { @@ -1892,7 +1892,7 @@ pub fn constructor_load64(ctx: &mut C, arg0: &MemArg) -> Option // Generated as internal constructor for term loadrev16. pub fn constructor_loadrev16(ctx: &mut C, arg0: &MemArg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/s390x/inst.isle line 1485. + // Rule at src/isa/s390x/inst.isle line 1477. let expr0_0: Type = I32; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::LoadRev16 { @@ -1907,7 +1907,7 @@ pub fn constructor_loadrev16(ctx: &mut C, arg0: &MemArg) -> Option(ctx: &mut C, arg0: &MemArg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/s390x/inst.isle line 1492. + // Rule at src/isa/s390x/inst.isle line 1484. let expr0_0: Type = I32; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::LoadRev32 { @@ -1922,7 +1922,7 @@ pub fn constructor_loadrev32(ctx: &mut C, arg0: &MemArg) -> Option(ctx: &mut C, arg0: &MemArg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/s390x/inst.isle line 1499. + // Rule at src/isa/s390x/inst.isle line 1491. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::LoadRev64 { @@ -1942,7 +1942,7 @@ pub fn constructor_store8( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1506. + // Rule at src/isa/s390x/inst.isle line 1498. let expr0_0 = MInst::Store8 { rd: pattern0_0, mem: pattern1_0.clone(), @@ -1959,7 +1959,7 @@ pub fn constructor_store16( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1511. + // Rule at src/isa/s390x/inst.isle line 1503. let expr0_0 = MInst::Store16 { rd: pattern0_0, mem: pattern1_0.clone(), @@ -1976,7 +1976,7 @@ pub fn constructor_store32( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1516. + // Rule at src/isa/s390x/inst.isle line 1508. let expr0_0 = MInst::Store32 { rd: pattern0_0, mem: pattern1_0.clone(), @@ -1993,7 +1993,7 @@ pub fn constructor_store64( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1521. + // Rule at src/isa/s390x/inst.isle line 1513. let expr0_0 = MInst::Store64 { rd: pattern0_0, mem: pattern1_0.clone(), @@ -2010,7 +2010,7 @@ pub fn constructor_store8_imm( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1526. + // Rule at src/isa/s390x/inst.isle line 1518. let expr0_0 = MInst::StoreImm8 { imm: pattern0_0, mem: pattern1_0.clone(), @@ -2027,7 +2027,7 @@ pub fn constructor_store16_imm( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1531. + // Rule at src/isa/s390x/inst.isle line 1523. let expr0_0 = MInst::StoreImm16 { imm: pattern0_0, mem: pattern1_0.clone(), @@ -2044,7 +2044,7 @@ pub fn constructor_store32_simm16( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1536. + // Rule at src/isa/s390x/inst.isle line 1528. let expr0_0 = MInst::StoreImm32SExt16 { imm: pattern0_0, mem: pattern1_0.clone(), @@ -2061,7 +2061,7 @@ pub fn constructor_store64_simm16( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1541. + // Rule at src/isa/s390x/inst.isle line 1533. let expr0_0 = MInst::StoreImm64SExt16 { imm: pattern0_0, mem: pattern1_0.clone(), @@ -2078,7 +2078,7 @@ pub fn constructor_storerev16( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1546. + // Rule at src/isa/s390x/inst.isle line 1538. let expr0_0 = MInst::StoreRev16 { rd: pattern0_0, mem: pattern1_0.clone(), @@ -2095,7 +2095,7 @@ pub fn constructor_storerev32( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1551. + // Rule at src/isa/s390x/inst.isle line 1543. let expr0_0 = MInst::StoreRev32 { rd: pattern0_0, mem: pattern1_0.clone(), @@ -2112,7 +2112,7 @@ pub fn constructor_storerev64( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1556. + // Rule at src/isa/s390x/inst.isle line 1548. let expr0_0 = MInst::StoreRev64 { rd: pattern0_0, mem: pattern1_0.clone(), @@ -2131,7 +2131,7 @@ pub fn constructor_fpu_rr( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1561. + // Rule at src/isa/s390x/inst.isle line 1553. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::FpuRR { fpu_op: pattern1_0.clone(), @@ -2155,7 +2155,7 @@ pub fn constructor_fpu_rrr( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 1568. + // Rule at src/isa/s390x/inst.isle line 1560. let expr0_0 = constructor_copy_writable_reg(ctx, pattern0_0, pattern2_0)?; let expr1_0 = MInst::FpuRRR { fpu_op: pattern1_0.clone(), @@ -2181,7 +2181,7 @@ pub fn constructor_fpu_rrrr( let pattern2_0 = arg2; let pattern3_0 = arg3; let pattern4_0 = arg4; - // Rule at src/isa/s390x/inst.isle line 1575. + // Rule at src/isa/s390x/inst.isle line 1567. let expr0_0 = constructor_copy_writable_reg(ctx, pattern0_0, pattern2_0)?; let expr1_0 = MInst::FpuRRRR { fpu_op: pattern1_0.clone(), @@ -2204,7 +2204,7 @@ pub fn constructor_fpu_copysign( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1582. + // Rule at src/isa/s390x/inst.isle line 1574. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::FpuCopysign { rd: expr0_0, @@ -2224,7 +2224,7 @@ pub fn constructor_fpu_cmp32( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1589. + // Rule at src/isa/s390x/inst.isle line 1581. let expr0_0 = MInst::FpuCmp32 { rn: pattern0_0, rm: pattern1_0, @@ -2245,7 +2245,7 @@ pub fn constructor_fpu_cmp64( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1595. + // Rule at src/isa/s390x/inst.isle line 1587. let expr0_0 = MInst::FpuCmp64 { rn: pattern0_0, rm: pattern1_0, @@ -2268,7 +2268,7 @@ pub fn constructor_fpu_to_int( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1601. + // Rule at src/isa/s390x/inst.isle line 1593. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::FpuToInt { op: pattern1_0.clone(), @@ -2293,7 +2293,7 @@ pub fn constructor_int_to_fpu( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1608. + // Rule at src/isa/s390x/inst.isle line 1600. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::IntToFpu { op: pattern1_0.clone(), @@ -2315,7 +2315,7 @@ pub fn constructor_fpu_round( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1615. + // Rule at src/isa/s390x/inst.isle line 1607. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::FpuRound { op: pattern1_0.clone(), @@ -2339,7 +2339,7 @@ pub fn constructor_fpuvec_rrr( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 1622. + // Rule at src/isa/s390x/inst.isle line 1614. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::FpuVecRRR { fpu_op: pattern1_0.clone(), @@ -2355,7 +2355,7 @@ pub fn constructor_fpuvec_rrr( // Generated as internal constructor for term mov_to_fpr. pub fn constructor_mov_to_fpr(ctx: &mut C, arg0: Reg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/s390x/inst.isle line 1629. + // Rule at src/isa/s390x/inst.isle line 1621. let expr0_0: Type = F64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::MovToFpr { @@ -2370,7 +2370,7 @@ pub fn constructor_mov_to_fpr(ctx: &mut C, arg0: Reg) -> Option // Generated as internal constructor for term mov_from_fpr. pub fn constructor_mov_from_fpr(ctx: &mut C, arg0: Reg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/s390x/inst.isle line 1636. + // Rule at src/isa/s390x/inst.isle line 1628. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::MovFromFpr { @@ -2385,7 +2385,7 @@ pub fn constructor_mov_from_fpr(ctx: &mut C, arg0: Reg) -> Option(ctx: &mut C, arg0: &MemArg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/s390x/inst.isle line 1643. + // Rule at src/isa/s390x/inst.isle line 1635. let expr0_0: Type = F32; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::FpuLoad32 { @@ -2400,7 +2400,7 @@ pub fn constructor_fpu_load32(ctx: &mut C, arg0: &MemArg) -> Option< // Generated as internal constructor for term fpu_load64. pub fn constructor_fpu_load64(ctx: &mut C, arg0: &MemArg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/s390x/inst.isle line 1650. + // Rule at src/isa/s390x/inst.isle line 1642. let expr0_0: Type = F64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::FpuLoad64 { @@ -2415,7 +2415,7 @@ pub fn constructor_fpu_load64(ctx: &mut C, arg0: &MemArg) -> Option< // Generated as internal constructor for term fpu_loadrev32. pub fn constructor_fpu_loadrev32(ctx: &mut C, arg0: &MemArg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/s390x/inst.isle line 1657. + // Rule at src/isa/s390x/inst.isle line 1649. let expr0_0: Type = F32; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::FpuLoadRev32 { @@ -2430,7 +2430,7 @@ pub fn constructor_fpu_loadrev32(ctx: &mut C, arg0: &MemArg) -> Opti // Generated as internal constructor for term fpu_loadrev64. pub fn constructor_fpu_loadrev64(ctx: &mut C, arg0: &MemArg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/s390x/inst.isle line 1664. + // Rule at src/isa/s390x/inst.isle line 1656. let expr0_0: Type = F64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::FpuLoadRev64 { @@ -2450,7 +2450,7 @@ pub fn constructor_fpu_store32( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1671. + // Rule at src/isa/s390x/inst.isle line 1663. let expr0_0 = MInst::FpuStore32 { rd: pattern0_0, mem: pattern1_0.clone(), @@ -2467,7 +2467,7 @@ pub fn constructor_fpu_store64( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1676. + // Rule at src/isa/s390x/inst.isle line 1668. let expr0_0 = MInst::FpuStore64 { rd: pattern0_0, mem: pattern1_0.clone(), @@ -2484,7 +2484,7 @@ pub fn constructor_fpu_storerev32( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1681. + // Rule at src/isa/s390x/inst.isle line 1673. let expr0_0 = MInst::FpuStoreRev32 { rd: pattern0_0, mem: pattern1_0.clone(), @@ -2501,7 +2501,7 @@ pub fn constructor_fpu_storerev64( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1686. + // Rule at src/isa/s390x/inst.isle line 1678. let expr0_0 = MInst::FpuStoreRev64 { rd: pattern0_0, mem: pattern1_0.clone(), @@ -2518,7 +2518,7 @@ pub fn constructor_load_ext_name_far( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1691. + // Rule at src/isa/s390x/inst.isle line 1683. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::LoadExtNameFar { @@ -2534,7 +2534,7 @@ pub fn constructor_load_ext_name_far( // Generated as internal constructor for term load_addr. pub fn constructor_load_addr(ctx: &mut C, arg0: &MemArg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/s390x/inst.isle line 1698. + // Rule at src/isa/s390x/inst.isle line 1690. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::LoadAddr { @@ -2552,7 +2552,7 @@ pub fn constructor_jump_impl( arg0: MachLabel, ) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/s390x/inst.isle line 1705. + // Rule at src/isa/s390x/inst.isle line 1697. let expr0_0 = MInst::Jump { dest: pattern0_0 }; let expr1_0 = SideEffectNoResult::Inst { inst: expr0_0 }; return Some(expr1_0); @@ -2568,7 +2568,7 @@ pub fn constructor_cond_br( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1710. + // Rule at src/isa/s390x/inst.isle line 1702. let expr0_0 = MInst::CondBr { taken: pattern0_0, not_taken: pattern1_0, @@ -2586,7 +2586,7 @@ pub fn constructor_oneway_cond_br( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1715. + // Rule at src/isa/s390x/inst.isle line 1707. let expr0_0 = MInst::OneWayCondBr { target: pattern0_0, cond: pattern1_0.clone(), @@ -2603,7 +2603,7 @@ pub fn constructor_jt_sequence( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1720. + // Rule at src/isa/s390x/inst.isle line 1712. let expr0_0 = MInst::JTSequence { ridx: pattern0_0, targets: pattern1_0.clone(), @@ -2620,7 +2620,7 @@ pub fn constructor_drop_flags(ctx: &mut C, arg0: &ProducesFlags) -> result: pattern1_1, } = pattern0_0 { - // Rule at src/isa/s390x/inst.isle line 1725. + // Rule at src/isa/s390x/inst.isle line 1717. let expr0_0 = C::emit(ctx, &pattern1_0); return Some(pattern1_1); } @@ -2638,7 +2638,7 @@ pub fn constructor_emit_mov( if pattern0_0 == F32 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1741. + // Rule at src/isa/s390x/inst.isle line 1733. let expr0_0 = MInst::FpuMove32 { rd: pattern2_0, rn: pattern3_0, @@ -2649,7 +2649,7 @@ pub fn constructor_emit_mov( if pattern0_0 == F64 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1744. + // Rule at src/isa/s390x/inst.isle line 1736. let expr0_0 = MInst::FpuMove64 { rd: pattern2_0, rn: pattern3_0, @@ -2660,7 +2660,7 @@ pub fn constructor_emit_mov( if let Some(pattern1_0) = C::gpr32_ty(ctx, pattern0_0) { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1735. + // Rule at src/isa/s390x/inst.isle line 1727. let expr0_0 = MInst::Mov32 { rd: pattern2_0, rm: pattern3_0, @@ -2671,7 +2671,7 @@ pub fn constructor_emit_mov( if let Some(pattern1_0) = C::gpr64_ty(ctx, pattern0_0) { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1738. + // Rule at src/isa/s390x/inst.isle line 1730. let expr0_0 = MInst::Mov64 { rd: pattern2_0, rm: pattern3_0, @@ -2690,7 +2690,7 @@ pub fn constructor_copy_writable_reg( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1749. + // Rule at src/isa/s390x/inst.isle line 1741. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = constructor_emit_mov(ctx, pattern0_0, expr0_0, pattern1_0)?; return Some(expr0_0); @@ -2700,7 +2700,7 @@ pub fn constructor_copy_writable_reg( pub fn constructor_copy_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1756. + // Rule at src/isa/s390x/inst.isle line 1748. let expr0_0 = constructor_copy_writable_reg(ctx, pattern0_0, pattern1_0)?; let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); return Some(expr1_0); @@ -2717,7 +2717,7 @@ pub fn constructor_emit_imm( if pattern0_0 == F32 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1811. + // Rule at src/isa/s390x/inst.isle line 1803. let expr0_0 = C::u64_as_u32(ctx, pattern3_0); let expr1_0 = MInst::LoadFpuConst32 { rd: pattern2_0, @@ -2729,7 +2729,7 @@ pub fn constructor_emit_imm( if pattern0_0 == F64 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1816. + // Rule at src/isa/s390x/inst.isle line 1808. let expr0_0 = MInst::LoadFpuConst64 { rd: pattern2_0, const_data: pattern3_0, @@ -2740,7 +2740,7 @@ pub fn constructor_emit_imm( if let Some(pattern1_0) = C::fits_in_16(ctx, pattern0_0) { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1765. + // Rule at src/isa/s390x/inst.isle line 1757. let expr0_0 = C::u64_as_i16(ctx, pattern3_0); let expr1_0 = MInst::Mov32SImm16 { rd: pattern2_0, @@ -2753,7 +2753,7 @@ pub fn constructor_emit_imm( let pattern2_0 = arg1; let pattern3_0 = arg2; if let Some(pattern4_0) = C::i16_from_u64(ctx, pattern3_0) { - // Rule at src/isa/s390x/inst.isle line 1769. + // Rule at src/isa/s390x/inst.isle line 1761. let expr0_0 = MInst::Mov32SImm16 { rd: pattern2_0, imm: pattern4_0, @@ -2761,7 +2761,7 @@ pub fn constructor_emit_imm( let expr1_0 = C::emit(ctx, &expr0_0); return Some(expr1_0); } - // Rule at src/isa/s390x/inst.isle line 1773. + // Rule at src/isa/s390x/inst.isle line 1765. let expr0_0 = C::u64_as_u32(ctx, pattern3_0); let expr1_0 = MInst::Mov32Imm { rd: pattern2_0, @@ -2775,14 +2775,14 @@ pub fn constructor_emit_imm( let pattern3_0 = arg2; if let Some(pattern4_0) = C::u64_nonzero_hipart(ctx, pattern3_0) { if let Some(pattern5_0) = C::u64_nonzero_lopart(ctx, pattern3_0) { - // Rule at src/isa/s390x/inst.isle line 1793. + // Rule at src/isa/s390x/inst.isle line 1785. let expr0_0 = constructor_emit_imm(ctx, pattern1_0, pattern2_0, pattern4_0)?; let expr1_0 = constructor_emit_insert_imm(ctx, pattern2_0, pattern5_0)?; return Some(expr1_0); } } if let Some(pattern4_0) = C::i16_from_u64(ctx, pattern3_0) { - // Rule at src/isa/s390x/inst.isle line 1777. + // Rule at src/isa/s390x/inst.isle line 1769. let expr0_0 = MInst::Mov64SImm16 { rd: pattern2_0, imm: pattern4_0, @@ -2791,7 +2791,7 @@ pub fn constructor_emit_imm( return Some(expr1_0); } if let Some(pattern4_0) = C::i32_from_u64(ctx, pattern3_0) { - // Rule at src/isa/s390x/inst.isle line 1781. + // Rule at src/isa/s390x/inst.isle line 1773. let expr0_0 = MInst::Mov64SImm32 { rd: pattern2_0, imm: pattern4_0, @@ -2800,7 +2800,7 @@ pub fn constructor_emit_imm( return Some(expr1_0); } if let Some(pattern4_0) = C::uimm32shifted_from_u64(ctx, pattern3_0) { - // Rule at src/isa/s390x/inst.isle line 1789. + // Rule at src/isa/s390x/inst.isle line 1781. let expr0_0 = MInst::Mov64UImm32Shifted { rd: pattern2_0, imm: pattern4_0, @@ -2809,7 +2809,7 @@ pub fn constructor_emit_imm( return Some(expr1_0); } if let Some(pattern4_0) = C::uimm16shifted_from_u64(ctx, pattern3_0) { - // Rule at src/isa/s390x/inst.isle line 1785. + // Rule at src/isa/s390x/inst.isle line 1777. let expr0_0 = MInst::Mov64UImm16Shifted { rd: pattern2_0, imm: pattern4_0, @@ -2830,7 +2830,7 @@ pub fn constructor_emit_insert_imm( let pattern0_0 = arg0; let pattern1_0 = arg1; if let Some(pattern2_0) = C::uimm32shifted_from_u64(ctx, pattern1_0) { - // Rule at src/isa/s390x/inst.isle line 1806. + // Rule at src/isa/s390x/inst.isle line 1798. let expr0_0 = MInst::Insert64UImm32Shifted { rd: pattern0_0, imm: pattern2_0, @@ -2839,7 +2839,7 @@ pub fn constructor_emit_insert_imm( return Some(expr1_0); } if let Some(pattern2_0) = C::uimm16shifted_from_u64(ctx, pattern1_0) { - // Rule at src/isa/s390x/inst.isle line 1802. + // Rule at src/isa/s390x/inst.isle line 1794. let expr0_0 = MInst::Insert64UImm16Shifted { rd: pattern0_0, imm: pattern2_0, @@ -2854,7 +2854,7 @@ pub fn constructor_emit_insert_imm( pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1821. + // Rule at src/isa/s390x/inst.isle line 1813. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = constructor_emit_imm(ctx, pattern0_0, expr0_0, pattern1_0)?; let expr2_0 = C::writable_reg_to_reg(ctx, expr0_0); @@ -2871,7 +2871,7 @@ pub fn constructor_imm_regpair_lo( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1829. + // Rule at src/isa/s390x/inst.isle line 1821. let expr0_0 = constructor_copy_writable_regpair(ctx, pattern2_0)?; let expr1_0 = constructor_writable_regpair_lo(ctx, &expr0_0)?; let expr2_0 = constructor_emit_imm(ctx, pattern0_0, expr1_0, pattern1_0)?; @@ -2889,7 +2889,7 @@ pub fn constructor_imm_regpair_hi( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1837. + // Rule at src/isa/s390x/inst.isle line 1829. let expr0_0 = constructor_copy_writable_regpair(ctx, pattern2_0)?; let expr1_0 = constructor_writable_regpair_hi(ctx, &expr0_0)?; let expr2_0 = constructor_emit_imm(ctx, pattern0_0, expr1_0, pattern1_0)?; @@ -2901,22 +2901,22 @@ pub fn constructor_imm_regpair_hi( pub fn constructor_ty_ext32(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I8 { - // Rule at src/isa/s390x/inst.isle line 1847. + // Rule at src/isa/s390x/inst.isle line 1839. let expr0_0: Type = I32; return Some(expr0_0); } if pattern0_0 == I16 { - // Rule at src/isa/s390x/inst.isle line 1848. + // Rule at src/isa/s390x/inst.isle line 1840. let expr0_0: Type = I32; return Some(expr0_0); } if pattern0_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 1849. + // Rule at src/isa/s390x/inst.isle line 1841. let expr0_0: Type = I32; return Some(expr0_0); } if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 1850. + // Rule at src/isa/s390x/inst.isle line 1842. let expr0_0: Type = I64; return Some(expr0_0); } @@ -2927,22 +2927,22 @@ pub fn constructor_ty_ext32(ctx: &mut C, arg0: Type) -> Option pub fn constructor_ty_ext64(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I8 { - // Rule at src/isa/s390x/inst.isle line 1854. + // Rule at src/isa/s390x/inst.isle line 1846. let expr0_0: Type = I64; return Some(expr0_0); } if pattern0_0 == I16 { - // Rule at src/isa/s390x/inst.isle line 1855. + // Rule at src/isa/s390x/inst.isle line 1847. let expr0_0: Type = I64; return Some(expr0_0); } if pattern0_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 1856. + // Rule at src/isa/s390x/inst.isle line 1848. let expr0_0: Type = I64; return Some(expr0_0); } if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 1857. + // Rule at src/isa/s390x/inst.isle line 1849. let expr0_0: Type = I64; return Some(expr0_0); } @@ -2959,7 +2959,7 @@ pub fn constructor_emit_zext32_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1862. + // Rule at src/isa/s390x/inst.isle line 1854. let expr0_0: bool = false; let expr1_0 = C::ty_bits(ctx, pattern1_0); let expr2_0: u8 = 32; @@ -2984,7 +2984,7 @@ pub fn constructor_emit_sext32_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1868. + // Rule at src/isa/s390x/inst.isle line 1860. let expr0_0: bool = true; let expr1_0 = C::ty_bits(ctx, pattern1_0); let expr2_0: u8 = 32; @@ -3009,7 +3009,7 @@ pub fn constructor_emit_zext64_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1874. + // Rule at src/isa/s390x/inst.isle line 1866. let expr0_0: bool = false; let expr1_0 = C::ty_bits(ctx, pattern1_0); let expr2_0: u8 = 64; @@ -3034,7 +3034,7 @@ pub fn constructor_emit_sext64_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1880. + // Rule at src/isa/s390x/inst.isle line 1872. let expr0_0: bool = true; let expr1_0 = C::ty_bits(ctx, pattern1_0); let expr2_0: u8 = 64; @@ -3053,7 +3053,7 @@ pub fn constructor_emit_sext64_reg( pub fn constructor_zext32_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1886. + // Rule at src/isa/s390x/inst.isle line 1878. let expr0_0: Type = I32; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = constructor_emit_zext32_reg(ctx, expr1_0, pattern0_0, pattern1_0)?; @@ -3065,7 +3065,7 @@ pub fn constructor_zext32_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> pub fn constructor_sext32_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1894. + // Rule at src/isa/s390x/inst.isle line 1886. let expr0_0: Type = I32; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = constructor_emit_sext32_reg(ctx, expr1_0, pattern0_0, pattern1_0)?; @@ -3077,7 +3077,7 @@ pub fn constructor_sext32_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> pub fn constructor_zext64_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1902. + // Rule at src/isa/s390x/inst.isle line 1894. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = constructor_emit_zext64_reg(ctx, expr1_0, pattern0_0, pattern1_0)?; @@ -3089,7 +3089,7 @@ pub fn constructor_zext64_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> pub fn constructor_sext64_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1910. + // Rule at src/isa/s390x/inst.isle line 1902. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = constructor_emit_sext64_reg(ctx, expr1_0, pattern0_0, pattern1_0)?; @@ -3108,7 +3108,7 @@ pub fn constructor_emit_zext32_mem( let pattern1_0 = arg1; if pattern1_0 == I8 { let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1918. + // Rule at src/isa/s390x/inst.isle line 1910. let expr0_0 = MInst::Load32ZExt8 { rd: pattern0_0, mem: pattern3_0.clone(), @@ -3118,7 +3118,7 @@ pub fn constructor_emit_zext32_mem( } if pattern1_0 == I16 { let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1919. + // Rule at src/isa/s390x/inst.isle line 1911. let expr0_0 = MInst::Load32ZExt16 { rd: pattern0_0, mem: pattern3_0.clone(), @@ -3140,7 +3140,7 @@ pub fn constructor_emit_sext32_mem( let pattern1_0 = arg1; if pattern1_0 == I8 { let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1923. + // Rule at src/isa/s390x/inst.isle line 1915. let expr0_0 = MInst::Load32SExt8 { rd: pattern0_0, mem: pattern3_0.clone(), @@ -3150,7 +3150,7 @@ pub fn constructor_emit_sext32_mem( } if pattern1_0 == I16 { let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1924. + // Rule at src/isa/s390x/inst.isle line 1916. let expr0_0 = MInst::Load32SExt16 { rd: pattern0_0, mem: pattern3_0.clone(), @@ -3172,7 +3172,7 @@ pub fn constructor_emit_zext64_mem( let pattern1_0 = arg1; if pattern1_0 == I8 { let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1928. + // Rule at src/isa/s390x/inst.isle line 1920. let expr0_0 = MInst::Load64ZExt8 { rd: pattern0_0, mem: pattern3_0.clone(), @@ -3182,7 +3182,7 @@ pub fn constructor_emit_zext64_mem( } if pattern1_0 == I16 { let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1929. + // Rule at src/isa/s390x/inst.isle line 1921. let expr0_0 = MInst::Load64ZExt16 { rd: pattern0_0, mem: pattern3_0.clone(), @@ -3192,7 +3192,7 @@ pub fn constructor_emit_zext64_mem( } if pattern1_0 == I32 { let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1930. + // Rule at src/isa/s390x/inst.isle line 1922. let expr0_0 = MInst::Load64ZExt32 { rd: pattern0_0, mem: pattern3_0.clone(), @@ -3214,7 +3214,7 @@ pub fn constructor_emit_sext64_mem( let pattern1_0 = arg1; if pattern1_0 == I8 { let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1934. + // Rule at src/isa/s390x/inst.isle line 1926. let expr0_0 = MInst::Load64SExt8 { rd: pattern0_0, mem: pattern3_0.clone(), @@ -3224,7 +3224,7 @@ pub fn constructor_emit_sext64_mem( } if pattern1_0 == I16 { let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1935. + // Rule at src/isa/s390x/inst.isle line 1927. let expr0_0 = MInst::Load64SExt16 { rd: pattern0_0, mem: pattern3_0.clone(), @@ -3234,7 +3234,7 @@ pub fn constructor_emit_sext64_mem( } if pattern1_0 == I32 { let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 1936. + // Rule at src/isa/s390x/inst.isle line 1928. let expr0_0 = MInst::Load64SExt32 { rd: pattern0_0, mem: pattern3_0.clone(), @@ -3249,7 +3249,7 @@ pub fn constructor_emit_sext64_mem( pub fn constructor_zext32_mem(ctx: &mut C, arg0: Type, arg1: &MemArg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1940. + // Rule at src/isa/s390x/inst.isle line 1932. let expr0_0: Type = I32; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = constructor_emit_zext32_mem(ctx, expr1_0, pattern0_0, pattern1_0)?; @@ -3261,7 +3261,7 @@ pub fn constructor_zext32_mem(ctx: &mut C, arg0: Type, arg1: &MemArg pub fn constructor_sext32_mem(ctx: &mut C, arg0: Type, arg1: &MemArg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1947. + // Rule at src/isa/s390x/inst.isle line 1939. let expr0_0: Type = I32; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = constructor_emit_sext32_mem(ctx, expr1_0, pattern0_0, pattern1_0)?; @@ -3273,7 +3273,7 @@ pub fn constructor_sext32_mem(ctx: &mut C, arg0: Type, arg1: &MemArg pub fn constructor_zext64_mem(ctx: &mut C, arg0: Type, arg1: &MemArg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1954. + // Rule at src/isa/s390x/inst.isle line 1946. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = constructor_emit_zext64_mem(ctx, expr1_0, pattern0_0, pattern1_0)?; @@ -3285,7 +3285,7 @@ pub fn constructor_zext64_mem(ctx: &mut C, arg0: Type, arg1: &MemArg pub fn constructor_sext64_mem(ctx: &mut C, arg0: Type, arg1: &MemArg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 1961. + // Rule at src/isa/s390x/inst.isle line 1953. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = constructor_emit_sext64_mem(ctx, expr1_0, pattern0_0, pattern1_0)?; @@ -3303,7 +3303,7 @@ pub fn constructor_emit_put_in_reg_zext32( let pattern1_0 = arg1; let pattern2_0 = C::value_type(ctx, pattern1_0); if let Some(pattern3_0) = C::u64_from_value(ctx, pattern1_0) { - // Rule at src/isa/s390x/inst.isle line 1969. + // Rule at src/isa/s390x/inst.isle line 1961. let expr0_0 = constructor_ty_ext32(ctx, pattern2_0)?; let expr1_0 = constructor_emit_imm(ctx, expr0_0, pattern0_0, pattern3_0)?; return Some(expr1_0); @@ -3320,7 +3320,7 @@ pub fn constructor_emit_put_in_reg_zext32( { if let &Opcode::Load = &pattern6_0 { if let Some(()) = C::bigendian(ctx, pattern6_2) { - // Rule at src/isa/s390x/inst.isle line 1971. + // Rule at src/isa/s390x/inst.isle line 1963. let expr0_0 = constructor_sink_load(ctx, pattern4_0)?; let expr1_0 = constructor_emit_zext32_mem(ctx, pattern0_0, pattern3_0, &expr0_0)?; @@ -3329,13 +3329,13 @@ pub fn constructor_emit_put_in_reg_zext32( } } } - // Rule at src/isa/s390x/inst.isle line 1973. + // Rule at src/isa/s390x/inst.isle line 1965. let expr0_0 = C::put_in_reg(ctx, pattern1_0); let expr1_0 = constructor_emit_zext32_reg(ctx, pattern0_0, pattern3_0, expr0_0)?; return Some(expr1_0); } if let Some(pattern3_0) = C::ty_32_or_64(ctx, pattern2_0) { - // Rule at src/isa/s390x/inst.isle line 1975. + // Rule at src/isa/s390x/inst.isle line 1967. let expr0_0 = C::put_in_reg(ctx, pattern1_0); let expr1_0 = constructor_emit_mov(ctx, pattern3_0, pattern0_0, expr0_0)?; return Some(expr1_0); @@ -3353,7 +3353,7 @@ pub fn constructor_emit_put_in_reg_sext32( let pattern1_0 = arg1; let pattern2_0 = C::value_type(ctx, pattern1_0); if let Some(pattern3_0) = C::u64_from_signed_value(ctx, pattern1_0) { - // Rule at src/isa/s390x/inst.isle line 1980. + // Rule at src/isa/s390x/inst.isle line 1972. let expr0_0 = constructor_ty_ext32(ctx, pattern2_0)?; let expr1_0 = constructor_emit_imm(ctx, expr0_0, pattern0_0, pattern3_0)?; return Some(expr1_0); @@ -3370,7 +3370,7 @@ pub fn constructor_emit_put_in_reg_sext32( { if let &Opcode::Load = &pattern6_0 { if let Some(()) = C::bigendian(ctx, pattern6_2) { - // Rule at src/isa/s390x/inst.isle line 1982. + // Rule at src/isa/s390x/inst.isle line 1974. let expr0_0 = constructor_sink_load(ctx, pattern4_0)?; let expr1_0 = constructor_emit_sext32_mem(ctx, pattern0_0, pattern3_0, &expr0_0)?; @@ -3379,13 +3379,13 @@ pub fn constructor_emit_put_in_reg_sext32( } } } - // Rule at src/isa/s390x/inst.isle line 1984. + // Rule at src/isa/s390x/inst.isle line 1976. let expr0_0 = C::put_in_reg(ctx, pattern1_0); let expr1_0 = constructor_emit_sext32_reg(ctx, pattern0_0, pattern3_0, expr0_0)?; return Some(expr1_0); } if let Some(pattern3_0) = C::ty_32_or_64(ctx, pattern2_0) { - // Rule at src/isa/s390x/inst.isle line 1986. + // Rule at src/isa/s390x/inst.isle line 1978. let expr0_0 = C::put_in_reg(ctx, pattern1_0); let expr1_0 = constructor_emit_mov(ctx, pattern3_0, pattern0_0, expr0_0)?; return Some(expr1_0); @@ -3403,7 +3403,7 @@ pub fn constructor_emit_put_in_reg_zext64( let pattern1_0 = arg1; let pattern2_0 = C::value_type(ctx, pattern1_0); if let Some(pattern3_0) = C::u64_from_value(ctx, pattern1_0) { - // Rule at src/isa/s390x/inst.isle line 1991. + // Rule at src/isa/s390x/inst.isle line 1983. let expr0_0 = constructor_ty_ext64(ctx, pattern2_0)?; let expr1_0 = constructor_emit_imm(ctx, expr0_0, pattern0_0, pattern3_0)?; return Some(expr1_0); @@ -3420,7 +3420,7 @@ pub fn constructor_emit_put_in_reg_zext64( { if let &Opcode::Load = &pattern6_0 { if let Some(()) = C::bigendian(ctx, pattern6_2) { - // Rule at src/isa/s390x/inst.isle line 1993. + // Rule at src/isa/s390x/inst.isle line 1985. let expr0_0 = constructor_sink_load(ctx, pattern4_0)?; let expr1_0 = constructor_emit_zext64_mem(ctx, pattern0_0, pattern3_0, &expr0_0)?; @@ -3429,13 +3429,13 @@ pub fn constructor_emit_put_in_reg_zext64( } } } - // Rule at src/isa/s390x/inst.isle line 1995. + // Rule at src/isa/s390x/inst.isle line 1987. let expr0_0 = C::put_in_reg(ctx, pattern1_0); let expr1_0 = constructor_emit_zext64_reg(ctx, pattern0_0, pattern3_0, expr0_0)?; return Some(expr1_0); } if let Some(pattern3_0) = C::gpr64_ty(ctx, pattern2_0) { - // Rule at src/isa/s390x/inst.isle line 1997. + // Rule at src/isa/s390x/inst.isle line 1989. let expr0_0 = C::put_in_reg(ctx, pattern1_0); let expr1_0 = constructor_emit_mov(ctx, pattern3_0, pattern0_0, expr0_0)?; return Some(expr1_0); @@ -3453,7 +3453,7 @@ pub fn constructor_emit_put_in_reg_sext64( let pattern1_0 = arg1; let pattern2_0 = C::value_type(ctx, pattern1_0); if let Some(pattern3_0) = C::u64_from_signed_value(ctx, pattern1_0) { - // Rule at src/isa/s390x/inst.isle line 2002. + // Rule at src/isa/s390x/inst.isle line 1994. let expr0_0 = constructor_ty_ext64(ctx, pattern2_0)?; let expr1_0 = constructor_emit_imm(ctx, expr0_0, pattern0_0, pattern3_0)?; return Some(expr1_0); @@ -3470,7 +3470,7 @@ pub fn constructor_emit_put_in_reg_sext64( { if let &Opcode::Load = &pattern6_0 { if let Some(()) = C::bigendian(ctx, pattern6_2) { - // Rule at src/isa/s390x/inst.isle line 2004. + // Rule at src/isa/s390x/inst.isle line 1996. let expr0_0 = constructor_sink_load(ctx, pattern4_0)?; let expr1_0 = constructor_emit_sext64_mem(ctx, pattern0_0, pattern3_0, &expr0_0)?; @@ -3479,13 +3479,13 @@ pub fn constructor_emit_put_in_reg_sext64( } } } - // Rule at src/isa/s390x/inst.isle line 2006. + // Rule at src/isa/s390x/inst.isle line 1998. let expr0_0 = C::put_in_reg(ctx, pattern1_0); let expr1_0 = constructor_emit_sext64_reg(ctx, pattern0_0, pattern3_0, expr0_0)?; return Some(expr1_0); } if let Some(pattern3_0) = C::gpr64_ty(ctx, pattern2_0) { - // Rule at src/isa/s390x/inst.isle line 2008. + // Rule at src/isa/s390x/inst.isle line 2000. let expr0_0 = C::put_in_reg(ctx, pattern1_0); let expr1_0 = constructor_emit_mov(ctx, pattern3_0, pattern0_0, expr0_0)?; return Some(expr1_0); @@ -3498,7 +3498,7 @@ pub fn constructor_put_in_reg_zext32(ctx: &mut C, arg0: Value) -> Op let pattern0_0 = arg0; let pattern1_0 = C::value_type(ctx, pattern0_0); if let Some(pattern2_0) = C::u64_from_value(ctx, pattern0_0) { - // Rule at src/isa/s390x/inst.isle line 2013. + // Rule at src/isa/s390x/inst.isle line 2005. let expr0_0 = constructor_ty_ext32(ctx, pattern1_0)?; let expr1_0 = constructor_imm(ctx, expr0_0, pattern2_0)?; return Some(expr1_0); @@ -3515,7 +3515,7 @@ pub fn constructor_put_in_reg_zext32(ctx: &mut C, arg0: Value) -> Op { if let &Opcode::Load = &pattern5_0 { if let Some(()) = C::bigendian(ctx, pattern5_2) { - // Rule at src/isa/s390x/inst.isle line 2015. + // Rule at src/isa/s390x/inst.isle line 2007. let expr0_0 = constructor_sink_load(ctx, pattern3_0)?; let expr1_0 = constructor_zext32_mem(ctx, pattern2_0, &expr0_0)?; return Some(expr1_0); @@ -3523,13 +3523,13 @@ pub fn constructor_put_in_reg_zext32(ctx: &mut C, arg0: Value) -> Op } } } - // Rule at src/isa/s390x/inst.isle line 2017. + // Rule at src/isa/s390x/inst.isle line 2009. let expr0_0 = C::put_in_reg(ctx, pattern0_0); let expr1_0 = constructor_zext32_reg(ctx, pattern2_0, expr0_0)?; return Some(expr1_0); } if let Some(pattern2_0) = C::ty_32_or_64(ctx, pattern1_0) { - // Rule at src/isa/s390x/inst.isle line 2019. + // Rule at src/isa/s390x/inst.isle line 2011. let expr0_0 = C::put_in_reg(ctx, pattern0_0); return Some(expr0_0); } @@ -3541,7 +3541,7 @@ pub fn constructor_put_in_reg_sext32(ctx: &mut C, arg0: Value) -> Op let pattern0_0 = arg0; let pattern1_0 = C::value_type(ctx, pattern0_0); if let Some(pattern2_0) = C::u64_from_signed_value(ctx, pattern0_0) { - // Rule at src/isa/s390x/inst.isle line 2024. + // Rule at src/isa/s390x/inst.isle line 2016. let expr0_0 = constructor_ty_ext32(ctx, pattern1_0)?; let expr1_0 = constructor_imm(ctx, expr0_0, pattern2_0)?; return Some(expr1_0); @@ -3558,7 +3558,7 @@ pub fn constructor_put_in_reg_sext32(ctx: &mut C, arg0: Value) -> Op { if let &Opcode::Load = &pattern5_0 { if let Some(()) = C::bigendian(ctx, pattern5_2) { - // Rule at src/isa/s390x/inst.isle line 2026. + // Rule at src/isa/s390x/inst.isle line 2018. let expr0_0 = constructor_sink_load(ctx, pattern3_0)?; let expr1_0 = constructor_sext32_mem(ctx, pattern2_0, &expr0_0)?; return Some(expr1_0); @@ -3566,13 +3566,13 @@ pub fn constructor_put_in_reg_sext32(ctx: &mut C, arg0: Value) -> Op } } } - // Rule at src/isa/s390x/inst.isle line 2028. + // Rule at src/isa/s390x/inst.isle line 2020. let expr0_0 = C::put_in_reg(ctx, pattern0_0); let expr1_0 = constructor_sext32_reg(ctx, pattern2_0, expr0_0)?; return Some(expr1_0); } if let Some(pattern2_0) = C::ty_32_or_64(ctx, pattern1_0) { - // Rule at src/isa/s390x/inst.isle line 2030. + // Rule at src/isa/s390x/inst.isle line 2022. let expr0_0 = C::put_in_reg(ctx, pattern0_0); return Some(expr0_0); } @@ -3584,7 +3584,7 @@ pub fn constructor_put_in_reg_zext64(ctx: &mut C, arg0: Value) -> Op let pattern0_0 = arg0; let pattern1_0 = C::value_type(ctx, pattern0_0); if let Some(pattern2_0) = C::u64_from_value(ctx, pattern0_0) { - // Rule at src/isa/s390x/inst.isle line 2035. + // Rule at src/isa/s390x/inst.isle line 2027. let expr0_0 = constructor_ty_ext64(ctx, pattern1_0)?; let expr1_0 = constructor_imm(ctx, expr0_0, pattern2_0)?; return Some(expr1_0); @@ -3601,7 +3601,7 @@ pub fn constructor_put_in_reg_zext64(ctx: &mut C, arg0: Value) -> Op { if let &Opcode::Load = &pattern5_0 { if let Some(()) = C::bigendian(ctx, pattern5_2) { - // Rule at src/isa/s390x/inst.isle line 2037. + // Rule at src/isa/s390x/inst.isle line 2029. let expr0_0 = constructor_sink_load(ctx, pattern3_0)?; let expr1_0 = constructor_zext64_mem(ctx, pattern2_0, &expr0_0)?; return Some(expr1_0); @@ -3609,13 +3609,13 @@ pub fn constructor_put_in_reg_zext64(ctx: &mut C, arg0: Value) -> Op } } } - // Rule at src/isa/s390x/inst.isle line 2039. + // Rule at src/isa/s390x/inst.isle line 2031. let expr0_0 = C::put_in_reg(ctx, pattern0_0); let expr1_0 = constructor_zext64_reg(ctx, pattern2_0, expr0_0)?; return Some(expr1_0); } if let Some(pattern2_0) = C::gpr64_ty(ctx, pattern1_0) { - // Rule at src/isa/s390x/inst.isle line 2041. + // Rule at src/isa/s390x/inst.isle line 2033. let expr0_0 = C::put_in_reg(ctx, pattern0_0); return Some(expr0_0); } @@ -3627,7 +3627,7 @@ pub fn constructor_put_in_reg_sext64(ctx: &mut C, arg0: Value) -> Op let pattern0_0 = arg0; let pattern1_0 = C::value_type(ctx, pattern0_0); if let Some(pattern2_0) = C::u64_from_signed_value(ctx, pattern0_0) { - // Rule at src/isa/s390x/inst.isle line 2046. + // Rule at src/isa/s390x/inst.isle line 2038. let expr0_0 = constructor_ty_ext64(ctx, pattern1_0)?; let expr1_0 = constructor_imm(ctx, expr0_0, pattern2_0)?; return Some(expr1_0); @@ -3644,7 +3644,7 @@ pub fn constructor_put_in_reg_sext64(ctx: &mut C, arg0: Value) -> Op { if let &Opcode::Load = &pattern5_0 { if let Some(()) = C::bigendian(ctx, pattern5_2) { - // Rule at src/isa/s390x/inst.isle line 2048. + // Rule at src/isa/s390x/inst.isle line 2040. let expr0_0 = constructor_sink_load(ctx, pattern3_0)?; let expr1_0 = constructor_sext64_mem(ctx, pattern2_0, &expr0_0)?; return Some(expr1_0); @@ -3652,13 +3652,13 @@ pub fn constructor_put_in_reg_sext64(ctx: &mut C, arg0: Value) -> Op } } } - // Rule at src/isa/s390x/inst.isle line 2050. + // Rule at src/isa/s390x/inst.isle line 2042. let expr0_0 = C::put_in_reg(ctx, pattern0_0); let expr1_0 = constructor_sext64_reg(ctx, pattern2_0, expr0_0)?; return Some(expr1_0); } if let Some(pattern2_0) = C::gpr64_ty(ctx, pattern1_0) { - // Rule at src/isa/s390x/inst.isle line 2052. + // Rule at src/isa/s390x/inst.isle line 2044. let expr0_0 = C::put_in_reg(ctx, pattern0_0); return Some(expr0_0); } @@ -3673,7 +3673,7 @@ pub fn constructor_put_in_regpair_lo_zext32( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2058. + // Rule at src/isa/s390x/inst.isle line 2050. let expr0_0 = constructor_copy_writable_regpair(ctx, pattern1_0)?; let expr1_0 = constructor_writable_regpair_lo(ctx, &expr0_0)?; let expr2_0 = constructor_emit_put_in_reg_zext32(ctx, expr1_0, pattern0_0)?; @@ -3689,7 +3689,7 @@ pub fn constructor_put_in_regpair_lo_sext32( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2066. + // Rule at src/isa/s390x/inst.isle line 2058. let expr0_0 = constructor_copy_writable_regpair(ctx, pattern1_0)?; let expr1_0 = constructor_writable_regpair_lo(ctx, &expr0_0)?; let expr2_0 = constructor_emit_put_in_reg_sext32(ctx, expr1_0, pattern0_0)?; @@ -3705,7 +3705,7 @@ pub fn constructor_put_in_regpair_lo_zext64( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2074. + // Rule at src/isa/s390x/inst.isle line 2066. let expr0_0 = constructor_copy_writable_regpair(ctx, pattern1_0)?; let expr1_0 = constructor_writable_regpair_lo(ctx, &expr0_0)?; let expr2_0 = constructor_emit_put_in_reg_zext64(ctx, expr1_0, pattern0_0)?; @@ -3721,7 +3721,7 @@ pub fn constructor_put_in_regpair_lo_sext64( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2082. + // Rule at src/isa/s390x/inst.isle line 2074. let expr0_0 = constructor_copy_writable_regpair(ctx, pattern1_0)?; let expr1_0 = constructor_writable_regpair_lo(ctx, &expr0_0)?; let expr2_0 = constructor_emit_put_in_reg_sext64(ctx, expr1_0, pattern0_0)?; @@ -3742,7 +3742,7 @@ pub fn constructor_emit_cmov_imm( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 2092. + // Rule at src/isa/s390x/inst.isle line 2084. let expr0_0 = MInst::CMov32SImm16 { rd: pattern2_0, cond: pattern3_0.clone(), @@ -3759,7 +3759,7 @@ pub fn constructor_emit_cmov_imm( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 2095. + // Rule at src/isa/s390x/inst.isle line 2087. let expr0_0 = MInst::CMov64SImm16 { rd: pattern2_0, cond: pattern3_0.clone(), @@ -3787,7 +3787,7 @@ pub fn constructor_cmov_imm( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 2101. + // Rule at src/isa/s390x/inst.isle line 2093. let expr0_0 = constructor_copy_writable_reg(ctx, pattern0_0, pattern3_0)?; let expr1_0 = constructor_emit_cmov_imm(ctx, pattern0_0, expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -3807,7 +3807,7 @@ pub fn constructor_cmov_imm_regpair_lo( let pattern2_0 = arg2; let pattern3_0 = arg3; let pattern4_0 = arg4; - // Rule at src/isa/s390x/inst.isle line 2108. + // Rule at src/isa/s390x/inst.isle line 2100. let expr0_0 = constructor_copy_writable_regpair(ctx, pattern4_0)?; let expr1_0 = constructor_writable_regpair_lo(ctx, &expr0_0)?; let expr2_0 = constructor_emit_cmov_imm(ctx, pattern0_0, expr1_0, pattern2_0, pattern3_0)?; @@ -3830,7 +3830,7 @@ pub fn constructor_cmov_imm_regpair_hi( let pattern2_0 = arg2; let pattern3_0 = arg3; let pattern4_0 = arg4; - // Rule at src/isa/s390x/inst.isle line 2117. + // Rule at src/isa/s390x/inst.isle line 2109. let expr0_0 = constructor_copy_writable_regpair(ctx, pattern4_0)?; let expr1_0 = constructor_writable_regpair_hi(ctx, &expr0_0)?; let expr2_0 = constructor_emit_cmov_imm(ctx, pattern0_0, expr1_0, pattern2_0, pattern3_0)?; @@ -3852,7 +3852,7 @@ pub fn constructor_emit_cmov_reg( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 2131. + // Rule at src/isa/s390x/inst.isle line 2123. let expr0_0 = MInst::FpuCMov32 { rd: pattern2_0, cond: pattern3_0.clone(), @@ -3869,7 +3869,7 @@ pub fn constructor_emit_cmov_reg( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 2134. + // Rule at src/isa/s390x/inst.isle line 2126. let expr0_0 = MInst::FpuCMov64 { rd: pattern2_0, cond: pattern3_0.clone(), @@ -3886,7 +3886,7 @@ pub fn constructor_emit_cmov_reg( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 2125. + // Rule at src/isa/s390x/inst.isle line 2117. let expr0_0 = MInst::CMov32 { rd: pattern2_0, cond: pattern3_0.clone(), @@ -3903,7 +3903,7 @@ pub fn constructor_emit_cmov_reg( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 2128. + // Rule at src/isa/s390x/inst.isle line 2120. let expr0_0 = MInst::CMov64 { rd: pattern2_0, cond: pattern3_0.clone(), @@ -3931,7 +3931,7 @@ pub fn constructor_cmov_reg( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 2140. + // Rule at src/isa/s390x/inst.isle line 2132. let expr0_0 = constructor_copy_writable_reg(ctx, pattern0_0, pattern3_0)?; let expr1_0 = constructor_emit_cmov_reg(ctx, pattern0_0, expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -3952,7 +3952,7 @@ pub fn constructor_trap_if( { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2148. + // Rule at src/isa/s390x/inst.isle line 2140. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = MInst::TrapIf { cond: pattern2_0.clone(), @@ -3978,7 +3978,7 @@ pub fn constructor_icmps_reg_and_trap( let pattern2_0 = arg2; let pattern3_0 = arg3; let pattern4_0 = arg4; - // Rule at src/isa/s390x/inst.isle line 2154. + // Rule at src/isa/s390x/inst.isle line 2146. let expr0_0 = constructor_cmpop_cmps(ctx, pattern0_0)?; let expr1_0 = MInst::CmpTrapRR { op: expr0_0, @@ -4006,7 +4006,7 @@ pub fn constructor_icmps_simm16_and_trap( let pattern2_0 = arg2; let pattern3_0 = arg3; let pattern4_0 = arg4; - // Rule at src/isa/s390x/inst.isle line 2160. + // Rule at src/isa/s390x/inst.isle line 2152. let expr0_0 = constructor_cmpop_cmps(ctx, pattern0_0)?; let expr1_0 = MInst::CmpTrapRSImm16 { op: expr0_0, @@ -4034,7 +4034,7 @@ pub fn constructor_icmpu_reg_and_trap( let pattern2_0 = arg2; let pattern3_0 = arg3; let pattern4_0 = arg4; - // Rule at src/isa/s390x/inst.isle line 2166. + // Rule at src/isa/s390x/inst.isle line 2158. let expr0_0 = constructor_cmpop_cmpu(ctx, pattern0_0)?; let expr1_0 = MInst::CmpTrapRR { op: expr0_0, @@ -4062,7 +4062,7 @@ pub fn constructor_icmpu_uimm16_and_trap( let pattern2_0 = arg2; let pattern3_0 = arg3; let pattern4_0 = arg4; - // Rule at src/isa/s390x/inst.isle line 2172. + // Rule at src/isa/s390x/inst.isle line 2164. let expr0_0 = constructor_cmpop_cmpu(ctx, pattern0_0)?; let expr1_0 = MInst::CmpTrapRUImm16 { op: expr0_0, @@ -4082,7 +4082,7 @@ pub fn constructor_trap_impl( arg0: &TrapCode, ) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/s390x/inst.isle line 2178. + // Rule at src/isa/s390x/inst.isle line 2170. let expr0_0 = MInst::Trap { trap_code: pattern0_0.clone(), }; @@ -4098,7 +4098,7 @@ pub fn constructor_trap_if_impl( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2182. + // Rule at src/isa/s390x/inst.isle line 2174. let expr0_0 = MInst::TrapIf { cond: pattern0_0.clone(), trap_code: pattern1_0.clone(), @@ -4109,7 +4109,7 @@ pub fn constructor_trap_if_impl( // Generated as internal constructor for term debugtrap_impl. pub fn constructor_debugtrap_impl(ctx: &mut C) -> Option { - // Rule at src/isa/s390x/inst.isle line 2186. + // Rule at src/isa/s390x/inst.isle line 2178. let expr0_0 = MInst::Debugtrap; let expr1_0 = SideEffectNoResult::Inst { inst: expr0_0 }; return Some(expr1_0); @@ -4123,7 +4123,7 @@ pub fn constructor_bool( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2197. + // Rule at src/isa/s390x/inst.isle line 2189. let expr0_0 = ProducesBool::ProducesBool { producer: pattern0_0.clone(), cond: pattern1_0.clone(), @@ -4142,7 +4142,7 @@ pub fn constructor_invert_bool( cond: ref pattern1_1, } = pattern0_0 { - // Rule at src/isa/s390x/inst.isle line 2201. + // Rule at src/isa/s390x/inst.isle line 2193. let expr0_0 = C::invert_cond(ctx, &pattern1_1); let expr1_0 = constructor_bool(ctx, &pattern1_0, &expr0_0)?; return Some(expr1_0); @@ -4158,7 +4158,7 @@ pub fn constructor_emit_producer(ctx: &mut C, arg0: &ProducesFlags) result: pattern1_1, } = pattern0_0 { - // Rule at src/isa/s390x/inst.isle line 2210. + // Rule at src/isa/s390x/inst.isle line 2202. let expr0_0 = C::emit(ctx, &pattern1_0); return Some(expr0_0); } @@ -4173,7 +4173,7 @@ pub fn constructor_emit_consumer(ctx: &mut C, arg0: &ConsumesFlags) result: pattern1_1, } = pattern0_0 { - // Rule at src/isa/s390x/inst.isle line 2212. + // Rule at src/isa/s390x/inst.isle line 2204. let expr0_0 = C::emit(ctx, &pattern1_0); return Some(expr0_0); } @@ -4197,7 +4197,7 @@ pub fn constructor_select_bool_reg( { let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 2216. + // Rule at src/isa/s390x/inst.isle line 2208. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = constructor_emit_producer(ctx, &pattern2_0)?; let expr2_0 = constructor_emit_mov(ctx, pattern0_0, expr0_0, pattern4_0)?; @@ -4226,7 +4226,7 @@ pub fn constructor_select_bool_imm( { let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 2225. + // Rule at src/isa/s390x/inst.isle line 2217. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = constructor_emit_producer(ctx, &pattern2_0)?; let expr2_0 = constructor_emit_imm(ctx, pattern0_0, expr0_0, pattern4_0)?; @@ -4247,7 +4247,7 @@ pub fn constructor_lower_bool( let pattern0_0 = arg0; if pattern0_0 == B1 { let pattern2_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2235. + // Rule at src/isa/s390x/inst.isle line 2227. let expr0_0: Type = B1; let expr1_0: i16 = 1; let expr2_0: u64 = 0; @@ -4256,7 +4256,7 @@ pub fn constructor_lower_bool( } if pattern0_0 == B8 { let pattern2_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2236. + // Rule at src/isa/s390x/inst.isle line 2228. let expr0_0: Type = B8; let expr1_0: i16 = -1; let expr2_0: u64 = 0; @@ -4265,7 +4265,7 @@ pub fn constructor_lower_bool( } if pattern0_0 == B16 { let pattern2_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2237. + // Rule at src/isa/s390x/inst.isle line 2229. let expr0_0: Type = B16; let expr1_0: i16 = -1; let expr2_0: u64 = 0; @@ -4274,7 +4274,7 @@ pub fn constructor_lower_bool( } if pattern0_0 == B32 { let pattern2_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2238. + // Rule at src/isa/s390x/inst.isle line 2230. let expr0_0: Type = B32; let expr1_0: i16 = -1; let expr2_0: u64 = 0; @@ -4283,7 +4283,7 @@ pub fn constructor_lower_bool( } if pattern0_0 == B64 { let pattern2_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2239. + // Rule at src/isa/s390x/inst.isle line 2231. let expr0_0: Type = B64; let expr1_0: i16 = -1; let expr2_0: u64 = 0; @@ -4308,7 +4308,7 @@ pub fn constructor_cond_br_bool( { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2243. + // Rule at src/isa/s390x/inst.isle line 2235. let expr0_0 = constructor_emit_producer(ctx, &pattern1_0)?; let expr1_0 = constructor_cond_br(ctx, pattern2_0, pattern3_0, &pattern1_1)?; return Some(expr1_0); @@ -4329,7 +4329,7 @@ pub fn constructor_oneway_cond_br_bool( } = pattern0_0 { let pattern2_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2249. + // Rule at src/isa/s390x/inst.isle line 2241. let expr0_0 = constructor_emit_producer(ctx, &pattern1_0)?; let expr1_0 = constructor_oneway_cond_br(ctx, pattern2_0, &pattern1_1)?; return Some(expr1_0); @@ -4350,7 +4350,7 @@ pub fn constructor_trap_if_bool( } = pattern0_0 { let pattern2_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2255. + // Rule at src/isa/s390x/inst.isle line 2247. let expr0_0 = constructor_emit_producer(ctx, &pattern1_0)?; let expr1_0 = constructor_trap_if_impl(ctx, &pattern1_1, pattern2_0)?; return Some(expr1_0); @@ -4363,7 +4363,7 @@ pub fn constructor_clz_reg(ctx: &mut C, arg0: i16, arg1: Reg) -> Opt let pattern0_0 = arg0; if pattern0_0 == 64 { let pattern2_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2266. + // Rule at src/isa/s390x/inst.isle line 2258. let expr0_0 = constructor_temp_writable_regpair(ctx)?; let expr1_0 = MInst::Flogr { rn: pattern2_0 }; let expr2_0 = C::emit(ctx, &expr1_0); @@ -4371,7 +4371,7 @@ pub fn constructor_clz_reg(ctx: &mut C, arg0: i16, arg1: Reg) -> Opt return Some(expr3_0); } let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2275. + // Rule at src/isa/s390x/inst.isle line 2267. let expr0_0 = constructor_temp_writable_regpair(ctx)?; let expr1_0 = MInst::Flogr { rn: pattern1_0 }; let expr2_0 = C::emit(ctx, &expr1_0); @@ -4392,22 +4392,22 @@ pub fn constructor_clz_reg(ctx: &mut C, arg0: i16, arg1: Reg) -> Opt pub fn constructor_aluop_add(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I8 { - // Rule at src/isa/s390x/inst.isle line 2286. + // Rule at src/isa/s390x/inst.isle line 2278. let expr0_0 = ALUOp::Add32; return Some(expr0_0); } if pattern0_0 == I16 { - // Rule at src/isa/s390x/inst.isle line 2287. + // Rule at src/isa/s390x/inst.isle line 2279. let expr0_0 = ALUOp::Add32; return Some(expr0_0); } if pattern0_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 2288. + // Rule at src/isa/s390x/inst.isle line 2280. let expr0_0 = ALUOp::Add32; return Some(expr0_0); } if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2289. + // Rule at src/isa/s390x/inst.isle line 2281. let expr0_0 = ALUOp::Add64; return Some(expr0_0); } @@ -4418,17 +4418,17 @@ pub fn constructor_aluop_add(ctx: &mut C, arg0: Type) -> Option(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I16 { - // Rule at src/isa/s390x/inst.isle line 2292. + // Rule at src/isa/s390x/inst.isle line 2284. let expr0_0 = ALUOp::Add32Ext16; return Some(expr0_0); } if pattern0_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 2293. + // Rule at src/isa/s390x/inst.isle line 2285. let expr0_0 = ALUOp::Add32Ext16; return Some(expr0_0); } if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2294. + // Rule at src/isa/s390x/inst.isle line 2286. let expr0_0 = ALUOp::Add64Ext16; return Some(expr0_0); } @@ -4439,7 +4439,7 @@ pub fn constructor_aluop_add_sext16(ctx: &mut C, arg0: Type) -> Opti pub fn constructor_aluop_add_sext32(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2297. + // Rule at src/isa/s390x/inst.isle line 2289. let expr0_0 = ALUOp::Add64Ext32; return Some(expr0_0); } @@ -4456,7 +4456,7 @@ pub fn constructor_add_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2300. + // Rule at src/isa/s390x/inst.isle line 2292. let expr0_0 = constructor_aluop_add(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rrr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4472,7 +4472,7 @@ pub fn constructor_add_reg_sext32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2303. + // Rule at src/isa/s390x/inst.isle line 2295. let expr0_0 = constructor_aluop_add_sext32(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4488,7 +4488,7 @@ pub fn constructor_add_simm16( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2306. + // Rule at src/isa/s390x/inst.isle line 2298. let expr0_0 = constructor_aluop_add(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rrsimm16(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4504,7 +4504,7 @@ pub fn constructor_add_simm32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2309. + // Rule at src/isa/s390x/inst.isle line 2301. let expr0_0 = constructor_aluop_add(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rsimm32(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4520,7 +4520,7 @@ pub fn constructor_add_mem( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2312. + // Rule at src/isa/s390x/inst.isle line 2304. let expr0_0 = constructor_aluop_add(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rx(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4536,7 +4536,7 @@ pub fn constructor_add_mem_sext16( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2315. + // Rule at src/isa/s390x/inst.isle line 2307. let expr0_0 = constructor_aluop_add_sext16(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rx(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4552,7 +4552,7 @@ pub fn constructor_add_mem_sext32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2318. + // Rule at src/isa/s390x/inst.isle line 2310. let expr0_0 = constructor_aluop_add_sext32(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rx(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4562,12 +4562,12 @@ pub fn constructor_add_mem_sext32( pub fn constructor_aluop_add_logical(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 2324. + // Rule at src/isa/s390x/inst.isle line 2316. let expr0_0 = ALUOp::AddLogical32; return Some(expr0_0); } if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2325. + // Rule at src/isa/s390x/inst.isle line 2317. let expr0_0 = ALUOp::AddLogical64; return Some(expr0_0); } @@ -4578,7 +4578,7 @@ pub fn constructor_aluop_add_logical(ctx: &mut C, arg0: Type) -> Opt pub fn constructor_aluop_add_logical_zext32(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2328. + // Rule at src/isa/s390x/inst.isle line 2320. let expr0_0 = ALUOp::AddLogical64Ext32; return Some(expr0_0); } @@ -4595,7 +4595,7 @@ pub fn constructor_add_logical_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2331. + // Rule at src/isa/s390x/inst.isle line 2323. let expr0_0 = constructor_aluop_add_logical(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rrr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4611,7 +4611,7 @@ pub fn constructor_add_logical_reg_zext32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2334. + // Rule at src/isa/s390x/inst.isle line 2326. let expr0_0 = constructor_aluop_add_logical_zext32(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4627,7 +4627,7 @@ pub fn constructor_add_logical_zimm32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2337. + // Rule at src/isa/s390x/inst.isle line 2329. let expr0_0 = constructor_aluop_add_logical(ctx, pattern0_0)?; let expr1_0 = constructor_alu_ruimm32(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4643,7 +4643,7 @@ pub fn constructor_add_logical_mem( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2340. + // Rule at src/isa/s390x/inst.isle line 2332. let expr0_0 = constructor_aluop_add_logical(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rx(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4659,7 +4659,7 @@ pub fn constructor_add_logical_mem_zext32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2343. + // Rule at src/isa/s390x/inst.isle line 2335. let expr0_0 = constructor_aluop_add_logical_zext32(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rx(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4669,22 +4669,22 @@ pub fn constructor_add_logical_mem_zext32( pub fn constructor_aluop_sub(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I8 { - // Rule at src/isa/s390x/inst.isle line 2349. + // Rule at src/isa/s390x/inst.isle line 2341. let expr0_0 = ALUOp::Sub32; return Some(expr0_0); } if pattern0_0 == I16 { - // Rule at src/isa/s390x/inst.isle line 2350. + // Rule at src/isa/s390x/inst.isle line 2342. let expr0_0 = ALUOp::Sub32; return Some(expr0_0); } if pattern0_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 2351. + // Rule at src/isa/s390x/inst.isle line 2343. let expr0_0 = ALUOp::Sub32; return Some(expr0_0); } if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2352. + // Rule at src/isa/s390x/inst.isle line 2344. let expr0_0 = ALUOp::Sub64; return Some(expr0_0); } @@ -4695,17 +4695,17 @@ pub fn constructor_aluop_sub(ctx: &mut C, arg0: Type) -> Option(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I16 { - // Rule at src/isa/s390x/inst.isle line 2355. + // Rule at src/isa/s390x/inst.isle line 2347. let expr0_0 = ALUOp::Sub32Ext16; return Some(expr0_0); } if pattern0_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 2356. + // Rule at src/isa/s390x/inst.isle line 2348. let expr0_0 = ALUOp::Sub32Ext16; return Some(expr0_0); } if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2357. + // Rule at src/isa/s390x/inst.isle line 2349. let expr0_0 = ALUOp::Sub64Ext16; return Some(expr0_0); } @@ -4716,7 +4716,7 @@ pub fn constructor_aluop_sub_sext16(ctx: &mut C, arg0: Type) -> Opti pub fn constructor_aluop_sub_sext32(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2360. + // Rule at src/isa/s390x/inst.isle line 2352. let expr0_0 = ALUOp::Sub64Ext32; return Some(expr0_0); } @@ -4733,7 +4733,7 @@ pub fn constructor_sub_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2363. + // Rule at src/isa/s390x/inst.isle line 2355. let expr0_0 = constructor_aluop_sub(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rrr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4749,7 +4749,7 @@ pub fn constructor_sub_reg_sext32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2366. + // Rule at src/isa/s390x/inst.isle line 2358. let expr0_0 = constructor_aluop_sub_sext32(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4765,7 +4765,7 @@ pub fn constructor_sub_mem( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2369. + // Rule at src/isa/s390x/inst.isle line 2361. let expr0_0 = constructor_aluop_sub(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rx(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4781,7 +4781,7 @@ pub fn constructor_sub_mem_sext16( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2372. + // Rule at src/isa/s390x/inst.isle line 2364. let expr0_0 = constructor_aluop_sub_sext16(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rx(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4797,7 +4797,7 @@ pub fn constructor_sub_mem_sext32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2375. + // Rule at src/isa/s390x/inst.isle line 2367. let expr0_0 = constructor_aluop_sub_sext32(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rx(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4807,12 +4807,12 @@ pub fn constructor_sub_mem_sext32( pub fn constructor_aluop_sub_logical(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 2381. + // Rule at src/isa/s390x/inst.isle line 2373. let expr0_0 = ALUOp::SubLogical32; return Some(expr0_0); } if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2382. + // Rule at src/isa/s390x/inst.isle line 2374. let expr0_0 = ALUOp::SubLogical64; return Some(expr0_0); } @@ -4823,7 +4823,7 @@ pub fn constructor_aluop_sub_logical(ctx: &mut C, arg0: Type) -> Opt pub fn constructor_aluop_sub_logical_zext32(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2385. + // Rule at src/isa/s390x/inst.isle line 2377. let expr0_0 = ALUOp::SubLogical64Ext32; return Some(expr0_0); } @@ -4840,7 +4840,7 @@ pub fn constructor_sub_logical_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2388. + // Rule at src/isa/s390x/inst.isle line 2380. let expr0_0 = constructor_aluop_sub_logical(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rrr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4856,7 +4856,7 @@ pub fn constructor_sub_logical_reg_zext32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2391. + // Rule at src/isa/s390x/inst.isle line 2383. let expr0_0 = constructor_aluop_sub_logical_zext32(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4872,7 +4872,7 @@ pub fn constructor_sub_logical_zimm32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2394. + // Rule at src/isa/s390x/inst.isle line 2386. let expr0_0 = constructor_aluop_sub_logical(ctx, pattern0_0)?; let expr1_0 = constructor_alu_ruimm32(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4888,7 +4888,7 @@ pub fn constructor_sub_logical_mem( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2397. + // Rule at src/isa/s390x/inst.isle line 2389. let expr0_0 = constructor_aluop_sub_logical(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rx(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4904,7 +4904,7 @@ pub fn constructor_sub_logical_mem_zext32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2400. + // Rule at src/isa/s390x/inst.isle line 2392. let expr0_0 = constructor_aluop_sub_logical(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rx(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4914,22 +4914,22 @@ pub fn constructor_sub_logical_mem_zext32( pub fn constructor_aluop_mul(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I8 { - // Rule at src/isa/s390x/inst.isle line 2406. + // Rule at src/isa/s390x/inst.isle line 2398. let expr0_0 = ALUOp::Mul32; return Some(expr0_0); } if pattern0_0 == I16 { - // Rule at src/isa/s390x/inst.isle line 2407. + // Rule at src/isa/s390x/inst.isle line 2399. let expr0_0 = ALUOp::Mul32; return Some(expr0_0); } if pattern0_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 2408. + // Rule at src/isa/s390x/inst.isle line 2400. let expr0_0 = ALUOp::Mul32; return Some(expr0_0); } if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2409. + // Rule at src/isa/s390x/inst.isle line 2401. let expr0_0 = ALUOp::Mul64; return Some(expr0_0); } @@ -4940,17 +4940,17 @@ pub fn constructor_aluop_mul(ctx: &mut C, arg0: Type) -> Option(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I16 { - // Rule at src/isa/s390x/inst.isle line 2412. + // Rule at src/isa/s390x/inst.isle line 2404. let expr0_0 = ALUOp::Mul32Ext16; return Some(expr0_0); } if pattern0_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 2413. + // Rule at src/isa/s390x/inst.isle line 2405. let expr0_0 = ALUOp::Mul32Ext16; return Some(expr0_0); } if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2414. + // Rule at src/isa/s390x/inst.isle line 2406. let expr0_0 = ALUOp::Mul64Ext16; return Some(expr0_0); } @@ -4961,7 +4961,7 @@ pub fn constructor_aluop_mul_sext16(ctx: &mut C, arg0: Type) -> Opti pub fn constructor_aluop_mul_sext32(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2417. + // Rule at src/isa/s390x/inst.isle line 2409. let expr0_0 = ALUOp::Mul64Ext32; return Some(expr0_0); } @@ -4978,7 +4978,7 @@ pub fn constructor_mul_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2420. + // Rule at src/isa/s390x/inst.isle line 2412. let expr0_0 = constructor_aluop_mul(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rrr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4994,7 +4994,7 @@ pub fn constructor_mul_reg_sext32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2423. + // Rule at src/isa/s390x/inst.isle line 2415. let expr0_0 = constructor_aluop_mul_sext32(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -5010,7 +5010,7 @@ pub fn constructor_mul_simm16( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2426. + // Rule at src/isa/s390x/inst.isle line 2418. let expr0_0 = constructor_aluop_mul(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rsimm16(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -5026,7 +5026,7 @@ pub fn constructor_mul_simm32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2429. + // Rule at src/isa/s390x/inst.isle line 2421. let expr0_0 = constructor_aluop_mul(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rsimm32(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -5042,7 +5042,7 @@ pub fn constructor_mul_mem( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2432. + // Rule at src/isa/s390x/inst.isle line 2424. let expr0_0 = constructor_aluop_mul(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rx(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -5058,7 +5058,7 @@ pub fn constructor_mul_mem_sext16( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2435. + // Rule at src/isa/s390x/inst.isle line 2427. let expr0_0 = constructor_aluop_mul_sext16(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rx(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -5074,7 +5074,7 @@ pub fn constructor_mul_mem_sext32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2438. + // Rule at src/isa/s390x/inst.isle line 2430. let expr0_0 = constructor_aluop_mul_sext32(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rx(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -5091,14 +5091,14 @@ pub fn constructor_udivmod( if pattern0_0 == I32 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2444. + // Rule at src/isa/s390x/inst.isle line 2436. let expr0_0 = constructor_udivmod32(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if pattern0_0 == I64 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2445. + // Rule at src/isa/s390x/inst.isle line 2437. let expr0_0 = constructor_udivmod64(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } @@ -5116,14 +5116,14 @@ pub fn constructor_sdivmod( if pattern0_0 == I32 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2451. + // Rule at src/isa/s390x/inst.isle line 2443. let expr0_0 = constructor_sdivmod32(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if pattern0_0 == I64 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2452. + // Rule at src/isa/s390x/inst.isle line 2444. let expr0_0 = constructor_sdivmod64(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } @@ -5134,12 +5134,12 @@ pub fn constructor_sdivmod( pub fn constructor_aluop_and(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if let Some(pattern1_0) = C::gpr32_ty(ctx, pattern0_0) { - // Rule at src/isa/s390x/inst.isle line 2458. + // Rule at src/isa/s390x/inst.isle line 2450. let expr0_0 = ALUOp::And32; return Some(expr0_0); } if let Some(pattern1_0) = C::gpr64_ty(ctx, pattern0_0) { - // Rule at src/isa/s390x/inst.isle line 2459. + // Rule at src/isa/s390x/inst.isle line 2451. let expr0_0 = ALUOp::And64; return Some(expr0_0); } @@ -5156,7 +5156,7 @@ pub fn constructor_and_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2462. + // Rule at src/isa/s390x/inst.isle line 2454. let expr0_0 = constructor_aluop_and(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rrr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -5172,7 +5172,7 @@ pub fn constructor_and_uimm16shifted( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2465. + // Rule at src/isa/s390x/inst.isle line 2457. let expr0_0 = constructor_aluop_and(ctx, pattern0_0)?; let expr1_0 = constructor_alu_ruimm16shifted(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; @@ -5189,7 +5189,7 @@ pub fn constructor_and_uimm32shifted( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2468. + // Rule at src/isa/s390x/inst.isle line 2460. let expr0_0 = constructor_aluop_and(ctx, pattern0_0)?; let expr1_0 = constructor_alu_ruimm32shifted(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; @@ -5206,7 +5206,7 @@ pub fn constructor_and_mem( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2471. + // Rule at src/isa/s390x/inst.isle line 2463. let expr0_0 = constructor_aluop_and(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rx(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -5216,12 +5216,12 @@ pub fn constructor_and_mem( pub fn constructor_aluop_or(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if let Some(pattern1_0) = C::gpr32_ty(ctx, pattern0_0) { - // Rule at src/isa/s390x/inst.isle line 2477. + // Rule at src/isa/s390x/inst.isle line 2469. let expr0_0 = ALUOp::Orr32; return Some(expr0_0); } if let Some(pattern1_0) = C::gpr64_ty(ctx, pattern0_0) { - // Rule at src/isa/s390x/inst.isle line 2478. + // Rule at src/isa/s390x/inst.isle line 2470. let expr0_0 = ALUOp::Orr64; return Some(expr0_0); } @@ -5238,7 +5238,7 @@ pub fn constructor_or_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2481. + // Rule at src/isa/s390x/inst.isle line 2473. let expr0_0 = constructor_aluop_or(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rrr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -5254,7 +5254,7 @@ pub fn constructor_or_uimm16shifted( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2484. + // Rule at src/isa/s390x/inst.isle line 2476. let expr0_0 = constructor_aluop_or(ctx, pattern0_0)?; let expr1_0 = constructor_alu_ruimm16shifted(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; @@ -5271,7 +5271,7 @@ pub fn constructor_or_uimm32shifted( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2487. + // Rule at src/isa/s390x/inst.isle line 2479. let expr0_0 = constructor_aluop_or(ctx, pattern0_0)?; let expr1_0 = constructor_alu_ruimm32shifted(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; @@ -5288,7 +5288,7 @@ pub fn constructor_or_mem( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2490. + // Rule at src/isa/s390x/inst.isle line 2482. let expr0_0 = constructor_aluop_or(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rx(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -5298,12 +5298,12 @@ pub fn constructor_or_mem( pub fn constructor_aluop_xor(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if let Some(pattern1_0) = C::gpr32_ty(ctx, pattern0_0) { - // Rule at src/isa/s390x/inst.isle line 2496. + // Rule at src/isa/s390x/inst.isle line 2488. let expr0_0 = ALUOp::Xor32; return Some(expr0_0); } if let Some(pattern1_0) = C::gpr64_ty(ctx, pattern0_0) { - // Rule at src/isa/s390x/inst.isle line 2497. + // Rule at src/isa/s390x/inst.isle line 2489. let expr0_0 = ALUOp::Xor64; return Some(expr0_0); } @@ -5320,7 +5320,7 @@ pub fn constructor_xor_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2500. + // Rule at src/isa/s390x/inst.isle line 2492. let expr0_0 = constructor_aluop_xor(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rrr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -5336,7 +5336,7 @@ pub fn constructor_xor_uimm32shifted( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2503. + // Rule at src/isa/s390x/inst.isle line 2495. let expr0_0 = constructor_aluop_xor(ctx, pattern0_0)?; let expr1_0 = constructor_alu_ruimm32shifted(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; @@ -5353,7 +5353,7 @@ pub fn constructor_xor_mem( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2506. + // Rule at src/isa/s390x/inst.isle line 2498. let expr0_0 = constructor_aluop_xor(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rx(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -5364,7 +5364,7 @@ pub fn constructor_not_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> Op let pattern0_0 = arg0; if let Some(pattern1_0) = C::gpr32_ty(ctx, pattern0_0) { let pattern2_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2512. + // Rule at src/isa/s390x/inst.isle line 2504. let expr0_0: u32 = 4294967295; let expr1_0: u8 = 0; let expr2_0 = C::uimm32shifted(ctx, expr0_0, expr1_0); @@ -5373,7 +5373,7 @@ pub fn constructor_not_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> Op } if let Some(pattern1_0) = C::gpr64_ty(ctx, pattern0_0) { let pattern2_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2514. + // Rule at src/isa/s390x/inst.isle line 2506. let expr0_0: u32 = 4294967295; let expr1_0: u8 = 0; let expr2_0 = C::uimm32shifted(ctx, expr0_0, expr1_0); @@ -5391,12 +5391,12 @@ pub fn constructor_not_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> Op pub fn constructor_aluop_and_not(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if let Some(pattern1_0) = C::gpr32_ty(ctx, pattern0_0) { - // Rule at src/isa/s390x/inst.isle line 2523. + // Rule at src/isa/s390x/inst.isle line 2515. let expr0_0 = ALUOp::AndNot32; return Some(expr0_0); } if let Some(pattern1_0) = C::gpr64_ty(ctx, pattern0_0) { - // Rule at src/isa/s390x/inst.isle line 2524. + // Rule at src/isa/s390x/inst.isle line 2516. let expr0_0 = ALUOp::AndNot64; return Some(expr0_0); } @@ -5413,7 +5413,7 @@ pub fn constructor_and_not_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2527. + // Rule at src/isa/s390x/inst.isle line 2519. let expr0_0 = constructor_aluop_and_not(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rrr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -5423,12 +5423,12 @@ pub fn constructor_and_not_reg( pub fn constructor_aluop_or_not(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if let Some(pattern1_0) = C::gpr32_ty(ctx, pattern0_0) { - // Rule at src/isa/s390x/inst.isle line 2533. + // Rule at src/isa/s390x/inst.isle line 2525. let expr0_0 = ALUOp::OrrNot32; return Some(expr0_0); } if let Some(pattern1_0) = C::gpr64_ty(ctx, pattern0_0) { - // Rule at src/isa/s390x/inst.isle line 2534. + // Rule at src/isa/s390x/inst.isle line 2526. let expr0_0 = ALUOp::OrrNot64; return Some(expr0_0); } @@ -5445,7 +5445,7 @@ pub fn constructor_or_not_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2537. + // Rule at src/isa/s390x/inst.isle line 2529. let expr0_0 = constructor_aluop_or_not(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rrr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -5455,12 +5455,12 @@ pub fn constructor_or_not_reg( pub fn constructor_aluop_xor_not(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if let Some(pattern1_0) = C::gpr32_ty(ctx, pattern0_0) { - // Rule at src/isa/s390x/inst.isle line 2543. + // Rule at src/isa/s390x/inst.isle line 2535. let expr0_0 = ALUOp::XorNot32; return Some(expr0_0); } if let Some(pattern1_0) = C::gpr64_ty(ctx, pattern0_0) { - // Rule at src/isa/s390x/inst.isle line 2544. + // Rule at src/isa/s390x/inst.isle line 2536. let expr0_0 = ALUOp::XorNot64; return Some(expr0_0); } @@ -5477,7 +5477,7 @@ pub fn constructor_xor_not_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2547. + // Rule at src/isa/s390x/inst.isle line 2539. let expr0_0 = constructor_aluop_xor_not(ctx, pattern0_0)?; let expr1_0 = constructor_alu_rrr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -5487,12 +5487,12 @@ pub fn constructor_xor_not_reg( pub fn constructor_unaryop_abs(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 2553. + // Rule at src/isa/s390x/inst.isle line 2545. let expr0_0 = UnaryOp::Abs32; return Some(expr0_0); } if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2554. + // Rule at src/isa/s390x/inst.isle line 2546. let expr0_0 = UnaryOp::Abs64; return Some(expr0_0); } @@ -5503,7 +5503,7 @@ pub fn constructor_unaryop_abs(ctx: &mut C, arg0: Type) -> Option(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2557. + // Rule at src/isa/s390x/inst.isle line 2549. let expr0_0 = UnaryOp::Abs64Ext32; return Some(expr0_0); } @@ -5514,7 +5514,7 @@ pub fn constructor_unaryop_abs_sext32(ctx: &mut C, arg0: Type) -> Op pub fn constructor_abs_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2560. + // Rule at src/isa/s390x/inst.isle line 2552. let expr0_0 = constructor_unaryop_abs(ctx, pattern0_0)?; let expr1_0 = constructor_unary_rr(ctx, pattern0_0, &expr0_0, pattern1_0)?; return Some(expr1_0); @@ -5524,7 +5524,7 @@ pub fn constructor_abs_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> Op pub fn constructor_abs_reg_sext32(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2563. + // Rule at src/isa/s390x/inst.isle line 2555. let expr0_0 = constructor_unaryop_abs_sext32(ctx, pattern0_0)?; let expr1_0 = constructor_unary_rr(ctx, pattern0_0, &expr0_0, pattern1_0)?; return Some(expr1_0); @@ -5534,22 +5534,22 @@ pub fn constructor_abs_reg_sext32(ctx: &mut C, arg0: Type, arg1: Reg pub fn constructor_unaryop_neg(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I8 { - // Rule at src/isa/s390x/inst.isle line 2569. + // Rule at src/isa/s390x/inst.isle line 2561. let expr0_0 = UnaryOp::Neg32; return Some(expr0_0); } if pattern0_0 == I16 { - // Rule at src/isa/s390x/inst.isle line 2570. + // Rule at src/isa/s390x/inst.isle line 2562. let expr0_0 = UnaryOp::Neg32; return Some(expr0_0); } if pattern0_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 2571. + // Rule at src/isa/s390x/inst.isle line 2563. let expr0_0 = UnaryOp::Neg32; return Some(expr0_0); } if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2572. + // Rule at src/isa/s390x/inst.isle line 2564. let expr0_0 = UnaryOp::Neg64; return Some(expr0_0); } @@ -5560,7 +5560,7 @@ pub fn constructor_unaryop_neg(ctx: &mut C, arg0: Type) -> Option(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2575. + // Rule at src/isa/s390x/inst.isle line 2567. let expr0_0 = UnaryOp::Neg64Ext32; return Some(expr0_0); } @@ -5571,7 +5571,7 @@ pub fn constructor_unaryop_neg_sext32(ctx: &mut C, arg0: Type) -> Op pub fn constructor_neg_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2578. + // Rule at src/isa/s390x/inst.isle line 2570. let expr0_0 = constructor_unaryop_neg(ctx, pattern0_0)?; let expr1_0 = constructor_unary_rr(ctx, pattern0_0, &expr0_0, pattern1_0)?; return Some(expr1_0); @@ -5581,7 +5581,7 @@ pub fn constructor_neg_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> Op pub fn constructor_neg_reg_sext32(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2581. + // Rule at src/isa/s390x/inst.isle line 2573. let expr0_0 = constructor_unaryop_neg_sext32(ctx, pattern0_0)?; let expr1_0 = constructor_unary_rr(ctx, pattern0_0, &expr0_0, pattern1_0)?; return Some(expr1_0); @@ -5591,12 +5591,12 @@ pub fn constructor_neg_reg_sext32(ctx: &mut C, arg0: Type, arg1: Reg pub fn constructor_shiftop_rot(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 2587. + // Rule at src/isa/s390x/inst.isle line 2579. let expr0_0 = ShiftOp::RotL32; return Some(expr0_0); } if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2588. + // Rule at src/isa/s390x/inst.isle line 2580. let expr0_0 = ShiftOp::RotL64; return Some(expr0_0); } @@ -5613,7 +5613,7 @@ pub fn constructor_rot_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2591. + // Rule at src/isa/s390x/inst.isle line 2583. let expr0_0 = constructor_shiftop_rot(ctx, pattern0_0)?; let expr1_0: u8 = 0; let expr2_0 = constructor_shift_rr(ctx, pattern0_0, &expr0_0, pattern1_0, expr1_0, pattern2_0)?; @@ -5630,7 +5630,7 @@ pub fn constructor_rot_imm( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2595. + // Rule at src/isa/s390x/inst.isle line 2587. let expr0_0 = constructor_shiftop_rot(ctx, pattern0_0)?; let expr1_0 = C::zero_reg(ctx); let expr2_0 = constructor_shift_rr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0, expr1_0)?; @@ -5641,22 +5641,22 @@ pub fn constructor_rot_imm( pub fn constructor_shiftop_lshl(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I8 { - // Rule at src/isa/s390x/inst.isle line 2602. + // Rule at src/isa/s390x/inst.isle line 2594. let expr0_0 = ShiftOp::LShL32; return Some(expr0_0); } if pattern0_0 == I16 { - // Rule at src/isa/s390x/inst.isle line 2603. + // Rule at src/isa/s390x/inst.isle line 2595. let expr0_0 = ShiftOp::LShL32; return Some(expr0_0); } if pattern0_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 2604. + // Rule at src/isa/s390x/inst.isle line 2596. let expr0_0 = ShiftOp::LShL32; return Some(expr0_0); } if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2605. + // Rule at src/isa/s390x/inst.isle line 2597. let expr0_0 = ShiftOp::LShL64; return Some(expr0_0); } @@ -5673,7 +5673,7 @@ pub fn constructor_lshl_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2608. + // Rule at src/isa/s390x/inst.isle line 2600. let expr0_0 = constructor_shiftop_lshl(ctx, pattern0_0)?; let expr1_0: u8 = 0; let expr2_0 = constructor_shift_rr(ctx, pattern0_0, &expr0_0, pattern1_0, expr1_0, pattern2_0)?; @@ -5690,7 +5690,7 @@ pub fn constructor_lshl_imm( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2612. + // Rule at src/isa/s390x/inst.isle line 2604. let expr0_0 = constructor_shiftop_lshl(ctx, pattern0_0)?; let expr1_0 = C::zero_reg(ctx); let expr2_0 = constructor_shift_rr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0, expr1_0)?; @@ -5701,12 +5701,12 @@ pub fn constructor_lshl_imm( pub fn constructor_shiftop_lshr(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 2619. + // Rule at src/isa/s390x/inst.isle line 2611. let expr0_0 = ShiftOp::LShR32; return Some(expr0_0); } if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2620. + // Rule at src/isa/s390x/inst.isle line 2612. let expr0_0 = ShiftOp::LShR64; return Some(expr0_0); } @@ -5723,7 +5723,7 @@ pub fn constructor_lshr_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2623. + // Rule at src/isa/s390x/inst.isle line 2615. let expr0_0 = constructor_shiftop_lshr(ctx, pattern0_0)?; let expr1_0: u8 = 0; let expr2_0 = constructor_shift_rr(ctx, pattern0_0, &expr0_0, pattern1_0, expr1_0, pattern2_0)?; @@ -5740,7 +5740,7 @@ pub fn constructor_lshr_imm( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2627. + // Rule at src/isa/s390x/inst.isle line 2619. let expr0_0 = constructor_shiftop_lshr(ctx, pattern0_0)?; let expr1_0 = C::zero_reg(ctx); let expr2_0 = constructor_shift_rr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0, expr1_0)?; @@ -5751,12 +5751,12 @@ pub fn constructor_lshr_imm( pub fn constructor_shiftop_ashr(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 2634. + // Rule at src/isa/s390x/inst.isle line 2626. let expr0_0 = ShiftOp::AShR32; return Some(expr0_0); } if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2635. + // Rule at src/isa/s390x/inst.isle line 2627. let expr0_0 = ShiftOp::AShR64; return Some(expr0_0); } @@ -5773,7 +5773,7 @@ pub fn constructor_ashr_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2638. + // Rule at src/isa/s390x/inst.isle line 2630. let expr0_0 = constructor_shiftop_ashr(ctx, pattern0_0)?; let expr1_0: u8 = 0; let expr2_0 = constructor_shift_rr(ctx, pattern0_0, &expr0_0, pattern1_0, expr1_0, pattern2_0)?; @@ -5790,7 +5790,7 @@ pub fn constructor_ashr_imm( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2642. + // Rule at src/isa/s390x/inst.isle line 2634. let expr0_0 = constructor_shiftop_ashr(ctx, pattern0_0)?; let expr1_0 = C::zero_reg(ctx); let expr2_0 = constructor_shift_rr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0, expr1_0)?; @@ -5800,7 +5800,7 @@ pub fn constructor_ashr_imm( // Generated as internal constructor for term popcnt_byte. pub fn constructor_popcnt_byte(ctx: &mut C, arg0: Reg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/s390x/inst.isle line 2649. + // Rule at src/isa/s390x/inst.isle line 2641. let expr0_0: Type = I64; let expr1_0 = UnaryOp::PopcntByte; let expr2_0 = constructor_unary_rr(ctx, expr0_0, &expr1_0, pattern0_0)?; @@ -5810,7 +5810,7 @@ pub fn constructor_popcnt_byte(ctx: &mut C, arg0: Reg) -> Option(ctx: &mut C, arg0: Reg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/s390x/inst.isle line 2652. + // Rule at src/isa/s390x/inst.isle line 2644. let expr0_0: Type = I64; let expr1_0 = UnaryOp::PopcntReg; let expr2_0 = constructor_unary_rr(ctx, expr0_0, &expr1_0, pattern0_0)?; @@ -5828,7 +5828,7 @@ pub fn constructor_atomic_rmw_and( if pattern0_0 == I32 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2658. + // Rule at src/isa/s390x/inst.isle line 2650. let expr0_0: Type = I32; let expr1_0 = ALUOp::And32; let expr2_0 = constructor_atomic_rmw_impl(ctx, expr0_0, &expr1_0, pattern2_0, pattern3_0)?; @@ -5837,7 +5837,7 @@ pub fn constructor_atomic_rmw_and( if pattern0_0 == I64 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2659. + // Rule at src/isa/s390x/inst.isle line 2651. let expr0_0: Type = I64; let expr1_0 = ALUOp::And64; let expr2_0 = constructor_atomic_rmw_impl(ctx, expr0_0, &expr1_0, pattern2_0, pattern3_0)?; @@ -5857,7 +5857,7 @@ pub fn constructor_atomic_rmw_or( if pattern0_0 == I32 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2662. + // Rule at src/isa/s390x/inst.isle line 2654. let expr0_0: Type = I32; let expr1_0 = ALUOp::Orr32; let expr2_0 = constructor_atomic_rmw_impl(ctx, expr0_0, &expr1_0, pattern2_0, pattern3_0)?; @@ -5866,7 +5866,7 @@ pub fn constructor_atomic_rmw_or( if pattern0_0 == I64 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2663. + // Rule at src/isa/s390x/inst.isle line 2655. let expr0_0: Type = I64; let expr1_0 = ALUOp::Orr64; let expr2_0 = constructor_atomic_rmw_impl(ctx, expr0_0, &expr1_0, pattern2_0, pattern3_0)?; @@ -5886,7 +5886,7 @@ pub fn constructor_atomic_rmw_xor( if pattern0_0 == I32 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2666. + // Rule at src/isa/s390x/inst.isle line 2658. let expr0_0: Type = I32; let expr1_0 = ALUOp::Xor32; let expr2_0 = constructor_atomic_rmw_impl(ctx, expr0_0, &expr1_0, pattern2_0, pattern3_0)?; @@ -5895,7 +5895,7 @@ pub fn constructor_atomic_rmw_xor( if pattern0_0 == I64 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2667. + // Rule at src/isa/s390x/inst.isle line 2659. let expr0_0: Type = I64; let expr1_0 = ALUOp::Xor64; let expr2_0 = constructor_atomic_rmw_impl(ctx, expr0_0, &expr1_0, pattern2_0, pattern3_0)?; @@ -5915,7 +5915,7 @@ pub fn constructor_atomic_rmw_add( if pattern0_0 == I32 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2670. + // Rule at src/isa/s390x/inst.isle line 2662. let expr0_0: Type = I32; let expr1_0 = ALUOp::Add32; let expr2_0 = constructor_atomic_rmw_impl(ctx, expr0_0, &expr1_0, pattern2_0, pattern3_0)?; @@ -5924,7 +5924,7 @@ pub fn constructor_atomic_rmw_add( if pattern0_0 == I64 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2671. + // Rule at src/isa/s390x/inst.isle line 2663. let expr0_0: Type = I64; let expr1_0 = ALUOp::Add64; let expr2_0 = constructor_atomic_rmw_impl(ctx, expr0_0, &expr1_0, pattern2_0, pattern3_0)?; @@ -5937,12 +5937,12 @@ pub fn constructor_atomic_rmw_add( pub fn constructor_fpuop2_add(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32 { - // Rule at src/isa/s390x/inst.isle line 2677. + // Rule at src/isa/s390x/inst.isle line 2669. let expr0_0 = FPUOp2::Add32; return Some(expr0_0); } if pattern0_0 == F64 { - // Rule at src/isa/s390x/inst.isle line 2678. + // Rule at src/isa/s390x/inst.isle line 2670. let expr0_0 = FPUOp2::Add64; return Some(expr0_0); } @@ -5959,7 +5959,7 @@ pub fn constructor_fadd_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2681. + // Rule at src/isa/s390x/inst.isle line 2673. let expr0_0 = constructor_fpuop2_add(ctx, pattern0_0)?; let expr1_0 = constructor_fpu_rrr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -5969,12 +5969,12 @@ pub fn constructor_fadd_reg( pub fn constructor_fpuop2_sub(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32 { - // Rule at src/isa/s390x/inst.isle line 2687. + // Rule at src/isa/s390x/inst.isle line 2679. let expr0_0 = FPUOp2::Sub32; return Some(expr0_0); } if pattern0_0 == F64 { - // Rule at src/isa/s390x/inst.isle line 2688. + // Rule at src/isa/s390x/inst.isle line 2680. let expr0_0 = FPUOp2::Sub64; return Some(expr0_0); } @@ -5991,7 +5991,7 @@ pub fn constructor_fsub_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2691. + // Rule at src/isa/s390x/inst.isle line 2683. let expr0_0 = constructor_fpuop2_sub(ctx, pattern0_0)?; let expr1_0 = constructor_fpu_rrr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -6001,12 +6001,12 @@ pub fn constructor_fsub_reg( pub fn constructor_fpuop2_mul(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32 { - // Rule at src/isa/s390x/inst.isle line 2697. + // Rule at src/isa/s390x/inst.isle line 2689. let expr0_0 = FPUOp2::Mul32; return Some(expr0_0); } if pattern0_0 == F64 { - // Rule at src/isa/s390x/inst.isle line 2698. + // Rule at src/isa/s390x/inst.isle line 2690. let expr0_0 = FPUOp2::Mul64; return Some(expr0_0); } @@ -6023,7 +6023,7 @@ pub fn constructor_fmul_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2701. + // Rule at src/isa/s390x/inst.isle line 2693. let expr0_0 = constructor_fpuop2_mul(ctx, pattern0_0)?; let expr1_0 = constructor_fpu_rrr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -6033,12 +6033,12 @@ pub fn constructor_fmul_reg( pub fn constructor_fpuop2_div(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32 { - // Rule at src/isa/s390x/inst.isle line 2707. + // Rule at src/isa/s390x/inst.isle line 2699. let expr0_0 = FPUOp2::Div32; return Some(expr0_0); } if pattern0_0 == F64 { - // Rule at src/isa/s390x/inst.isle line 2708. + // Rule at src/isa/s390x/inst.isle line 2700. let expr0_0 = FPUOp2::Div64; return Some(expr0_0); } @@ -6055,7 +6055,7 @@ pub fn constructor_fdiv_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2711. + // Rule at src/isa/s390x/inst.isle line 2703. let expr0_0 = constructor_fpuop2_div(ctx, pattern0_0)?; let expr1_0 = constructor_fpu_rrr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -6065,12 +6065,12 @@ pub fn constructor_fdiv_reg( pub fn constructor_fpuop2_min(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32 { - // Rule at src/isa/s390x/inst.isle line 2717. + // Rule at src/isa/s390x/inst.isle line 2709. let expr0_0 = FPUOp2::Min32; return Some(expr0_0); } if pattern0_0 == F64 { - // Rule at src/isa/s390x/inst.isle line 2718. + // Rule at src/isa/s390x/inst.isle line 2710. let expr0_0 = FPUOp2::Min64; return Some(expr0_0); } @@ -6087,7 +6087,7 @@ pub fn constructor_fmin_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2721. + // Rule at src/isa/s390x/inst.isle line 2713. let expr0_0 = constructor_fpuop2_min(ctx, pattern0_0)?; let expr1_0 = constructor_fpuvec_rrr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -6097,12 +6097,12 @@ pub fn constructor_fmin_reg( pub fn constructor_fpuop2_max(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32 { - // Rule at src/isa/s390x/inst.isle line 2727. + // Rule at src/isa/s390x/inst.isle line 2719. let expr0_0 = FPUOp2::Max32; return Some(expr0_0); } if pattern0_0 == F64 { - // Rule at src/isa/s390x/inst.isle line 2728. + // Rule at src/isa/s390x/inst.isle line 2720. let expr0_0 = FPUOp2::Max64; return Some(expr0_0); } @@ -6119,7 +6119,7 @@ pub fn constructor_fmax_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2731. + // Rule at src/isa/s390x/inst.isle line 2723. let expr0_0 = constructor_fpuop2_max(ctx, pattern0_0)?; let expr1_0 = constructor_fpuvec_rrr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -6129,12 +6129,12 @@ pub fn constructor_fmax_reg( pub fn constructor_fpuop3_fma(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32 { - // Rule at src/isa/s390x/inst.isle line 2737. + // Rule at src/isa/s390x/inst.isle line 2729. let expr0_0 = FPUOp3::MAdd32; return Some(expr0_0); } if pattern0_0 == F64 { - // Rule at src/isa/s390x/inst.isle line 2738. + // Rule at src/isa/s390x/inst.isle line 2730. let expr0_0 = FPUOp3::MAdd64; return Some(expr0_0); } @@ -6153,7 +6153,7 @@ pub fn constructor_fma_reg( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/s390x/inst.isle line 2741. + // Rule at src/isa/s390x/inst.isle line 2733. let expr0_0 = constructor_fpuop3_fma(ctx, pattern0_0)?; let expr1_0 = constructor_fpu_rrrr( ctx, pattern0_0, &expr0_0, pattern3_0, pattern1_0, pattern2_0, @@ -6165,12 +6165,12 @@ pub fn constructor_fma_reg( pub fn constructor_fpuop1_sqrt(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32 { - // Rule at src/isa/s390x/inst.isle line 2747. + // Rule at src/isa/s390x/inst.isle line 2739. let expr0_0 = FPUOp1::Sqrt32; return Some(expr0_0); } if pattern0_0 == F64 { - // Rule at src/isa/s390x/inst.isle line 2748. + // Rule at src/isa/s390x/inst.isle line 2740. let expr0_0 = FPUOp1::Sqrt64; return Some(expr0_0); } @@ -6181,7 +6181,7 @@ pub fn constructor_fpuop1_sqrt(ctx: &mut C, arg0: Type) -> Option(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2751. + // Rule at src/isa/s390x/inst.isle line 2743. let expr0_0 = constructor_fpuop1_sqrt(ctx, pattern0_0)?; let expr1_0 = constructor_fpu_rr(ctx, pattern0_0, &expr0_0, pattern1_0)?; return Some(expr1_0); @@ -6191,12 +6191,12 @@ pub fn constructor_sqrt_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> O pub fn constructor_fpuop1_neg(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32 { - // Rule at src/isa/s390x/inst.isle line 2757. + // Rule at src/isa/s390x/inst.isle line 2749. let expr0_0 = FPUOp1::Neg32; return Some(expr0_0); } if pattern0_0 == F64 { - // Rule at src/isa/s390x/inst.isle line 2758. + // Rule at src/isa/s390x/inst.isle line 2750. let expr0_0 = FPUOp1::Neg64; return Some(expr0_0); } @@ -6207,7 +6207,7 @@ pub fn constructor_fpuop1_neg(ctx: &mut C, arg0: Type) -> Option(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2761. + // Rule at src/isa/s390x/inst.isle line 2753. let expr0_0 = constructor_fpuop1_neg(ctx, pattern0_0)?; let expr1_0 = constructor_fpu_rr(ctx, pattern0_0, &expr0_0, pattern1_0)?; return Some(expr1_0); @@ -6217,12 +6217,12 @@ pub fn constructor_fneg_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> O pub fn constructor_fpuop1_abs(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32 { - // Rule at src/isa/s390x/inst.isle line 2767. + // Rule at src/isa/s390x/inst.isle line 2759. let expr0_0 = FPUOp1::Abs32; return Some(expr0_0); } if pattern0_0 == F64 { - // Rule at src/isa/s390x/inst.isle line 2768. + // Rule at src/isa/s390x/inst.isle line 2760. let expr0_0 = FPUOp1::Abs64; return Some(expr0_0); } @@ -6233,7 +6233,7 @@ pub fn constructor_fpuop1_abs(ctx: &mut C, arg0: Type) -> Option(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2771. + // Rule at src/isa/s390x/inst.isle line 2763. let expr0_0 = constructor_fpuop1_abs(ctx, pattern0_0)?; let expr1_0 = constructor_fpu_rr(ctx, pattern0_0, &expr0_0, pattern1_0)?; return Some(expr1_0); @@ -6243,12 +6243,12 @@ pub fn constructor_fabs_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> O pub fn constructor_fpuroundmode_ceil(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32 { - // Rule at src/isa/s390x/inst.isle line 2777. + // Rule at src/isa/s390x/inst.isle line 2769. let expr0_0 = FpuRoundMode::Plus32; return Some(expr0_0); } if pattern0_0 == F64 { - // Rule at src/isa/s390x/inst.isle line 2778. + // Rule at src/isa/s390x/inst.isle line 2770. let expr0_0 = FpuRoundMode::Plus64; return Some(expr0_0); } @@ -6259,7 +6259,7 @@ pub fn constructor_fpuroundmode_ceil(ctx: &mut C, arg0: Type) -> Opt pub fn constructor_ceil_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2781. + // Rule at src/isa/s390x/inst.isle line 2773. let expr0_0 = constructor_fpuroundmode_ceil(ctx, pattern0_0)?; let expr1_0 = constructor_fpu_round(ctx, pattern0_0, &expr0_0, pattern1_0)?; return Some(expr1_0); @@ -6269,12 +6269,12 @@ pub fn constructor_ceil_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> O pub fn constructor_fpuroundmode_floor(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32 { - // Rule at src/isa/s390x/inst.isle line 2787. + // Rule at src/isa/s390x/inst.isle line 2779. let expr0_0 = FpuRoundMode::Minus32; return Some(expr0_0); } if pattern0_0 == F64 { - // Rule at src/isa/s390x/inst.isle line 2788. + // Rule at src/isa/s390x/inst.isle line 2780. let expr0_0 = FpuRoundMode::Minus64; return Some(expr0_0); } @@ -6285,7 +6285,7 @@ pub fn constructor_fpuroundmode_floor(ctx: &mut C, arg0: Type) -> Op pub fn constructor_floor_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2791. + // Rule at src/isa/s390x/inst.isle line 2783. let expr0_0 = constructor_fpuroundmode_floor(ctx, pattern0_0)?; let expr1_0 = constructor_fpu_round(ctx, pattern0_0, &expr0_0, pattern1_0)?; return Some(expr1_0); @@ -6295,12 +6295,12 @@ pub fn constructor_floor_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> pub fn constructor_fpuroundmode_trunc(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32 { - // Rule at src/isa/s390x/inst.isle line 2797. + // Rule at src/isa/s390x/inst.isle line 2789. let expr0_0 = FpuRoundMode::Zero32; return Some(expr0_0); } if pattern0_0 == F64 { - // Rule at src/isa/s390x/inst.isle line 2798. + // Rule at src/isa/s390x/inst.isle line 2790. let expr0_0 = FpuRoundMode::Zero64; return Some(expr0_0); } @@ -6311,7 +6311,7 @@ pub fn constructor_fpuroundmode_trunc(ctx: &mut C, arg0: Type) -> Op pub fn constructor_trunc_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2801. + // Rule at src/isa/s390x/inst.isle line 2793. let expr0_0 = constructor_fpuroundmode_trunc(ctx, pattern0_0)?; let expr1_0 = constructor_fpu_round(ctx, pattern0_0, &expr0_0, pattern1_0)?; return Some(expr1_0); @@ -6324,12 +6324,12 @@ pub fn constructor_fpuroundmode_nearest( ) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32 { - // Rule at src/isa/s390x/inst.isle line 2807. + // Rule at src/isa/s390x/inst.isle line 2799. let expr0_0 = FpuRoundMode::Nearest32; return Some(expr0_0); } if pattern0_0 == F64 { - // Rule at src/isa/s390x/inst.isle line 2808. + // Rule at src/isa/s390x/inst.isle line 2800. let expr0_0 = FpuRoundMode::Nearest64; return Some(expr0_0); } @@ -6340,7 +6340,7 @@ pub fn constructor_fpuroundmode_nearest( pub fn constructor_nearest_reg(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/s390x/inst.isle line 2811. + // Rule at src/isa/s390x/inst.isle line 2803. let expr0_0 = constructor_fpuroundmode_nearest(ctx, pattern0_0)?; let expr1_0 = constructor_fpu_round(ctx, pattern0_0, &expr0_0, pattern1_0)?; return Some(expr1_0); @@ -6356,7 +6356,7 @@ pub fn constructor_fpuop1_promote( if pattern0_0 == F64 { let pattern2_0 = arg1; if pattern2_0 == F32 { - // Rule at src/isa/s390x/inst.isle line 2817. + // Rule at src/isa/s390x/inst.isle line 2809. let expr0_0 = FPUOp1::Cvt32To64; return Some(expr0_0); } @@ -6374,7 +6374,7 @@ pub fn constructor_fpromote_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2820. + // Rule at src/isa/s390x/inst.isle line 2812. let expr0_0 = constructor_fpuop1_promote(ctx, pattern0_0, pattern1_0)?; let expr1_0 = constructor_fpu_rr(ctx, pattern0_0, &expr0_0, pattern2_0)?; return Some(expr1_0); @@ -6390,7 +6390,7 @@ pub fn constructor_fpuop1_demote( if pattern0_0 == F32 { let pattern2_0 = arg1; if pattern2_0 == F64 { - // Rule at src/isa/s390x/inst.isle line 2827. + // Rule at src/isa/s390x/inst.isle line 2819. let expr0_0 = FPUOp1::Cvt64To32; return Some(expr0_0); } @@ -6408,7 +6408,7 @@ pub fn constructor_fdemote_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2830. + // Rule at src/isa/s390x/inst.isle line 2822. let expr0_0 = constructor_fpuop1_demote(ctx, pattern0_0, pattern1_0)?; let expr1_0 = constructor_fpu_rr(ctx, pattern0_0, &expr0_0, pattern2_0)?; return Some(expr1_0); @@ -6424,12 +6424,12 @@ pub fn constructor_uint_to_fpu_op( if pattern0_0 == F32 { let pattern2_0 = arg1; if pattern2_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 2837. + // Rule at src/isa/s390x/inst.isle line 2829. let expr0_0 = IntToFpuOp::U32ToF32; return Some(expr0_0); } if pattern2_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2839. + // Rule at src/isa/s390x/inst.isle line 2831. let expr0_0 = IntToFpuOp::U64ToF32; return Some(expr0_0); } @@ -6437,12 +6437,12 @@ pub fn constructor_uint_to_fpu_op( if pattern0_0 == F64 { let pattern2_0 = arg1; if pattern2_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 2838. + // Rule at src/isa/s390x/inst.isle line 2830. let expr0_0 = IntToFpuOp::U32ToF64; return Some(expr0_0); } if pattern2_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2840. + // Rule at src/isa/s390x/inst.isle line 2832. let expr0_0 = IntToFpuOp::U64ToF64; return Some(expr0_0); } @@ -6460,7 +6460,7 @@ pub fn constructor_fcvt_from_uint_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2843. + // Rule at src/isa/s390x/inst.isle line 2835. let expr0_0 = constructor_uint_to_fpu_op(ctx, pattern0_0, pattern1_0)?; let expr1_0 = constructor_int_to_fpu(ctx, pattern0_0, &expr0_0, pattern2_0)?; return Some(expr1_0); @@ -6476,12 +6476,12 @@ pub fn constructor_sint_to_fpu_op( if pattern0_0 == F32 { let pattern2_0 = arg1; if pattern2_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 2850. + // Rule at src/isa/s390x/inst.isle line 2842. let expr0_0 = IntToFpuOp::I32ToF32; return Some(expr0_0); } if pattern2_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2852. + // Rule at src/isa/s390x/inst.isle line 2844. let expr0_0 = IntToFpuOp::I64ToF32; return Some(expr0_0); } @@ -6489,12 +6489,12 @@ pub fn constructor_sint_to_fpu_op( if pattern0_0 == F64 { let pattern2_0 = arg1; if pattern2_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 2851. + // Rule at src/isa/s390x/inst.isle line 2843. let expr0_0 = IntToFpuOp::I32ToF64; return Some(expr0_0); } if pattern2_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2853. + // Rule at src/isa/s390x/inst.isle line 2845. let expr0_0 = IntToFpuOp::I64ToF64; return Some(expr0_0); } @@ -6512,7 +6512,7 @@ pub fn constructor_fcvt_from_sint_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2856. + // Rule at src/isa/s390x/inst.isle line 2848. let expr0_0 = constructor_sint_to_fpu_op(ctx, pattern0_0, pattern1_0)?; let expr1_0 = constructor_int_to_fpu(ctx, pattern0_0, &expr0_0, pattern2_0)?; return Some(expr1_0); @@ -6528,12 +6528,12 @@ pub fn constructor_fpu_to_uint_op( if pattern0_0 == I32 { let pattern2_0 = arg1; if pattern2_0 == F32 { - // Rule at src/isa/s390x/inst.isle line 2863. + // Rule at src/isa/s390x/inst.isle line 2855. let expr0_0 = FpuToIntOp::F32ToU32; return Some(expr0_0); } if pattern2_0 == F64 { - // Rule at src/isa/s390x/inst.isle line 2864. + // Rule at src/isa/s390x/inst.isle line 2856. let expr0_0 = FpuToIntOp::F64ToU32; return Some(expr0_0); } @@ -6541,12 +6541,12 @@ pub fn constructor_fpu_to_uint_op( if pattern0_0 == I64 { let pattern2_0 = arg1; if pattern2_0 == F32 { - // Rule at src/isa/s390x/inst.isle line 2865. + // Rule at src/isa/s390x/inst.isle line 2857. let expr0_0 = FpuToIntOp::F32ToU64; return Some(expr0_0); } if pattern2_0 == F64 { - // Rule at src/isa/s390x/inst.isle line 2866. + // Rule at src/isa/s390x/inst.isle line 2858. let expr0_0 = FpuToIntOp::F64ToU64; return Some(expr0_0); } @@ -6564,7 +6564,7 @@ pub fn constructor_fcvt_to_uint_reg_with_flags( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2869. + // Rule at src/isa/s390x/inst.isle line 2861. let expr0_0 = constructor_fpu_to_uint_op(ctx, pattern0_0, pattern1_0)?; let expr1_0 = constructor_fpu_to_int(ctx, pattern0_0, &expr0_0, pattern2_0)?; return Some(expr1_0); @@ -6580,7 +6580,7 @@ pub fn constructor_fcvt_to_uint_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2873. + // Rule at src/isa/s390x/inst.isle line 2865. let expr0_0 = constructor_fcvt_to_uint_reg_with_flags(ctx, pattern0_0, pattern1_0, pattern2_0)?; let expr1_0 = constructor_drop_flags(ctx, &expr0_0)?; return Some(expr1_0); @@ -6596,12 +6596,12 @@ pub fn constructor_fpu_to_sint_op( if pattern0_0 == I32 { let pattern2_0 = arg1; if pattern2_0 == F32 { - // Rule at src/isa/s390x/inst.isle line 2880. + // Rule at src/isa/s390x/inst.isle line 2872. let expr0_0 = FpuToIntOp::F32ToI32; return Some(expr0_0); } if pattern2_0 == F64 { - // Rule at src/isa/s390x/inst.isle line 2881. + // Rule at src/isa/s390x/inst.isle line 2873. let expr0_0 = FpuToIntOp::F64ToI32; return Some(expr0_0); } @@ -6609,12 +6609,12 @@ pub fn constructor_fpu_to_sint_op( if pattern0_0 == I64 { let pattern2_0 = arg1; if pattern2_0 == F32 { - // Rule at src/isa/s390x/inst.isle line 2882. + // Rule at src/isa/s390x/inst.isle line 2874. let expr0_0 = FpuToIntOp::F32ToI64; return Some(expr0_0); } if pattern2_0 == F64 { - // Rule at src/isa/s390x/inst.isle line 2883. + // Rule at src/isa/s390x/inst.isle line 2875. let expr0_0 = FpuToIntOp::F64ToI64; return Some(expr0_0); } @@ -6632,7 +6632,7 @@ pub fn constructor_fcvt_to_sint_reg_with_flags( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2886. + // Rule at src/isa/s390x/inst.isle line 2878. let expr0_0 = constructor_fpu_to_sint_op(ctx, pattern0_0, pattern1_0)?; let expr1_0 = constructor_fpu_to_int(ctx, pattern0_0, &expr0_0, pattern2_0)?; return Some(expr1_0); @@ -6648,7 +6648,7 @@ pub fn constructor_fcvt_to_sint_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2890. + // Rule at src/isa/s390x/inst.isle line 2882. let expr0_0 = constructor_fcvt_to_sint_reg_with_flags(ctx, pattern0_0, pattern1_0, pattern2_0)?; let expr1_0 = constructor_drop_flags(ctx, &expr0_0)?; return Some(expr1_0); @@ -6658,12 +6658,12 @@ pub fn constructor_fcvt_to_sint_reg( pub fn constructor_cmpop_cmps(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 2897. + // Rule at src/isa/s390x/inst.isle line 2889. let expr0_0 = CmpOp::CmpS32; return Some(expr0_0); } if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2898. + // Rule at src/isa/s390x/inst.isle line 2890. let expr0_0 = CmpOp::CmpS64; return Some(expr0_0); } @@ -6674,12 +6674,12 @@ pub fn constructor_cmpop_cmps(ctx: &mut C, arg0: Type) -> Option(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 2901. + // Rule at src/isa/s390x/inst.isle line 2893. let expr0_0 = CmpOp::CmpS32Ext16; return Some(expr0_0); } if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2902. + // Rule at src/isa/s390x/inst.isle line 2894. let expr0_0 = CmpOp::CmpS64Ext16; return Some(expr0_0); } @@ -6690,7 +6690,7 @@ pub fn constructor_cmpop_cmps_sext16(ctx: &mut C, arg0: Type) -> Opt pub fn constructor_cmpop_cmps_sext32(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2905. + // Rule at src/isa/s390x/inst.isle line 2897. let expr0_0 = CmpOp::CmpS64Ext32; return Some(expr0_0); } @@ -6707,7 +6707,7 @@ pub fn constructor_icmps_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2908. + // Rule at src/isa/s390x/inst.isle line 2900. let expr0_0 = constructor_cmpop_cmps(ctx, pattern0_0)?; let expr1_0 = constructor_cmp_rr(ctx, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -6723,7 +6723,7 @@ pub fn constructor_icmps_reg_sext32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2911. + // Rule at src/isa/s390x/inst.isle line 2903. let expr0_0 = constructor_cmpop_cmps_sext32(ctx, pattern0_0)?; let expr1_0 = constructor_cmp_rr(ctx, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -6739,7 +6739,7 @@ pub fn constructor_icmps_simm16( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2914. + // Rule at src/isa/s390x/inst.isle line 2906. let expr0_0 = constructor_cmpop_cmps(ctx, pattern0_0)?; let expr1_0 = constructor_cmp_rsimm16(ctx, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -6755,7 +6755,7 @@ pub fn constructor_icmps_simm32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2917. + // Rule at src/isa/s390x/inst.isle line 2909. let expr0_0 = constructor_cmpop_cmps(ctx, pattern0_0)?; let expr1_0 = constructor_cmp_rsimm32(ctx, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -6771,7 +6771,7 @@ pub fn constructor_icmps_mem( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2920. + // Rule at src/isa/s390x/inst.isle line 2912. let expr0_0 = constructor_cmpop_cmps(ctx, pattern0_0)?; let expr1_0 = constructor_cmp_rx(ctx, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -6787,7 +6787,7 @@ pub fn constructor_icmps_mem_sext16( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2923. + // Rule at src/isa/s390x/inst.isle line 2915. let expr0_0 = constructor_cmpop_cmps_sext16(ctx, pattern0_0)?; let expr1_0 = constructor_cmp_rx(ctx, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -6803,7 +6803,7 @@ pub fn constructor_icmps_mem_sext32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2926. + // Rule at src/isa/s390x/inst.isle line 2918. let expr0_0 = constructor_cmpop_cmps_sext32(ctx, pattern0_0)?; let expr1_0 = constructor_cmp_rx(ctx, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -6813,12 +6813,12 @@ pub fn constructor_icmps_mem_sext32( pub fn constructor_cmpop_cmpu(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 2932. + // Rule at src/isa/s390x/inst.isle line 2924. let expr0_0 = CmpOp::CmpL32; return Some(expr0_0); } if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2933. + // Rule at src/isa/s390x/inst.isle line 2925. let expr0_0 = CmpOp::CmpL64; return Some(expr0_0); } @@ -6829,12 +6829,12 @@ pub fn constructor_cmpop_cmpu(ctx: &mut C, arg0: Type) -> Option(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I32 { - // Rule at src/isa/s390x/inst.isle line 2936. + // Rule at src/isa/s390x/inst.isle line 2928. let expr0_0 = CmpOp::CmpL32Ext16; return Some(expr0_0); } if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2937. + // Rule at src/isa/s390x/inst.isle line 2929. let expr0_0 = CmpOp::CmpL64Ext16; return Some(expr0_0); } @@ -6845,7 +6845,7 @@ pub fn constructor_cmpop_cmpu_zext16(ctx: &mut C, arg0: Type) -> Opt pub fn constructor_cmpop_cmpu_zext32(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I64 { - // Rule at src/isa/s390x/inst.isle line 2940. + // Rule at src/isa/s390x/inst.isle line 2932. let expr0_0 = CmpOp::CmpL64Ext32; return Some(expr0_0); } @@ -6862,7 +6862,7 @@ pub fn constructor_icmpu_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2943. + // Rule at src/isa/s390x/inst.isle line 2935. let expr0_0 = constructor_cmpop_cmpu(ctx, pattern0_0)?; let expr1_0 = constructor_cmp_rr(ctx, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -6878,7 +6878,7 @@ pub fn constructor_icmpu_reg_zext32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2946. + // Rule at src/isa/s390x/inst.isle line 2938. let expr0_0 = constructor_cmpop_cmpu_zext32(ctx, pattern0_0)?; let expr1_0 = constructor_cmp_rr(ctx, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -6894,7 +6894,7 @@ pub fn constructor_icmpu_uimm32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2949. + // Rule at src/isa/s390x/inst.isle line 2941. let expr0_0 = constructor_cmpop_cmpu(ctx, pattern0_0)?; let expr1_0 = constructor_cmp_ruimm32(ctx, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -6910,7 +6910,7 @@ pub fn constructor_icmpu_mem( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2952. + // Rule at src/isa/s390x/inst.isle line 2944. let expr0_0 = constructor_cmpop_cmpu(ctx, pattern0_0)?; let expr1_0 = constructor_cmp_rx(ctx, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -6926,7 +6926,7 @@ pub fn constructor_icmpu_mem_zext16( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2955. + // Rule at src/isa/s390x/inst.isle line 2947. let expr0_0 = constructor_cmpop_cmpu_zext16(ctx, pattern0_0)?; let expr1_0 = constructor_cmp_rx(ctx, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -6942,7 +6942,7 @@ pub fn constructor_icmpu_mem_zext32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2958. + // Rule at src/isa/s390x/inst.isle line 2950. let expr0_0 = constructor_cmpop_cmpu_zext32(ctx, pattern0_0)?; let expr1_0 = constructor_cmp_rx(ctx, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -6959,14 +6959,14 @@ pub fn constructor_fcmp_reg( if pattern0_0 == F32 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2964. + // Rule at src/isa/s390x/inst.isle line 2956. let expr0_0 = constructor_fpu_cmp32(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if pattern0_0 == F64 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/s390x/inst.isle line 2965. + // Rule at src/isa/s390x/inst.isle line 2957. let expr0_0 = constructor_fpu_cmp64(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } diff --git a/cranelift/codegen/src/isa/x64/encoding/rex.rs b/cranelift/codegen/src/isa/x64/encoding/rex.rs index 17e5408cc1b9..509309205df5 100644 --- a/cranelift/codegen/src/isa/x64/encoding/rex.rs +++ b/cranelift/codegen/src/isa/x64/encoding/rex.rs @@ -47,7 +47,8 @@ pub(crate) fn encode_sib(shift: u8, enc_index: u8, enc_base: u8) -> u8 { /// Get the encoding number of a GPR. #[inline(always)] -pub(crate) fn int_reg_enc(reg: Reg) -> u8 { +pub(crate) fn int_reg_enc(reg: impl Into) -> u8 { + let reg = reg.into(); debug_assert!(reg.is_real()); debug_assert_eq!(reg.get_class(), RegClass::I64); reg.get_hw_encoding() @@ -55,7 +56,8 @@ pub(crate) fn int_reg_enc(reg: Reg) -> u8 { /// Get the encoding number of any register. #[inline(always)] -pub(crate) fn reg_enc(reg: Reg) -> u8 { +pub(crate) fn reg_enc(reg: impl Into) -> u8 { + let reg = reg.into(); debug_assert!(reg.is_real()); reg.get_hw_encoding() } diff --git a/cranelift/codegen/src/isa/x64/inst.isle b/cranelift/codegen/src/isa/x64/inst.isle index 8c52609cbe03..6f3081a4bbf1 100644 --- a/cranelift/codegen/src/isa/x64/inst.isle +++ b/cranelift/codegen/src/isa/x64/inst.isle @@ -2,82 +2,486 @@ ;;;; `MInst` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -(type MInst extern - (enum (Nop (len u8)) - (Ud2 (trap_code TrapCode)) - (AluRmiR (size OperandSize) - (op AluRmiROpcode) - (src1 Reg) - (src2 RegMemImm) - (dst WritableReg)) - (MulHi (size OperandSize) - (signed bool) - (src1 Reg) - (src2 RegMem) - (dst_lo WritableReg) - (dst_hi WritableReg)) - (XmmRmR (op SseOpcode) - (src1 Reg) - (src2 RegMem) - (dst WritableReg)) - (XmmUnaryRmR (op SseOpcode) - (src RegMem) - (dst WritableReg)) - (XmmUnaryRmREvex (op Avx512Opcode) - (src RegMem) - (dst WritableReg)) - (XmmRmiReg (opcode SseOpcode) - (src1 Reg) - (src2 RegMemImm) +;; Don't build `MInst` variants directly, in general. Instead, use the +;; instruction-emitting helpers defined further down. + +(type MInst nodebug + (enum + ;; Nops of various sizes, including zero. + (Nop (len u8)) + + ;; ========================================= + ;; Integer instructions. + + ;; Integer arithmetic/bit-twiddling. + (AluRmiR (size OperandSize) ;; 4 or 8 + (op AluRmiROpcode) + (src1 Reg) + (src2 RegMemImm) + (dst WritableReg)) + + ;; Instructions on general-purpose registers that only read src and + ;; defines dst (dst is not modified). `bsr`, etc. + (UnaryRmR (size OperandSize) ;; 2, 4, or 8 + (op UnaryRmROpcode) + (src GprMem) + (dst WritableGpr)) + + ;; Bitwise not. + (Not (size OperandSize) ;; 1, 2, 4, or 8 + (src Gpr) + (dst WritableGpr)) + + ;; Integer negation. + (Neg (size OperandSize) ;; 1, 2, 4, or 8 + (src Gpr) + (dst WritableGpr)) + + ;; Integer quotient and remainder: (div idiv) $rax $rdx (reg addr) + (Div (size OperandSize) ;; 1, 2, 4, or 8 + (signed bool) + (divisor RegMem) + (dividend Reg) + (dst_quotient WritableReg) + (dst_remainder WritableReg)) + + ;; The high (and low) bits of a (un)signed multiply: `RDX:RAX := RAX * + ;; rhs`. + (MulHi (size OperandSize) + (signed bool) + (src1 Reg) + (src2 RegMem) + (dst_lo WritableReg) + (dst_hi WritableReg)) + + ;; A synthetic sequence to implement the right inline checks for + ;; remainder and division, assuming the dividend is in %rax. + ;; + ;; Puts the result back into %rax if is_div, %rdx if !is_div, to mimic + ;; what the div instruction does. + ;; + ;; The generated code sequence is described in the emit's function match + ;; arm for this instruction. + ;; + ;; Note: %rdx is marked as modified by this instruction, to avoid an + ;; early clobber problem with the temporary and divisor registers. Make + ;; sure to zero %rdx right before this instruction, or you might run into + ;; regalloc failures where %rdx is live before its first def! + (CheckedDivOrRemSeq (kind DivOrRemKind) + (size OperandSize) + (dividend Reg) + ;; The divisor operand. Note it's marked as modified + ;; so that it gets assigned a register different from + ;; the temporary. + (divisor WritableReg) + (dst_quotient WritableReg) + (dst_remainder WritableReg) + (tmp OptionWritableReg)) + + ;; Do a sign-extend based on the sign of the value in rax into rdx: (cwd + ;; cdq cqo) or al into ah: (cbw) + (SignExtendData (size OperandSize) ;; 1, 2, 4, or 8 + (src Reg) (dst WritableReg)) - (XmmRmRImm (op SseOpcode) - (src1 Reg) - (src2 RegMem) - (dst WritableReg) - (imm u8) - (size OperandSize)) - (XmmUninitializedValue (dst WritableReg)) - (CmpRmiR (size OperandSize) - (opcode CmpOpcode) - (src RegMemImm) - (dst Reg)) - (Imm (dst_size OperandSize) - (simm64 u64) + + ;; Constant materialization: (imm32 imm64) reg. + ;; + ;; Either: movl $imm32, %reg32 or movabsq $imm64, %reg32. + (Imm (dst_size OperandSize) ;; 4 or 8 + (simm64 u64) + (dst WritableReg)) + + ;; GPR to GPR move: mov (64 32) reg reg. + (MovRR (size OperandSize) ;; 4 or 8 + (src Reg) + (dst WritableReg)) + + ;; Zero-extended loads, except for 64 bits: movz (bl bq wl wq lq) addr + ;; reg. + ;; + ;; Note that the lq variant doesn't really exist since the default + ;; zero-extend rule makes it unnecessary. For that case we emit the + ;; equivalent "movl AM, reg32". + (MovzxRmR (ext_mode ExtMode) + (src RegMem) (dst WritableReg)) - (ShiftR (size OperandSize) - (kind ShiftKind) - (src Reg) - (num_bits Imm8Reg) - (dst WritableReg)) - (MovzxRmR (ext_mode ExtMode) - (src RegMem) - (dst WritableReg)) - (MovsxRmR (ext_mode ExtMode) - (src RegMem) - (dst WritableReg)) - (Mov64MR (src SyntheticAmode) - (dst WritableReg)) - (Cmove (size OperandSize) - (cc CC) - (consequent RegMem) - (alternative Reg) - (dst WritableReg)) - (XmmRmREvex (op Avx512Opcode) - (src1 RegMem) - (src2 Reg) - (dst WritableReg)) - (GprToXmm (op SseOpcode) - (src RegMem) - (dst WritableReg) - (src_size OperandSize)) - (Not (size OperandSize) - (src Reg) + + ;; A plain 64-bit integer load, since MovZX_RM_R can't represent that. + (Mov64MR (src SyntheticAmode) + (dst WritableReg)) + + ;; Loads the memory address of addr into dst. + (LoadEffectiveAddress (addr SyntheticAmode) + (dst WritableGpr)) + + ;; Sign-extended loads and moves: movs (bl bq wl wq lq) addr reg. + (MovsxRmR (ext_mode ExtMode) + (src RegMem) (dst WritableReg)) - (Neg (size OperandSize) + + ;; Integer stores: mov (b w l q) reg addr. + (MovRM (size OperandSize) ;; 1, 2, 4, or 8 + (src Reg) + (dst SyntheticAmode)) + + ;; Arithmetic shifts: (shl shr sar) (b w l q) imm reg. + (ShiftR (size OperandSize) ;; 1, 2, 4, or 8 + (kind ShiftKind) + (src Reg) + ;; shift count: `Imm8Reg::Imm8(0 .. #bits-in-type - 1)` or + ;; `Imm8Reg::Reg(r)` where `r` get's move mitosis'd into `%cl`. + (num_bits Imm8Reg) + (dst WritableReg)) + + ;; Arithmetic SIMD shifts. + (XmmRmiReg (opcode SseOpcode) + (src1 Xmm) + (src2 XmmMemImm) + (dst WritableXmm)) + + ;; Integer comparisons/tests: cmp or test (b w l q) (reg addr imm) reg. + (CmpRmiR (size OperandSize) ;; 1, 2, 4, or 8 + (opcode CmpOpcode) + (src RegMemImm) + (dst Reg)) + + ;; Materializes the requested condition code in the destinaton reg. + (Setcc (cc CC) + (dst WritableReg)) + + ;; Integer conditional move. + ;; + ;; Overwrites the destination register. + (Cmove (size OperandSize) + (cc CC) + (consequent RegMem) + (alternative Reg) + (dst WritableReg)) + + ;; ========================================= + ;; Stack manipulation. + + ;; pushq (reg addr imm) + (Push64 (src RegMemImm)) + + ;; popq reg + (Pop64 (dst WritableReg)) + + ;; ========================================= + ;; Floating-point operations. + + ;; XMM (scalar or vector) binary op: (add sub and or xor mul adc? sbb?) + ;; (32 64) (reg addr) reg + (XmmRmR (op SseOpcode) + (src1 Xmm) + (src2 XmmMem) + (dst WritableXmm)) + + ;; XMM (scalar or vector) binary op that relies on the EVEX prefix. + (XmmRmREvex (op Avx512Opcode) + (src1 XmmMem) + (src2 Xmm) + (dst WritableXmm)) + + ;; XMM (scalar or vector) unary op: mov between XMM registers (32 64) + ;; (reg addr) reg, sqrt, etc. + ;; + ;; This differs from XMM_RM_R in that the dst register of XmmUnaryRmR is + ;; not used in the computation of the instruction dst value and so does + ;; not have to be a previously valid value. This is characteristic of mov + ;; instructions. + (XmmUnaryRmR (op SseOpcode) + (src XmmMem) + (dst WritableXmm)) + + ;; XMM (scalar or vector) unary op that relies on the EVEX prefix. + (XmmUnaryRmREvex (op Avx512Opcode) + (src XmmMem) + (dst WritableXmm)) + + ;; XMM (scalar or vector) unary op (from xmm to reg/mem): stores, movd, + ;; movq + (XmmMovRM (op SseOpcode) (src Reg) + (dst SyntheticAmode)) + + ;; XMM (vector) unary op (to move a constant value into an xmm register): + ;; movups + (XmmLoadConst (src VCodeConstant) + (dst WritableReg) + (ty Type)) + + ;; XMM (scalar) unary op (from xmm to integer reg): movd, movq, + ;; cvtts{s,d}2si + (XmmToGpr (op SseOpcode) + (src Xmm) + (dst WritableGpr) + (dst_size OperandSize)) + + ;; XMM (scalar) unary op (from integer to float reg): movd, movq, + ;; cvtsi2s{s,d} + (GprToXmm (op SseOpcode) + (src RegMem) + (dst WritableXmm) + (src_size OperandSize)) + + ;; Converts an unsigned int64 to a float32/float64. + (CvtUint64ToFloatSeq (dst_size OperandSize) ;; 4 or 8 + ;; A copy of the source register, fed by + ;; lowering. It is marked as modified during + ;; register allocation to make sure that the + ;; temporary registers differ from the src register, + ;; since both registers are live at the same time in + ;; the generated code sequence. + (src WritableGpr) + (dst WritableXmm) + (tmp_gpr1 WritableGpr) + (tmp_gpr2 WritableGpr)) + + ;; Converts a scalar xmm to a signed int32/int64. + (CvtFloatToSintSeq (dst_size OperandSize) + (src_size OperandSize) + (is_saturating bool) + ;; A copy of the source register, fed by + ;; lowering. It is marked as modified during + ;; register allocation to make sure that the + ;; temporary registers differ from the src register, + ;; since both registers are live at the same time in + ;; the generated code sequence. + (src WritableXmm) + (dst WritableGpr) + (tmp_gpr WritableGpr) + (tmp_xmm WritableXmm)) + + ;; Converts a scalar xmm to an unsigned int32/int64. + (CvtFloatToUintSeq (dst_size OperandSize) + (src_size OperandSize) + (is_saturating bool) + ;; A copy of the source register, fed by + ;; lowering. It is marked as modified during + ;; register allocation to make sure that the + ;; temporary registers differ from the src register, + ;; since both registers are live at the same time in + ;; the generated code sequence. + (src WritableXmm) + (dst WritableGpr) + (tmp_gpr WritableGpr) + (tmp_xmm WritableXmm)) + + ;; A sequence to compute min/max with the proper NaN semantics for xmm + ;; registers. + (XmmMinMaxSeq (size OperandSize) + (is_min bool) + (lhs Reg) + (rhs_dst WritableReg)) + + ;; XMM (scalar) conditional move. + ;; + ;; Overwrites the destination register if cc is set. + (XmmCmove (size OperandSize) + (cc CC) + (src RegMem) (dst WritableReg)) - (LoadEffectiveAddress (addr SyntheticAmode) - (dst WritableReg)))) + + ;; Float comparisons/tests: cmp (b w l q) (reg addr imm) reg. + (XmmCmpRmR (op SseOpcode) + (src RegMem) + (dst Reg)) + + ;; A binary XMM instruction with an 8-bit immediate: e.g. cmp (ps pd) imm + ;; (reg addr) reg + ;; + ;; Note: this has to use `Reg*`, not `Xmm*`, operands because it is used + ;; in various lane insertion and extraction instructions that move + ;; between XMMs and GPRs. + (XmmRmRImm (op SseOpcode) + (src1 Reg) + (src2 RegMem) + (dst WritableReg) + (imm u8) + (size OperandSize)) + + ;; ========================================= + ;; Control flow instructions. + + ;; Direct call: call simm32. + (CallKnown (dest ExternalName) + (uses VecReg) + (defs VecWritableReg) + (opcode Opcode)) + + ;; Indirect call: callq (reg mem) + (CallUnknown (dest RegMem) + (uses VecReg) + (defs VecWritableReg) + (opcode Opcode)) + + ;; Return. + (Ret) + + ;; A placeholder instruction, generating no code, meaning that a function + ;; epilogue must be inserted there. + (EpiloguePlaceholder) + + ;; Jump to a known target: jmp simm32. + (JmpKnown (dst MachLabel)) + + + ;; One-way conditional branch: jcond cond target. + ;; + ;; This instruction is useful when we have conditional jumps depending on + ;; more than two conditions, see for instance the lowering of Brz/brnz + ;; with Fcmp inputs. + ;; + ;; A note of caution: in contexts where the branch target is another + ;; block, this has to be the same successor as the one specified in the + ;; terminator branch of the current block. Otherwise, this might confuse + ;; register allocation by creating new invisible edges. + (JmpIf (cc CC) + (taken MachLabel)) + + ;; Two-way conditional branch: jcond cond target target. + ;; + ;; Emitted as a compound sequence; the MachBuffer will shrink it as + ;; appropriate. + (JmpCond (cc CC) + (taken MachLabel) + (not_taken MachLabel)) + + ;; Jump-table sequence, as one compound instruction (see note in lower.rs + ;; for rationale). + ;; + ;; The generated code sequence is described in the emit's function match + ;; arm for this instruction. + ;; + ;; See comment in lowering about the temporaries signedness. + (JmpTableSeq (idx Reg) + (tmp1 WritableReg) + (tmp2 WritableReg) + (default_target MachLabel) + (targets VecMachLabel) + (targets_for_term VecMachLabel)) + + ;; Indirect jump: jmpq (reg mem). + (JmpUnknown (target RegMem)) + + ;; Traps if the condition code is set. + (TrapIf (cc CC) + (trap_code TrapCode)) + + ;; A debug trap. + (Hlt) + + ;; An instruction that will always trigger the illegal instruction + ;; exception. + (Ud2 (trap_code TrapCode)) + + ;; Loads an external symbol in a register, with a relocation: + ;; + ;; movq $name@GOTPCREL(%rip), dst if PIC is enabled, or + ;; movabsq $name, dst otherwise. + (LoadExtName (dst WritableReg) + (name BoxExternalName) + (offset i64)) + + ;; ========================================= + ;; Instructions pertaining to atomic memory accesses. + + ;; A standard (native) `lock cmpxchg src, (amode)`, with register + ;; conventions: + ;; + ;; `mem` (read) address + ;; `replacement` (read) replacement value + ;; %rax (modified) in: expected value, out: value that was actually at `dst` + ;; %rflags is written. Do not assume anything about it after the instruction. + ;; + ;; The instruction "succeeded" iff the lowest `ty` bits of %rax + ;; afterwards are the same as they were before. + (LockCmpxchg (ty Type) ;; I8, I16, I32, or I64 + (replacement Reg) + (expected Reg) + (mem SyntheticAmode) + (dst_old WritableReg)) + + ;; A synthetic instruction, based on a loop around a native `lock + ;; cmpxchg` instruction. + ;; + ;; This atomically modifies a value in memory and returns the old value. + ;; The sequence consists of an initial "normal" load from `dst`, followed + ;; by a loop which computes the new value and tries to compare-and-swap + ;; ("CAS") it into `dst`, using the native instruction `lock + ;; cmpxchg{b,w,l,q}` . The loop iterates until the CAS is successful. + ;; If there is no contention, there will be only one pass through the + ;; loop body. The sequence does *not* perform any explicit memory fence + ;; instructions (mfence/sfence/lfence). + ;; + ;; Note that the transaction is atomic in the sense that, as observed by + ;; some other thread, `dst` either has the initial or final value, but no + ;; other. It isn't atomic in the sense of guaranteeing that no other + ;; thread writes to `dst` in between the initial load and the CAS -- but + ;; that would cause the CAS to fail unless the other thread's last write + ;; before the CAS wrote the same value that was already there. In other + ;; words, this implementation suffers (unavoidably) from the A-B-A + ;; problem. + ;; + ;; This instruction sequence has fixed register uses as follows: + ;; + ;; %r9 (read) address + ;; %r10 (read) second operand for `op` + ;; %r11 (written) scratch reg; value afterwards has no meaning + ;; %rax (written) the old value at %r9 + ;; %rflags is written. Do not assume anything about it after the instruction. + (AtomicRmwSeq (ty Type) ;; I8, I16, I32, or I64 + (op AtomicRmwOp) + (address Reg) + (operand Reg) + (temp WritableReg) + (dst_old WritableReg)) + + ;; A memory fence (mfence, lfence or sfence). + (Fence (kind FenceKind)) + + ;; ========================================= + ;; Meta-instructions generating no code. + + ;; Marker, no-op in generated code: SP "virtual offset" is adjusted. + ;; + ;; This controls how `MemArg::NominalSPOffset` args are lowered. + (VirtualSPOffsetAdj (offset i64)) + + ;; Provides a way to tell the register allocator that the upcoming + ;; sequence of instructions will overwrite `dst` so it should be + ;; considered as a `def`; use this with care. + ;; + ;; This is useful when we have a sequence of instructions whose register + ;; usages are nominally `mod`s, but such that the combination of + ;; operations creates a result that is independent of the initial + ;; register value. It's thus semantically a `def`, not a `mod`, when all + ;; the instructions are taken together, so we want to ensure the register + ;; is defined (its live-range starts) prior to the sequence to keep + ;; analyses happy. + ;; + ;; One alternative would be a compound instruction that somehow + ;; encapsulates the others and reports its own `def`s/`use`s/`mod`s; this + ;; adds complexity (the instruction list is no longer flat) and requires + ;; knowledge about semantics and initial-value independence anyway. + (XmmUninitializedValue (dst WritableXmm)) + + ;; A call to the `ElfTlsGetAddr` libcall. Returns address of TLS symbol + ;; in `rax`. + (ElfTlsGetAddr (symbol ExternalName)) + + ;; A Mach-O TLS symbol access. Returns address of the TLS symbol in + ;; `rax`. + (MachOTlsGetAddr (symbol ExternalName)) + + ;; A definition of a value label. + (ValueLabelMarker (reg Reg) + (label ValueLabel)) + + ;; An unwind pseudoinstruction describing the state of the machine at + ;; this program point. + (Unwind (inst UnwindInst)))) (type OperandSize extern (enum Size8 @@ -85,6 +489,13 @@ Size32 Size64)) +(type VCodeConstant (primitive VCodeConstant)) + +(type FenceKind extern + (enum MFence + LFence + SFence)) + ;; Get the `OperandSize` for a given `Type`, rounding smaller types up to 32 bits. (decl operand_size_of_type_32_64 (Type) OperandSize) (extern constructor operand_size_of_type_32_64 operand_size_of_type_32_64) @@ -112,6 +523,19 @@ And8 Or8)) +(type UnaryRmROpcode extern + (enum Bsr + Bsf + Lzcnt + Tzcnt + Popcnt)) + +(type DivOrRemKind extern + (enum SignedDiv + UnsignedDiv + SignedRem + UnsignedRem)) + (type SseOpcode extern (enum Addps Addpd @@ -330,7 +754,7 @@ (type Amode extern (enum)) -(decl amode_imm_reg_reg_shift (u32 Reg Reg u8) Amode) +(decl amode_imm_reg_reg_shift (u32 Gpr Gpr u8) Amode) (extern constructor amode_imm_reg_reg_shift amode_imm_reg_reg_shift) (decl amode_to_synthetic_amode (Amode) SyntheticAmode) @@ -397,13 +821,176 @@ (decl encode_fcmp_imm (FcmpImm) u8) (extern constructor encode_fcmp_imm encode_fcmp_imm) +;;;; Newtypes for Different Register Classes ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(type Gpr (primitive Gpr)) +(type WritableGpr (primitive WritableGpr)) +(type GprMem extern (enum)) +(type GprMemImm extern (enum)) + +(type Xmm (primitive Xmm)) +(type WritableXmm (primitive WritableXmm)) +(type XmmMem extern (enum)) +(type XmmMemImm extern (enum)) + +;; Convert a `WritableGpr` to a `WritableReg`. +(decl writable_gpr_to_reg (WritableGpr) WritableReg) +(extern constructor writable_gpr_to_reg writable_gpr_to_reg) + +;; Convert a `WritableXmm` to a `WritableReg`. +(decl writable_xmm_to_reg (WritableXmm) WritableReg) +(extern constructor writable_xmm_to_reg writable_xmm_to_reg) + +;; Convert a `WritableReg` to a `WritableXmm`. +(decl writable_reg_to_xmm (WritableReg) WritableXmm) +(extern constructor writable_reg_to_xmm writable_reg_to_xmm) + +;; Convert a `WritableXmm` to an `Xmm`. +(decl writable_xmm_to_xmm (WritableXmm) Xmm) +(extern constructor writable_xmm_to_xmm writable_xmm_to_xmm) + +;; Convert a `WritableGpr` to an `Gpr`. +(decl writable_gpr_to_gpr (WritableGpr) Gpr) +(extern constructor writable_gpr_to_gpr writable_gpr_to_gpr) + +;; Convert an `Gpr` to a `Reg`. +(decl gpr_to_reg (Gpr) Reg) +(extern constructor gpr_to_reg gpr_to_reg) + +;; Convert an `Xmm` to a `Reg`. +(decl xmm_to_reg (Xmm) Reg) +(extern constructor xmm_to_reg xmm_to_reg) + +;; Convert an `Xmm` into an `XmmMemImm`. +(decl xmm_to_xmm_mem_imm (Xmm) XmmMemImm) +(extern constructor xmm_to_xmm_mem_imm xmm_to_xmm_mem_imm) + +;; Allocate a new temporary GPR register. +(decl temp_writable_gpr () WritableGpr) +(extern constructor temp_writable_gpr temp_writable_gpr) + +;; Allocate a new temporary XMM register. +(decl temp_writable_xmm () WritableXmm) +(extern constructor temp_writable_xmm temp_writable_xmm) + +;; Construct a new `XmmMem` from the given `RegMem`. +;; +;; Asserts that the `RegMem`'s register, if any, is an XMM register. +(decl xmm_mem_new (RegMem) XmmMem) +(extern constructor xmm_mem_new xmm_mem_new) + +;; Construct a new `GprMemImm` from the given `RegMemImm`. +;; +;; Asserts that the `RegMemImm`'s register, if any, is an GPR register. +(decl gpr_mem_imm_new (RegMemImm) GprMemImm) +(extern constructor gpr_mem_imm_new gpr_mem_imm_new) + +;; Construct a new `XmmMemImm` from the given `RegMemImm`. +;; +;; Asserts that the `RegMemImm`'s register, if any, is an XMM register. +(decl xmm_mem_imm_new (RegMemImm) XmmMemImm) +(extern constructor xmm_mem_imm_new xmm_mem_imm_new) + +;; Construct a new `XmmMem` from an `Xmm`. +(decl xmm_to_xmm_mem (Xmm) XmmMem) +(extern constructor xmm_to_xmm_mem xmm_to_xmm_mem) + +;; Construct a new `XmmMem` from an `RegMem`. +(decl xmm_mem_to_reg_mem (XmmMem) RegMem) +(extern constructor xmm_mem_to_reg_mem xmm_mem_to_reg_mem) + +;; Convert a `GprMem` to a `RegMem`. +(decl gpr_mem_to_reg_mem (GprMem) RegMem) +(extern constructor gpr_mem_to_reg_mem gpr_mem_to_reg_mem) + +;; Construct a new `Xmm` from a `Reg`. +;; +;; Asserts that the register is a XMM. +(decl xmm_new (Reg) Xmm) +(extern constructor xmm_new xmm_new) + +;; Construct a new `Gpr` from a `Reg`. +;; +;; Asserts that the register is a GPR. +(decl gpr_new (Reg) Gpr) +(extern constructor gpr_new gpr_new) + +;; Construct a new `GprMem` from a `RegMem`. +;; +;; Asserts that the `RegMem`'s register, if any, is a GPR. +(decl gpr_mem_new (RegMem) GprMem) +(extern constructor gpr_mem_new gpr_mem_new) + +;; Construct a `GprMem` from a `Reg`. +;; +;; Asserts that the `Reg` is a GPR. +(decl reg_to_gpr_mem (Reg) GprMem) +(extern constructor reg_to_gpr_mem reg_to_gpr_mem) + +;; Put a value into a GPR. +;; +;; Asserts that the value goes into a GPR. +(decl put_in_gpr (Value) Gpr) +(rule (put_in_gpr val) + (gpr_new (put_in_reg val))) + +;; Put a value into a `GprMem`. +;; +;; Asserts that the value goes into a GPR. +(decl put_in_gpr_mem (Value) GprMem) +(rule (put_in_gpr_mem val) + (gpr_mem_new (put_in_reg_mem val))) + +;; Put a value into a `GprMemImm`. +;; +;; Asserts that the value goes into a GPR. +(decl put_in_gpr_mem_imm (Value) GprMemImm) +(rule (put_in_gpr_mem_imm val) + (gpr_mem_imm_new (put_in_reg_mem_imm val))) + +;; Put a value into a XMM. +;; +;; Asserts that the value goes into a XMM. +(decl put_in_xmm (Value) Xmm) +(rule (put_in_xmm val) + (xmm_new (put_in_reg val))) + +;; Put a value into a `XmmMem`. +;; +;; Asserts that the value goes into a XMM. +(decl put_in_xmm_mem (Value) XmmMem) +(rule (put_in_xmm_mem val) + (xmm_mem_new (put_in_reg_mem val))) + +;; Put a value into a `XmmMemImm`. +;; +;; Asserts that the value goes into a XMM. +(decl put_in_xmm_mem_imm (Value) XmmMemImm) +(rule (put_in_xmm_mem_imm val) + (xmm_mem_imm_new (put_in_reg_mem_imm val))) + +;; Construct a `ValueRegs` out of a single GPR register. +(decl value_gpr (Gpr) ValueRegs) +(rule (value_gpr x) + (value_reg (gpr_to_reg x))) + +;; Construct a `ValueRegs` out of two GPR registers. +(decl value_gprs (Gpr Gpr) ValueRegs) +(rule (value_gprs x y) + (value_regs (gpr_to_reg x) (gpr_to_reg y))) + +;; Construct a `ValueRegs` out of a single XMM register. +(decl value_xmm (Xmm) ValueRegs) +(rule (value_xmm x) + (value_reg (xmm_to_reg x))) + ;;;; Helpers for Getting Particular Physical Registers ;;;;;;;;;;;;;;;;;;;;;;;;; ;; ;; These should only be used for legalization purposes, when we can't otherwise ;; rely on something like `Inst::mov_mitosis` to put an operand into the ;; appropriate physical register for whatever reason. -(decl xmm0 () WritableReg) +(decl xmm0 () WritableXmm) (extern constructor xmm0 xmm0) ;;;; Helpers for Querying Enabled ISA Extensions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -507,8 +1094,8 @@ (rule (sse_xor_op $F64X2) (SseOpcode.Xorpd)) (rule (sse_xor_op (multi_lane _bits _lanes)) (SseOpcode.Pxor)) -;; Performs an xor operation of the two operands specified -(decl sse_xor (Type Reg RegMem) Reg) +;; Performs an xor operation of the two operands specified. +(decl sse_xor (Type Xmm XmmMem) Xmm) (rule (sse_xor ty x y) (xmm_rm_r ty (sse_xor_op ty) x y)) ;; Determine the appropriate operation to compare two vectors of the specified @@ -532,45 +1119,46 @@ ;; then it runs the risk of comparing NaN against NaN and not actually producing ;; an all-ones mask. By using integer comparision operations we're guaranteeed ;; that everything is equal to itself. -(decl vector_all_ones (Type) Reg) +(decl vector_all_ones (Type) Xmm) (rule (vector_all_ones ty) - (let ((wr WritableReg (temp_writable_reg ty)) - (r Reg (writable_reg_to_reg wr)) + (let ((wr WritableXmm (temp_writable_xmm)) + (r Xmm (writable_xmm_to_xmm wr)) (_ Unit (emit (MInst.XmmRmR (sse_cmp_op $I32X4) r - (RegMem.Reg r) + (xmm_to_xmm_mem r) wr)))) r)) ;; Helper for creating an SSE register holding an `i64x2` from two `i64` values. -(decl make_i64x2_from_lanes (RegMem RegMem) Reg) +(decl make_i64x2_from_lanes (GprMem GprMem) Xmm) (rule (make_i64x2_from_lanes lo hi) - (let ((dst_w WritableReg (temp_writable_reg $I64X2)) - (dst_r Reg (writable_reg_to_reg dst_w)) - (_0 Unit (emit (MInst.XmmUninitializedValue dst_w))) + (let ((dst_xmm_w WritableXmm (temp_writable_xmm)) + (dst_reg_w WritableReg (writable_xmm_to_reg dst_xmm_w)) + (dst_xmm_r Xmm (writable_xmm_to_xmm dst_xmm_w)) + (dst_reg_r Reg (xmm_to_reg dst_xmm_r)) + (_0 Unit (emit (MInst.XmmUninitializedValue dst_xmm_w))) (_1 Unit (emit (MInst.XmmRmRImm (SseOpcode.Pinsrd) - dst_r - lo - dst_w + dst_reg_r + (gpr_mem_to_reg_mem lo) + dst_reg_w 0 (OperandSize.Size64)))) (_2 Unit (emit (MInst.XmmRmRImm (SseOpcode.Pinsrd) - dst_r - hi - dst_w + dst_reg_r + (gpr_mem_to_reg_mem hi) + dst_reg_w 1 (OperandSize.Size64))))) - dst_r)) + dst_xmm_r)) ;; Move a `RegMemImm.Reg` operand to an XMM register, if necessary. -(decl reg_mem_imm_to_xmm (RegMemImm) RegMemImm) -(rule (reg_mem_imm_to_xmm rmi @ (RegMemImm.Mem _)) rmi) -(rule (reg_mem_imm_to_xmm rmi @ (RegMemImm.Imm _)) rmi) -(rule (reg_mem_imm_to_xmm (RegMemImm.Reg r)) - (RegMemImm.Reg (gpr_to_xmm $I8X16 - (SseOpcode.Movd) - (RegMem.Reg r) - (OperandSize.Size32)))) +(decl mov_rmi_to_xmm (RegMemImm) XmmMemImm) +(rule (mov_rmi_to_xmm rmi @ (RegMemImm.Mem _)) (xmm_mem_imm_new rmi)) +(rule (mov_rmi_to_xmm rmi @ (RegMemImm.Imm _)) (xmm_mem_imm_new rmi)) +(rule (mov_rmi_to_xmm (RegMemImm.Reg r)) + (xmm_to_xmm_mem_imm (gpr_to_xmm (SseOpcode.Movd) + (reg_to_gpr_mem r) + (OperandSize.Size32)))) ;;;; Helpers for Emitting Loads ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -588,24 +1176,24 @@ (writable_reg_to_reg dst))) (rule (x64_load $F32 addr _ext_kind) - (xmm_unary_rm_r (SseOpcode.Movss) - (synthetic_amode_to_reg_mem addr))) + (xmm_to_reg (xmm_unary_rm_r (SseOpcode.Movss) + (xmm_mem_new (synthetic_amode_to_reg_mem addr))))) (rule (x64_load $F64 addr _ext_kind) - (xmm_unary_rm_r (SseOpcode.Movsd) - (synthetic_amode_to_reg_mem addr))) + (xmm_to_reg (xmm_unary_rm_r (SseOpcode.Movsd) + (xmm_mem_new (synthetic_amode_to_reg_mem addr))))) (rule (x64_load $F32X4 addr _ext_kind) - (xmm_unary_rm_r (SseOpcode.Movups) - (synthetic_amode_to_reg_mem addr))) + (xmm_to_reg (xmm_unary_rm_r (SseOpcode.Movups) + (xmm_mem_new (synthetic_amode_to_reg_mem addr))))) (rule (x64_load $F64X2 addr _ext_kind) - (xmm_unary_rm_r (SseOpcode.Movupd) - (synthetic_amode_to_reg_mem addr))) + (xmm_to_reg (xmm_unary_rm_r (SseOpcode.Movupd) + (xmm_mem_new (synthetic_amode_to_reg_mem addr))))) (rule (x64_load (multi_lane _bits _lanes) addr _ext_kind) - (xmm_unary_rm_r (SseOpcode.Movdqu) - (synthetic_amode_to_reg_mem addr))) + (xmm_to_reg (xmm_unary_rm_r (SseOpcode.Movdqu) + (xmm_mem_new (synthetic_amode_to_reg_mem addr))))) ;;;; Instruction Constructors ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; @@ -728,11 +1316,15 @@ ;; `f32` immediates. (rule (imm $F32 bits) - (gpr_to_xmm $F32 (SseOpcode.Movd) (RegMem.Reg (imm $I32 bits)) (OperandSize.Size32))) + (xmm_to_reg (gpr_to_xmm (SseOpcode.Movd) + (gpr_mem_new (RegMem.Reg (imm $I32 bits))) + (OperandSize.Size32)))) ;; `f64` immediates. (rule (imm $F64 bits) - (gpr_to_xmm $F64 (SseOpcode.Movq) (RegMem.Reg (imm $I64 bits)) (OperandSize.Size64))) + (xmm_to_reg (gpr_to_xmm (SseOpcode.Movq) + (gpr_mem_new (RegMem.Reg (imm $I64 bits))) + (OperandSize.Size64)))) (decl nonzero_u64_fits_in_u32 (u64) u64) (extern extractor nonzero_u64_fits_in_u32 nonzero_u64_fits_in_u32) @@ -759,35 +1351,35 @@ ;; Special case for zero immediates with vector types, they turn into an xor ;; specific to the vector type. (rule (imm ty @ (multi_lane _bits _lanes) 0) - (let ((wr WritableReg (temp_writable_reg ty)) - (r Reg (writable_reg_to_reg wr)) + (let ((wr WritableXmm (temp_writable_xmm)) + (r Xmm (writable_xmm_to_xmm wr)) (_ Unit (emit (MInst.XmmRmR (sse_xor_op ty) r - (RegMem.Reg r) + (xmm_to_xmm_mem r) wr)))) - r)) + (xmm_to_reg r))) ;; Special case for `f32` zero immediates to use `xorps`. (rule (imm $F32 0) - (let ((wr WritableReg (temp_writable_reg $F32)) - (r Reg (writable_reg_to_reg wr)) + (let ((wr WritableXmm (temp_writable_xmm)) + (r Xmm (writable_xmm_to_xmm wr)) (_ Unit (emit (MInst.XmmRmR (SseOpcode.Xorps) r - (RegMem.Reg r) + (xmm_to_xmm_mem r) wr)))) - r)) + (xmm_to_reg r))) ;; TODO: use cmpeqps for all 1s ;; Special case for `f64` zero immediates to use `xorpd`. (rule (imm $F64 0) - (let ((wr WritableReg (temp_writable_reg $F64)) - (r Reg (writable_reg_to_reg wr)) + (let ((wr WritableXmm (temp_writable_xmm)) + (r Xmm (writable_xmm_to_xmm wr)) (_ Unit (emit (MInst.XmmRmR (SseOpcode.Xorpd) r - (RegMem.Reg r) + (xmm_to_xmm_mem r) wr)))) - r)) + (xmm_to_reg r))) ;; TODO: use cmpeqpd for all 1s @@ -870,199 +1462,199 @@ (writable_reg_to_reg dst))) ;; Helper for creating `MInst.XmmRmR` instructions. -(decl xmm_rm_r (Type SseOpcode Reg RegMem) Reg) +(decl xmm_rm_r (Type SseOpcode Xmm XmmMem) Xmm) (rule (xmm_rm_r ty op src1 src2) - (let ((dst WritableReg (temp_writable_reg ty)) + (let ((dst WritableXmm (temp_writable_xmm)) (_ Unit (emit (MInst.XmmRmR op src1 src2 dst)))) - (writable_reg_to_reg dst))) + (writable_xmm_to_xmm dst))) ;; Helper for creating `paddb` instructions. -(decl paddb (Reg RegMem) Reg) +(decl paddb (Xmm XmmMem) Xmm) (rule (paddb src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Paddb) src1 src2)) ;; Helper for creating `paddw` instructions. -(decl paddw (Reg RegMem) Reg) +(decl paddw (Xmm XmmMem) Xmm) (rule (paddw src1 src2) (xmm_rm_r $I16X8 (SseOpcode.Paddw) src1 src2)) ;; Helper for creating `paddd` instructions. -(decl paddd (Reg RegMem) Reg) +(decl paddd (Xmm XmmMem) Xmm) (rule (paddd src1 src2) (xmm_rm_r $I32X4 (SseOpcode.Paddd) src1 src2)) ;; Helper for creating `paddq` instructions. -(decl paddq (Reg RegMem) Reg) +(decl paddq (Xmm XmmMem) Xmm) (rule (paddq src1 src2) (xmm_rm_r $I64X2 (SseOpcode.Paddq) src1 src2)) ;; Helper for creating `paddsb` instructions. -(decl paddsb (Reg RegMem) Reg) +(decl paddsb (Xmm XmmMem) Xmm) (rule (paddsb src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Paddsb) src1 src2)) ;; Helper for creating `paddsw` instructions. -(decl paddsw (Reg RegMem) Reg) +(decl paddsw (Xmm XmmMem) Xmm) (rule (paddsw src1 src2) (xmm_rm_r $I16X8 (SseOpcode.Paddsw) src1 src2)) ;; Helper for creating `paddusb` instructions. -(decl paddusb (Reg RegMem) Reg) +(decl paddusb (Xmm XmmMem) Xmm) (rule (paddusb src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Paddusb) src1 src2)) ;; Helper for creating `paddusw` instructions. -(decl paddusw (Reg RegMem) Reg) +(decl paddusw (Xmm XmmMem) Xmm) (rule (paddusw src1 src2) (xmm_rm_r $I16X8 (SseOpcode.Paddusw) src1 src2)) ;; Helper for creating `psubb` instructions. -(decl psubb (Reg RegMem) Reg) +(decl psubb (Xmm XmmMem) Xmm) (rule (psubb src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Psubb) src1 src2)) ;; Helper for creating `psubw` instructions. -(decl psubw (Reg RegMem) Reg) +(decl psubw (Xmm XmmMem) Xmm) (rule (psubw src1 src2) (xmm_rm_r $I16X8 (SseOpcode.Psubw) src1 src2)) ;; Helper for creating `psubd` instructions. -(decl psubd (Reg RegMem) Reg) +(decl psubd (Xmm XmmMem) Xmm) (rule (psubd src1 src2) (xmm_rm_r $I32X4 (SseOpcode.Psubd) src1 src2)) ;; Helper for creating `psubq` instructions. -(decl psubq (Reg RegMem) Reg) +(decl psubq (Xmm XmmMem) Xmm) (rule (psubq src1 src2) (xmm_rm_r $I64X2 (SseOpcode.Psubq) src1 src2)) ;; Helper for creating `psubsb` instructions. -(decl psubsb (Reg RegMem) Reg) +(decl psubsb (Xmm XmmMem) Xmm) (rule (psubsb src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Psubsb) src1 src2)) ;; Helper for creating `psubsw` instructions. -(decl psubsw (Reg RegMem) Reg) +(decl psubsw (Xmm XmmMem) Xmm) (rule (psubsw src1 src2) (xmm_rm_r $I16X8 (SseOpcode.Psubsw) src1 src2)) ;; Helper for creating `psubusb` instructions. -(decl psubusb (Reg RegMem) Reg) +(decl psubusb (Xmm XmmMem) Xmm) (rule (psubusb src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Psubusb) src1 src2)) ;; Helper for creating `psubusw` instructions. -(decl psubusw (Reg RegMem) Reg) +(decl psubusw (Xmm XmmMem) Xmm) (rule (psubusw src1 src2) (xmm_rm_r $I16X8 (SseOpcode.Psubusw) src1 src2)) ;; Helper for creating `pavgb` instructions. -(decl pavgb (Reg RegMem) Reg) +(decl pavgb (Xmm XmmMem) Xmm) (rule (pavgb src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Pavgb) src1 src2)) ;; Helper for creating `pavgw` instructions. -(decl pavgw (Reg RegMem) Reg) +(decl pavgw (Xmm XmmMem) Xmm) (rule (pavgw src1 src2) (xmm_rm_r $I16X8 (SseOpcode.Pavgw) src1 src2)) ;; Helper for creating `pand` instructions. -(decl pand (Reg RegMem) Reg) +(decl pand (Xmm XmmMem) Xmm) (rule (pand src1 src2) (xmm_rm_r $F32X4 (SseOpcode.Pand) src1 src2)) ;; Helper for creating `andps` instructions. -(decl andps (Reg RegMem) Reg) +(decl andps (Xmm XmmMem) Xmm) (rule (andps src1 src2) (xmm_rm_r $F32X4 (SseOpcode.Andps) src1 src2)) ;; Helper for creating `andpd` instructions. -(decl andpd (Reg RegMem) Reg) +(decl andpd (Xmm XmmMem) Xmm) (rule (andpd src1 src2) (xmm_rm_r $F64X2 (SseOpcode.Andpd) src1 src2)) ;; Helper for creating `por` instructions. -(decl por (Reg RegMem) Reg) +(decl por (Xmm XmmMem) Xmm) (rule (por src1 src2) (xmm_rm_r $F32X4 (SseOpcode.Por) src1 src2)) ;; Helper for creating `orps` instructions. -(decl orps (Reg RegMem) Reg) +(decl orps (Xmm XmmMem) Xmm) (rule (orps src1 src2) (xmm_rm_r $F32X4 (SseOpcode.Orps) src1 src2)) ;; Helper for creating `orpd` instructions. -(decl orpd (Reg RegMem) Reg) +(decl orpd (Xmm XmmMem) Xmm) (rule (orpd src1 src2) (xmm_rm_r $F64X2 (SseOpcode.Orpd) src1 src2)) ;; Helper for creating `pxor` instructions. -(decl pxor (Reg RegMem) Reg) +(decl pxor (Xmm XmmMem) Xmm) (rule (pxor src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Pxor) src1 src2)) ;; Helper for creating `xorps` instructions. -(decl xorps (Reg RegMem) Reg) +(decl xorps (Xmm XmmMem) Xmm) (rule (xorps src1 src2) (xmm_rm_r $F32X4 (SseOpcode.Xorps) src1 src2)) ;; Helper for creating `xorpd` instructions. -(decl xorpd (Reg RegMem) Reg) +(decl xorpd (Xmm XmmMem) Xmm) (rule (xorpd src1 src2) (xmm_rm_r $F64X2 (SseOpcode.Xorpd) src1 src2)) ;; Helper for creating `pmullw` instructions. -(decl pmullw (Reg RegMem) Reg) +(decl pmullw (Xmm XmmMem) Xmm) (rule (pmullw src1 src2) (xmm_rm_r $I16X8 (SseOpcode.Pmullw) src1 src2)) ;; Helper for creating `pmulld` instructions. -(decl pmulld (Reg RegMem) Reg) +(decl pmulld (Xmm XmmMem) Xmm) (rule (pmulld src1 src2) (xmm_rm_r $I16X8 (SseOpcode.Pmulld) src1 src2)) ;; Helper for creating `pmulhw` instructions. -(decl pmulhw (Reg RegMem) Reg) +(decl pmulhw (Xmm XmmMem) Xmm) (rule (pmulhw src1 src2) (xmm_rm_r $I16X8 (SseOpcode.Pmulhw) src1 src2)) ;; Helper for creating `pmulhuw` instructions. -(decl pmulhuw (Reg RegMem) Reg) +(decl pmulhuw (Xmm XmmMem) Xmm) (rule (pmulhuw src1 src2) (xmm_rm_r $I16X8 (SseOpcode.Pmulhuw) src1 src2)) ;; Helper for creating `pmuldq` instructions. -(decl pmuldq (Reg RegMem) Reg) +(decl pmuldq (Xmm XmmMem) Xmm) (rule (pmuldq src1 src2) (xmm_rm_r $I16X8 (SseOpcode.Pmuldq) src1 src2)) ;; Helper for creating `pmuludq` instructions. -(decl pmuludq (Reg RegMem) Reg) +(decl pmuludq (Xmm XmmMem) Xmm) (rule (pmuludq src1 src2) (xmm_rm_r $I64X2 (SseOpcode.Pmuludq) src1 src2)) ;; Helper for creating `punpckhwd` instructions. -(decl punpckhwd (Reg RegMem) Reg) +(decl punpckhwd (Xmm XmmMem) Xmm) (rule (punpckhwd src1 src2) (xmm_rm_r $I16X8 (SseOpcode.Punpckhwd) src1 src2)) ;; Helper for creating `punpcklwd` instructions. -(decl punpcklwd (Reg RegMem) Reg) +(decl punpcklwd (Xmm XmmMem) Xmm) (rule (punpcklwd src1 src2) (xmm_rm_r $I16X8 (SseOpcode.Punpcklwd) src1 src2)) ;; Helper for creating `andnps` instructions. -(decl andnps (Reg RegMem) Reg) +(decl andnps (Xmm XmmMem) Xmm) (rule (andnps src1 src2) (xmm_rm_r $F32X4 (SseOpcode.Andnps) src1 src2)) ;; Helper for creating `andnpd` instructions. -(decl andnpd (Reg RegMem) Reg) +(decl andnpd (Xmm XmmMem) Xmm) (rule (andnpd src1 src2) (xmm_rm_r $F64X2 (SseOpcode.Andnpd) src1 src2)) ;; Helper for creating `pandn` instructions. -(decl pandn (Reg RegMem) Reg) +(decl pandn (Xmm XmmMem) Xmm) (rule (pandn src1 src2) (xmm_rm_r $F64X2 (SseOpcode.Pandn) src1 src2)) @@ -1077,367 +1669,387 @@ (rule (sse_mov_op (multi_lane _bits _lanes)) (SseOpcode.Movdqa)) ;; Helper for creating `blendvp{d,s}` and `pblendvb` instructions. -(decl sse_blend (Type RegMem RegMem Reg) Reg) +(decl sse_blend (Type XmmMem XmmMem Xmm) Xmm) (rule (sse_blend ty mask src1 src2) ;; Move the mask into `xmm0`, as blend instructions implicitly operate on ;; that register. (This kind of thing would normally happen inside of ;; `Inst::mov_mitosis`, but has to happen here, where we still have the ;; mask register, because the mask is implicit and doesn't appear in the ;; `Inst` itself.) - (let ((mask2 WritableReg (xmm0)) - (_ Unit (emit (MInst.XmmUnaryRmR (sse_mov_op ty) mask mask2)))) + (let ((mask2 WritableXmm (xmm0)) + (_ Unit (emit (MInst.XmmUnaryRmR (sse_mov_op ty) + mask + mask2)))) (xmm_rm_r ty (sse_blend_op ty) src2 src1))) ;; Helper for creating `blendvpd` instructions. -(decl blendvpd (Reg RegMem Reg) Reg) +(decl blendvpd (Xmm XmmMem Xmm) Xmm) (rule (blendvpd src1 src2 mask) ;; Move the mask into `xmm0`, as `blendvpd` implicitly operates on that ;; register. (This kind of thing would normally happen inside of ;; `Inst::mov_mitosis`, but has to happen here, where we still have the ;; mask register, because the mask is implicit and doesn't appear in the ;; `Inst` itself.) - (let ((mask2 WritableReg (xmm0)) - (_ Unit (emit (MInst.XmmUnaryRmR (SseOpcode.Movapd) (RegMem.Reg mask) mask2)))) + (let ((mask2 WritableXmm (xmm0)) + (_ Unit (emit (MInst.XmmUnaryRmR (SseOpcode.Movapd) + (xmm_to_xmm_mem mask) + mask2)))) (xmm_rm_r $F64X2 (SseOpcode.Blendvpd) src1 src2))) ;; Helper for creating `movsd` instructions. -(decl movsd (Reg RegMem) Reg) +(decl movsd (Xmm XmmMem) Xmm) (rule (movsd src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Movsd) src1 src2)) ;; Helper for creating `movlhps` instructions. -(decl movlhps (Reg RegMem) Reg) +(decl movlhps (Xmm XmmMem) Xmm) (rule (movlhps src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Movlhps) src1 src2)) ;; Helper for creating `pmaxsb` instructions. -(decl pmaxsb (Reg RegMem) Reg) +(decl pmaxsb (Xmm XmmMem) Xmm) (rule (pmaxsb src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Pmaxsb) src1 src2)) ;; Helper for creating `pmaxsw` instructions. -(decl pmaxsw (Reg RegMem) Reg) +(decl pmaxsw (Xmm XmmMem) Xmm) (rule (pmaxsw src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Pmaxsw) src1 src2)) ;; Helper for creating `pmaxsd` instructions. -(decl pmaxsd (Reg RegMem) Reg) +(decl pmaxsd (Xmm XmmMem) Xmm) (rule (pmaxsd src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Pmaxsd) src1 src2)) ;; Helper for creating `pminsb` instructions. -(decl pminsb (Reg RegMem) Reg) +(decl pminsb (Xmm XmmMem) Xmm) (rule (pminsb src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Pminsb) src1 src2)) ;; Helper for creating `pminsw` instructions. -(decl pminsw (Reg RegMem) Reg) +(decl pminsw (Xmm XmmMem) Xmm) (rule (pminsw src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Pminsw) src1 src2)) ;; Helper for creating `pminsd` instructions. -(decl pminsd (Reg RegMem) Reg) +(decl pminsd (Xmm XmmMem) Xmm) (rule (pminsd src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Pminsd) src1 src2)) ;; Helper for creating `pmaxub` instructions. -(decl pmaxub (Reg RegMem) Reg) +(decl pmaxub (Xmm XmmMem) Xmm) (rule (pmaxub src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Pmaxub) src1 src2)) ;; Helper for creating `pmaxuw` instructions. -(decl pmaxuw (Reg RegMem) Reg) +(decl pmaxuw (Xmm XmmMem) Xmm) (rule (pmaxuw src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Pmaxuw) src1 src2)) ;; Helper for creating `pmaxud` instructions. -(decl pmaxud (Reg RegMem) Reg) +(decl pmaxud (Xmm XmmMem) Xmm) (rule (pmaxud src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Pmaxud) src1 src2)) ;; Helper for creating `pminub` instructions. -(decl pminub (Reg RegMem) Reg) +(decl pminub (Xmm XmmMem) Xmm) (rule (pminub src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Pminub) src1 src2)) ;; Helper for creating `pminuw` instructions. -(decl pminuw (Reg RegMem) Reg) +(decl pminuw (Xmm XmmMem) Xmm) (rule (pminuw src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Pminuw) src1 src2)) ;; Helper for creating `pminud` instructions. -(decl pminud (Reg RegMem) Reg) +(decl pminud (Xmm XmmMem) Xmm) (rule (pminud src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Pminud) src1 src2)) ;; Helper for creating `punpcklbw` instructions. -(decl punpcklbw (Reg RegMem) Reg) +(decl punpcklbw (Xmm XmmMem) Xmm) (rule (punpcklbw src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Punpcklbw) src1 src2)) ;; Helper for creating `punpckhbw` instructions. -(decl punpckhbw (Reg RegMem) Reg) +(decl punpckhbw (Xmm XmmMem) Xmm) (rule (punpckhbw src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Punpckhbw) src1 src2)) ;; Helper for creating `packsswb` instructions. -(decl packsswb (Reg RegMem) Reg) +(decl packsswb (Xmm XmmMem) Xmm) (rule (packsswb src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Packsswb) src1 src2)) ;; Helper for creating `MInst.XmmRmRImm` instructions. -(decl xmm_rm_r_imm (SseOpcode Reg RegMem u8 OperandSize) Reg) +(decl xmm_rm_r_imm (SseOpcode Reg RegMem u8 OperandSize) Xmm) (rule (xmm_rm_r_imm op src1 src2 imm size) - (let ((dst WritableReg (temp_writable_reg $I8X16)) + (let ((dst WritableXmm (temp_writable_xmm)) (_ Unit (emit (MInst.XmmRmRImm op src1 src2 - dst + (writable_xmm_to_reg dst) imm size)))) - (writable_reg_to_reg dst))) + (writable_xmm_to_xmm dst))) ;; Helper for creating `palignr` instructions. -(decl palignr (Reg RegMem u8 OperandSize) Reg) +(decl palignr (Xmm XmmMem u8 OperandSize) Xmm) (rule (palignr src1 src2 imm size) (xmm_rm_r_imm (SseOpcode.Palignr) - src1 - src2 + (xmm_to_reg src1) + (xmm_mem_to_reg_mem src2) imm size)) +;; Helper for creating `cmpps` instructions. +(decl cmpps (Xmm XmmMem FcmpImm) Xmm) +(rule (cmpps src1 src2 imm) + (xmm_rm_r_imm (SseOpcode.Cmpps) + (xmm_to_reg src1) + (xmm_mem_to_reg_mem src2) + (encode_fcmp_imm imm) + (OperandSize.Size32))) + +;; Helper for creating `pinsrb` instructions. +(decl pinsrb (Xmm GprMem u8) Xmm) +(rule (pinsrb src1 src2 lane) + (xmm_rm_r_imm (SseOpcode.Pinsrb) + (xmm_to_reg src1) + (gpr_mem_to_reg_mem src2) + lane + (OperandSize.Size32))) + +;; Helper for creating `pinsrw` instructions. +(decl pinsrw (Xmm GprMem u8) Xmm) +(rule (pinsrw src1 src2 lane) + (xmm_rm_r_imm (SseOpcode.Pinsrw) + (xmm_to_reg src1) + (gpr_mem_to_reg_mem src2) + lane + (OperandSize.Size32))) + +;; Helper for creating `pinsrd` instructions. +(decl pinsrd (Xmm GprMem u8 OperandSize) Xmm) +(rule (pinsrd src1 src2 lane size) + (xmm_rm_r_imm (SseOpcode.Pinsrd) + (xmm_to_reg src1) + (gpr_mem_to_reg_mem src2) + lane + size)) + +;; Helper for creating `insertps` instructions. +(decl insertps (Xmm XmmMem u8) Xmm) +(rule (insertps src1 src2 lane) + (xmm_rm_r_imm (SseOpcode.Insertps) + (xmm_to_reg src1) + (xmm_mem_to_reg_mem src2) + lane + (OperandSize.Size32))) + ;; Helper for creating `pshufd` instructions. -(decl pshufd (RegMem u8 OperandSize) Reg) +(decl pshufd (XmmMem u8 OperandSize) Xmm) (rule (pshufd src imm size) - (let ((w_dst WritableReg (temp_writable_reg $I8X16)) - (dst Reg (writable_reg_to_reg w_dst)) + (let ((w_dst WritableXmm (temp_writable_xmm)) + (dst Xmm (writable_xmm_to_xmm w_dst)) (_ Unit (emit (MInst.XmmRmRImm (SseOpcode.Pshufd) - dst - src - w_dst + (xmm_to_reg dst) + (xmm_mem_to_reg_mem src) + (writable_xmm_to_reg w_dst) imm size)))) dst)) ;; Helper for creating `MInst.XmmUnaryRmR` instructions. -(decl xmm_unary_rm_r (SseOpcode RegMem) Reg) +(decl xmm_unary_rm_r (SseOpcode XmmMem) Xmm) (rule (xmm_unary_rm_r op src) - (let ((dst WritableReg (temp_writable_reg $I8X16)) + (let ((dst WritableXmm (temp_writable_xmm)) (_ Unit (emit (MInst.XmmUnaryRmR op src dst)))) - (writable_reg_to_reg dst))) + (writable_xmm_to_xmm dst))) ;; Helper for creating `pmovsxbw` instructions. -(decl pmovsxbw (RegMem) Reg) +(decl pmovsxbw (XmmMem) Xmm) (rule (pmovsxbw src) (xmm_unary_rm_r (SseOpcode.Pmovsxbw) src)) ;; Helper for creating `pmovzxbw` instructions. -(decl pmovzxbw (RegMem) Reg) +(decl pmovzxbw (XmmMem) Xmm) (rule (pmovzxbw src) (xmm_unary_rm_r (SseOpcode.Pmovzxbw) src)) ;; Helper for creating `pabsb` instructions. -(decl pabsb (RegMem) Reg) +(decl pabsb (XmmMem) Xmm) (rule (pabsb src) (xmm_unary_rm_r (SseOpcode.Pabsb) src)) ;; Helper for creating `pabsw` instructions. -(decl pabsw (RegMem) Reg) +(decl pabsw (XmmMem) Xmm) (rule (pabsw src) (xmm_unary_rm_r (SseOpcode.Pabsw) src)) ;; Helper for creating `pabsd` instructions. -(decl pabsd (RegMem) Reg) +(decl pabsd (XmmMem) Xmm) (rule (pabsd src) (xmm_unary_rm_r (SseOpcode.Pabsd) src)) ;; Helper for creating `MInst.XmmUnaryRmREvex` instructions. -(decl xmm_unary_rm_r_evex (Avx512Opcode RegMem) Reg) +(decl xmm_unary_rm_r_evex (Avx512Opcode XmmMem) Xmm) (rule (xmm_unary_rm_r_evex op src) - (let ((dst WritableReg (temp_writable_reg $I8X16)) + (let ((dst WritableXmm (temp_writable_xmm)) (_ Unit (emit (MInst.XmmUnaryRmREvex op src dst)))) - (writable_reg_to_reg dst))) + (writable_xmm_to_xmm dst))) ;; Helper for creating `vpabsq` instructions. -(decl vpabsq (RegMem) Reg) +(decl vpabsq (XmmMem) Xmm) (rule (vpabsq src) (xmm_unary_rm_r_evex (Avx512Opcode.Vpabsq) src)) ;; Helper for creating `MInst.XmmRmREvex` instructions. -(decl xmm_rm_r_evex (Avx512Opcode RegMem Reg) Reg) +(decl xmm_rm_r_evex (Avx512Opcode XmmMem Xmm) Xmm) (rule (xmm_rm_r_evex op src1 src2) - (let ((dst WritableReg (temp_writable_reg $I8X16)) + (let ((dst WritableXmm (temp_writable_xmm)) (_ Unit (emit (MInst.XmmRmREvex op src1 src2 dst)))) - (writable_reg_to_reg dst))) + (writable_xmm_to_xmm dst))) ;; Helper for creating `vpmullq` instructions. ;; ;; Requires AVX-512 vl and dq. -(decl vpmullq (RegMem Reg) Reg) +(decl vpmullq (XmmMem Xmm) Xmm) (rule (vpmullq src1 src2) (xmm_rm_r_evex (Avx512Opcode.Vpmullq) src1 src2)) -;; Helper for creating `MInst.XmmRmiReg` instructions. -(decl xmm_rmi_reg (SseOpcode Reg RegMemImm) Reg) -(rule (xmm_rmi_reg op src1 src2) - (let ((dst WritableReg (temp_writable_reg $I8X16)) +;; Helper for creating `MInst.MulHi` instructions. +;; +;; Returns the (lo, hi) register halves of the multiplication. +(decl mul_hi (Type bool Reg RegMem) ValueRegs) +(rule (mul_hi ty signed src1 src2) + (let ((dst_lo WritableReg (temp_writable_reg ty)) + (dst_hi WritableReg (temp_writable_reg ty)) + (size OperandSize (operand_size_of_type_32_64 ty)) + (_ Unit (emit (MInst.MulHi size + signed + src1 + src2 + dst_lo + dst_hi)))) + (value_regs (writable_reg_to_reg dst_lo) + (writable_reg_to_reg dst_hi)))) + +;; Helper for creating `mul` instructions that return both the lower and +;; (unsigned) higher halves of the result. +(decl mulhi_u (Type Reg RegMem) ValueRegs) +(rule (mulhi_u ty src1 src2) + (mul_hi ty $false src1 src2)) + +;; Helper for creating `MInst.XmmRmiXmm` instructions. +(decl xmm_rmi_xmm (SseOpcode Xmm XmmMemImm) Xmm) +(rule (xmm_rmi_xmm op src1 src2) + (let ((dst WritableXmm (temp_writable_xmm)) (_ Unit (emit (MInst.XmmRmiReg op src1 src2 dst)))) - (writable_reg_to_reg dst))) + (writable_xmm_to_xmm dst))) ;; Helper for creating `psllw` instructions. -(decl psllw (Reg RegMemImm) Reg) +(decl psllw (Xmm XmmMemImm) Xmm) (rule (psllw src1 src2) - (xmm_rmi_reg (SseOpcode.Psllw) src1 src2)) + (xmm_rmi_xmm (SseOpcode.Psllw) src1 src2)) ;; Helper for creating `pslld` instructions. -(decl pslld (Reg RegMemImm) Reg) +(decl pslld (Xmm XmmMemImm) Xmm) (rule (pslld src1 src2) - (xmm_rmi_reg (SseOpcode.Pslld) src1 src2)) + (xmm_rmi_xmm (SseOpcode.Pslld) src1 src2)) ;; Helper for creating `psllq` instructions. -(decl psllq (Reg RegMemImm) Reg) +(decl psllq (Xmm XmmMemImm) Xmm) (rule (psllq src1 src2) - (xmm_rmi_reg (SseOpcode.Psllq) src1 src2)) + (xmm_rmi_xmm (SseOpcode.Psllq) src1 src2)) ;; Helper for creating `psrlw` instructions. -(decl psrlw (Reg RegMemImm) Reg) +(decl psrlw (Xmm XmmMemImm) Xmm) (rule (psrlw src1 src2) - (xmm_rmi_reg (SseOpcode.Psrlw) src1 src2)) + (xmm_rmi_xmm (SseOpcode.Psrlw) src1 src2)) ;; Helper for creating `psrld` instructions. -(decl psrld (Reg RegMemImm) Reg) +(decl psrld (Xmm XmmMemImm) Xmm) (rule (psrld src1 src2) - (xmm_rmi_reg (SseOpcode.Psrld) src1 src2)) + (xmm_rmi_xmm (SseOpcode.Psrld) src1 src2)) ;; Helper for creating `psrlq` instructions. -(decl psrlq (Reg RegMemImm) Reg) +(decl psrlq (Xmm XmmMemImm) Xmm) (rule (psrlq src1 src2) - (xmm_rmi_reg (SseOpcode.Psrlq) src1 src2)) + (xmm_rmi_xmm (SseOpcode.Psrlq) src1 src2)) ;; Helper for creating `psraw` instructions. -(decl psraw (Reg RegMemImm) Reg) +(decl psraw (Xmm XmmMemImm) Xmm) (rule (psraw src1 src2) - (xmm_rmi_reg (SseOpcode.Psraw) src1 src2)) + (xmm_rmi_xmm (SseOpcode.Psraw) src1 src2)) ;; Helper for creating `psrad` instructions. -(decl psrad (Reg RegMemImm) Reg) +(decl psrad (Xmm XmmMemImm) Xmm) (rule (psrad src1 src2) - (xmm_rmi_reg (SseOpcode.Psrad) src1 src2)) - -;; Helper for creating `MInst.MulHi` instructions. -;; -;; Returns the (lo, hi) register halves of the multiplication. -(decl mul_hi (Type bool Reg RegMem) ValueRegs) -(rule (mul_hi ty signed src1 src2) - (let ((dst_lo WritableReg (temp_writable_reg ty)) - (dst_hi WritableReg (temp_writable_reg ty)) - (size OperandSize (operand_size_of_type_32_64 ty)) - (_ Unit (emit (MInst.MulHi size - signed - src1 - src2 - dst_lo - dst_hi)))) - (value_regs (writable_reg_to_reg dst_lo) - (writable_reg_to_reg dst_hi)))) + (xmm_rmi_xmm (SseOpcode.Psrad) src1 src2)) -;; Helper for creating `mul` instructions that return both the lower and -;; (unsigned) higher halves of the result. -(decl mulhi_u (Type Reg RegMem) ValueRegs) -(rule (mulhi_u ty src1 src2) - (mul_hi ty $false src1 src2)) - -;; Helper for creating `cmpps` instructions. -(decl cmpps (Reg RegMem FcmpImm) Reg) -(rule (cmpps src1 src2 imm) - (xmm_rm_r_imm (SseOpcode.Cmpps) - src1 - src2 - (encode_fcmp_imm imm) - (OperandSize.Size32))) +;; Helper for creating `pextrd` instructions. +(decl pextrd (Type Xmm u8) Gpr) +(rule (pextrd ty src lane) + (let ((w_dst WritableGpr (temp_writable_gpr)) + (r_dst Gpr (writable_gpr_to_gpr w_dst)) + (_ Unit (emit (MInst.XmmRmRImm (SseOpcode.Pextrd) + (gpr_to_reg r_dst) + (RegMem.Reg (xmm_to_reg src)) + (writable_gpr_to_reg w_dst) + lane + (operand_size_of_type_32_64 (lane_type ty)))))) + r_dst)) ;; Helper for creating `cmppd` instructions. ;; ;; Note that `Size32` is intentional despite this being used for 64-bit ;; operations, since this presumably induces the correct encoding of the ;; instruction. -(decl cmppd (Reg RegMem FcmpImm) Reg) +(decl cmppd (Xmm XmmMem FcmpImm) Xmm) (rule (cmppd src1 src2 imm) (xmm_rm_r_imm (SseOpcode.Cmppd) - src1 - src2 + (xmm_to_reg src1) + (xmm_mem_to_reg_mem src2) (encode_fcmp_imm imm) (OperandSize.Size32))) ;; Helper for creating `MInst.GprToXmm` instructions. -(decl gpr_to_xmm (Type SseOpcode RegMem OperandSize) Reg) -(rule (gpr_to_xmm ty op src size) - (let ((dst WritableReg (temp_writable_reg ty)) - (_ Unit (emit (MInst.GprToXmm op src dst size)))) - (writable_reg_to_reg dst))) - -;; Helper for creating `pinsrb` instructions. -(decl pinsrb (Reg RegMem u8) Reg) -(rule (pinsrb src1 src2 lane) - (xmm_rm_r_imm (SseOpcode.Pinsrb) src1 src2 lane (OperandSize.Size32))) - -;; Helper for creating `pinsrw` instructions. -(decl pinsrw (Reg RegMem u8) Reg) -(rule (pinsrw src1 src2 lane) - (xmm_rm_r_imm (SseOpcode.Pinsrw) src1 src2 lane (OperandSize.Size32))) - -;; Helper for creating `pinsrd` instructions. -(decl pinsrd (Reg RegMem u8 OperandSize) Reg) -(rule (pinsrd src1 src2 lane size) - (xmm_rm_r_imm (SseOpcode.Pinsrd) src1 src2 lane size)) - -;; Helper for creating `insertps` instructions. -(decl insertps (Reg RegMem u8) Reg) -(rule (insertps src1 src2 lane) - (xmm_rm_r_imm (SseOpcode.Insertps) src1 src2 lane (OperandSize.Size32))) - -;; Helper for creating `pextrd` instructions. -(decl pextrd (Type Reg u8) Reg) -(rule (pextrd ty src lane) - (let ((w_dst WritableReg (temp_writable_reg ty)) - (r_dst Reg (writable_reg_to_reg w_dst)) - (_ Unit (emit (MInst.XmmRmRImm (SseOpcode.Pextrd) - r_dst - (RegMem.Reg src) - w_dst - lane - (operand_size_of_type_32_64 (lane_type ty)))))) - r_dst)) +(decl gpr_to_xmm (SseOpcode GprMem OperandSize) Xmm) +(rule (gpr_to_xmm op src size) + (let ((dst WritableXmm (temp_writable_xmm)) + (_ Unit (emit (MInst.GprToXmm op (gpr_mem_to_reg_mem src) dst size)))) + (writable_xmm_to_xmm dst))) ;; Helper for creating `not` instructions. -(decl not (Type Reg) Reg) +(decl not (Type Gpr) Gpr) (rule (not ty src) - (let ((dst WritableReg (temp_writable_reg ty)) + (let ((dst WritableGpr (temp_writable_gpr)) (size OperandSize (operand_size_of_type_32_64 ty)) (_ Unit (emit (MInst.Not size src dst)))) - (writable_reg_to_reg dst))) + (writable_gpr_to_gpr dst))) ;; Helper for creating `neg` instructions. -(decl neg (Type Reg) Reg) +(decl neg (Type Gpr) Gpr) (rule (neg ty src) - (let ((dst WritableReg (temp_writable_reg ty)) + (let ((dst WritableGpr (temp_writable_gpr)) (size OperandSize (operand_size_of_type_32_64 ty)) (_ Unit (emit (MInst.Neg size src dst)))) - (writable_reg_to_reg dst))) + (writable_gpr_to_gpr dst))) -(decl lea (SyntheticAmode) Reg) +(decl lea (SyntheticAmode) Gpr) (rule (lea addr) - (let ((dst WritableReg (temp_writable_reg $I64)) + (let ((dst WritableGpr (temp_writable_gpr)) (_ Unit (emit (MInst.LoadEffectiveAddress addr dst)))) - (writable_reg_to_reg dst))) + (writable_gpr_to_gpr dst))) ;; Helper for creating `ud2` instructions. (decl ud2 (TrapCode) SideEffectNoResult) diff --git a/cranelift/codegen/src/isa/x64/inst/args.rs b/cranelift/codegen/src/isa/x64/inst/args.rs index 9b9cf2fd0ca9..e27dc3130ec6 100644 --- a/cranelift/codegen/src/isa/x64/inst/args.rs +++ b/cranelift/codegen/src/isa/x64/inst/args.rs @@ -13,6 +13,309 @@ use smallvec::{smallvec, SmallVec}; use std::fmt; use std::string::String; +/// An extenstion trait for converting `Writable{Xmm,Gpr}` to `Writable`. +pub trait ToWritableReg { + fn to_writable_reg(&self) -> Writable; +} + +/// An extension trait for converting `Writable` to `Writable{Xmm,Gpr}`. +pub trait FromWritableReg: Sized { + fn from_writable_reg(w: Writable) -> Option; +} + +/// An extension trait for mapping register uses on `{Xmm,Gpr}`. +pub trait MapUseExt { + fn map_use(&mut self, mapper: &RM) + where + RM: RegMapper; +} + +/// An extension trait for mapping register mods and defs on +/// `Writable{Xmm,Gpr}`. +pub trait MapDefModExt { + fn map_def(&mut self, mapper: &RM) + where + RM: RegMapper; + + fn map_mod(&mut self, mapper: &RM) + where + RM: RegMapper; +} + +/// A macro for defining a newtype of `Reg` that enforces some invariant about +/// the wrapped `Reg` (such as that it is of a particular register class). +macro_rules! newtype_of_reg { + ( + $newtype_reg:ident, + $newtype_writable_reg:ident, + $newtype_reg_mem:ident, + $newtype_reg_mem_imm:ident, + |$check_reg:ident| $check:expr + ) => { + /// A newtype wrapper around `Reg`. + #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] + pub struct $newtype_reg(Reg); + + impl PartialEq for $newtype_reg { + fn eq(&self, other: &Reg) -> bool { + self.0 == *other + } + } + + impl From<$newtype_reg> for Reg { + fn from(r: $newtype_reg) -> Self { + r.0 + } + } + + impl PrettyPrint for $newtype_reg { + fn show_rru(&self, mb_rru: Option<&RealRegUniverse>) -> String { + self.0.show_rru(mb_rru) + } + } + + impl $newtype_reg { + /// Create this newtype from the given register, or return `None` if the register + /// is not a valid instance of this newtype. + pub fn new($check_reg: Reg) -> Option { + if $check { + Some(Self($check_reg)) + } else { + None + } + } + + /// Get this newtype's underlying `Reg`. + pub fn to_reg(self) -> Reg { + self.0 + } + } + + // Convenience impl so that people working with this newtype can use it + // "just like" a plain `Reg`. + // + // NB: We cannot implement `DerefMut` because that would let people do + // nasty stuff like `*my_gpr.deref_mut() = some_xmm_reg`, breaking the + // invariants that `Gpr` provides. + impl std::ops::Deref for $newtype_reg { + type Target = Reg; + + fn deref(&self) -> &Reg { + &self.0 + } + } + + impl MapUseExt for $newtype_reg { + fn map_use(&mut self, mapper: &RM) + where + RM: RegMapper, + { + let mut reg = self.0; + mapper.map_use(&mut reg); + debug_assert!({ + let $check_reg = reg; + $check + }); + *self = $newtype_reg(reg); + } + } + + pub type $newtype_writable_reg = Writable<$newtype_reg>; + + impl ToWritableReg for $newtype_writable_reg { + fn to_writable_reg(&self) -> Writable { + Writable::from_reg(self.to_reg().to_reg()) + } + } + + impl FromWritableReg for $newtype_writable_reg { + fn from_writable_reg(w: Writable) -> Option { + Some(Writable::from_reg($newtype_reg::new(w.to_reg())?)) + } + } + + impl MapDefModExt for $newtype_writable_reg { + fn map_def(&mut self, mapper: &RM) + where + RM: RegMapper, + { + let mut reg = self.to_writable_reg(); + mapper.map_def(&mut reg); + debug_assert!({ + let $check_reg = reg.to_reg(); + $check + }); + *self = Writable::from_reg($newtype_reg(reg.to_reg())); + } + + fn map_mod(&mut self, mapper: &RM) + where + RM: RegMapper, + { + let mut reg = self.to_writable_reg(); + mapper.map_mod(&mut reg); + debug_assert!({ + let $check_reg = reg.to_reg(); + $check + }); + *self = Writable::from_reg($newtype_reg(reg.to_reg())); + } + } + + /// A newtype wrapper around `RegMem` for general-purpose registers. + #[derive(Clone, Debug)] + pub struct $newtype_reg_mem(RegMem); + + impl From<$newtype_reg_mem> for RegMem { + fn from(rm: $newtype_reg_mem) -> Self { + rm.0 + } + } + + impl From<$newtype_reg> for $newtype_reg_mem { + fn from(r: $newtype_reg) -> Self { + $newtype_reg_mem(RegMem::reg(r.into())) + } + } + + impl $newtype_reg_mem { + /// Construct a `RegMem` newtype from the given `RegMem`, or return + /// `None` if the `RegMem` is not a valid instance of this `RegMem` + /// newtype. + pub fn new(rm: RegMem) -> Option { + match rm { + RegMem::Mem { addr: _ } => Some(Self(rm)), + RegMem::Reg { reg: $check_reg } if $check => Some(Self(rm)), + RegMem::Reg { reg: _ } => None, + } + } + + /// Convert this newtype into its underlying `RegMem`. + pub fn to_reg_mem(self) -> RegMem { + self.0 + } + + #[allow(dead_code)] // Used by some newtypes and not others. + pub fn map_uses(&mut self, mapper: &RM) + where + RM: RegMapper, + { + self.0.map_uses(mapper); + debug_assert!(match self.0 { + RegMem::Reg { reg: $check_reg } => $check, + _ => true, + }); + } + + #[allow(dead_code)] // Used by some newtypes and not others. + pub fn map_as_def(&mut self, mapper: &RM) + where + RM: RegMapper, + { + self.0.map_as_def(mapper); + debug_assert!(match self.0 { + RegMem::Reg { reg: $check_reg } => $check, + _ => true, + }); + } + } + + impl PrettyPrint for $newtype_reg_mem { + fn show_rru(&self, mb_rru: Option<&RealRegUniverse>) -> String { + self.0.show_rru(mb_rru) + } + } + + impl PrettyPrintSized for $newtype_reg_mem { + fn show_rru_sized(&self, mb_rru: Option<&RealRegUniverse>, size: u8) -> String { + self.0.show_rru_sized(mb_rru, size) + } + } + + /// A newtype wrapper around `RegMemImm`. + #[derive(Clone, Debug)] + pub struct $newtype_reg_mem_imm(RegMemImm); + + impl From<$newtype_reg_mem_imm> for RegMemImm { + fn from(rmi: $newtype_reg_mem_imm) -> RegMemImm { + rmi.0 + } + } + + impl From<$newtype_reg> for $newtype_reg_mem_imm { + fn from(r: $newtype_reg) -> Self { + $newtype_reg_mem_imm(RegMemImm::reg(r.into())) + } + } + + impl $newtype_reg_mem_imm { + /// Construct this newtype from the given `RegMemImm`, or return + /// `None` if the `RegMemImm` is not a valid instance of this + /// newtype. + pub fn new(rmi: RegMemImm) -> Option { + match rmi { + RegMemImm::Imm { .. } => Some(Self(rmi)), + RegMemImm::Mem { addr: _ } => Some(Self(rmi)), + RegMemImm::Reg { reg: $check_reg } if $check => Some(Self(rmi)), + RegMemImm::Reg { reg: _ } => None, + } + } + + /// Convert this newtype into its underlying `RegMemImm`. + #[allow(dead_code)] // Used by some newtypes and not others. + pub fn to_reg_mem_imm(self) -> RegMemImm { + self.0 + } + + #[allow(dead_code)] // Used by some newtypes and not others. + pub fn map_uses(&mut self, mapper: &RM) + where + RM: RegMapper, + { + self.0.map_uses(mapper); + debug_assert!(match self.0 { + RegMemImm::Reg { reg: $check_reg } => $check, + _ => true, + }); + } + + #[allow(dead_code)] // Used by some newtypes and not others. + pub fn map_as_def(&mut self, mapper: &RM) + where + RM: RegMapper, + { + self.0.map_as_def(mapper); + debug_assert!(match self.0 { + RegMemImm::Reg { reg: $check_reg } => $check, + _ => true, + }); + } + } + + impl PrettyPrint for $newtype_reg_mem_imm { + fn show_rru(&self, mb_rru: Option<&RealRegUniverse>) -> String { + self.0.show_rru(mb_rru) + } + } + + impl PrettyPrintSized for $newtype_reg_mem_imm { + fn show_rru_sized(&self, mb_rru: Option<&RealRegUniverse>, size: u8) -> String { + self.0.show_rru_sized(mb_rru, size) + } + } + }; +} + +// Define a newtype of `Reg` for general-purpose registers. +newtype_of_reg!(Gpr, WritableGpr, GprMem, GprMemImm, |reg| { + reg.get_class() == RegClass::I64 +}); + +// Define a newtype of `Reg` for XMM registers. +newtype_of_reg!(Xmm, WritableXmm, XmmMem, XmmMemImm, |reg| { + reg.get_class() == RegClass::V128 +}); + /// A possible addressing mode (amode) that can be used in instructions. /// These denote a 64-bit value only. #[derive(Clone, Debug)] @@ -27,8 +330,8 @@ pub enum Amode { /// sign-extend-32-to-64(Immediate) + Register1 + (Register2 << Shift) ImmRegRegShift { simm32: u32, - base: Reg, - index: Reg, + base: Gpr, + index: Gpr, shift: u8, /* 0 .. 3 only */ flags: MemFlags, }, @@ -48,7 +351,7 @@ impl Amode { } } - pub(crate) fn imm_reg_reg_shift(simm32: u32, base: Reg, index: Reg, shift: u8) -> Self { + pub(crate) fn imm_reg_reg_shift(simm32: u32, base: Gpr, index: Gpr, shift: u8) -> Self { debug_assert!(base.get_class() == RegClass::I64); debug_assert!(index.get_class() == RegClass::I64); debug_assert!(shift <= 3); @@ -96,8 +399,8 @@ impl Amode { collector.add_use(*base); } Amode::ImmRegRegShift { base, index, .. } => { - collector.add_use(*base); - collector.add_use(*index); + collector.add_use(base.to_reg()); + collector.add_use(index.to_reg()); } Amode::RipRelative { .. } => { // RIP isn't involved in regalloc. @@ -225,7 +528,7 @@ impl PrettyPrint for SyntheticAmode { /// denote an 8, 16, 32 or 64 bit value. For the Immediate form, in the 8- and 16-bit case, only /// the lower 8 or 16 bits of `simm32` is relevant. In the 64-bit case, the value denoted by /// `simm32` is its sign-extension out to 64 bits. -#[derive(Clone)] +#[derive(Clone, Debug)] pub enum RegMemImm { Reg { reg: Reg }, Mem { addr: SyntheticAmode }, diff --git a/cranelift/codegen/src/isa/x64/inst/emit.rs b/cranelift/codegen/src/isa/x64/inst/emit.rs index c4dc3f3d7a5d..bda2ddc0a241 100644 --- a/cranelift/codegen/src/isa/x64/inst/emit.rs +++ b/cranelift/codegen/src/isa/x64/inst/emit.rs @@ -298,14 +298,14 @@ pub(crate) fn emit( Popcnt => (0x0fb8, 2), }; - match src { + match src.clone().into() { RegMem::Reg { reg: src } => emit_std_reg_reg( sink, prefix, opcode, num_opcodes, - dst.to_reg(), - *src, + dst.to_reg().to_reg(), + src, rex_flags, ), RegMem::Mem { addr: src } => { @@ -317,7 +317,7 @@ pub(crate) fn emit( prefix, opcode, num_opcodes, - dst.to_reg(), + dst.to_reg().to_reg(), &amode, rex_flags, ); @@ -327,7 +327,7 @@ pub(crate) fn emit( Inst::Not { size, src, dst } => { debug_assert_eq!(*src, dst.to_reg()); - let rex_flags = RexFlags::from((*size, dst.to_reg())); + let rex_flags = RexFlags::from((*size, dst.to_writable_reg().to_reg())); let (opcode, prefix) = match size { OperandSize::Size8 => (0xF6, LegacyPrefixes::None), OperandSize::Size16 => (0xF7, LegacyPrefixes::_66), @@ -342,7 +342,7 @@ pub(crate) fn emit( Inst::Neg { size, src, dst } => { debug_assert_eq!(*src, dst.to_reg()); - let rex_flags = RexFlags::from((*size, dst.to_reg())); + let rex_flags = RexFlags::from((*size, dst.to_writable_reg().to_reg())); let (opcode, prefix) = match size { OperandSize::Size8 => (0xF6, LegacyPrefixes::None), OperandSize::Size16 => (0xF7, LegacyPrefixes::_66), @@ -728,7 +728,7 @@ pub(crate) fn emit( LegacyPrefixes::None, 0x8D, 1, - dst.to_reg(), + dst.to_reg().to_reg(), &amode, RexFlags::set_w(), ); @@ -884,6 +884,7 @@ pub(crate) fn emit( debug_assert_eq!(*src1, dst.to_reg()); let rex = RexFlags::clear_w(); let prefix = LegacyPrefixes::_66; + let src2 = src2.clone().to_reg_mem_imm(); if let RegMemImm::Imm { simm32 } = src2 { let (opcode_bytes, reg_digit) = match opcode { SseOpcode::Psllw => (0x0F71, 6), @@ -898,7 +899,7 @@ pub(crate) fn emit( }; let dst_enc = reg_enc(dst.to_reg()); emit_std_enc_enc(sink, prefix, opcode_bytes, 2, reg_digit, dst_enc, rex); - let imm = (*simm32) + let imm = (simm32) .try_into() .expect("the immediate must be convertible to a u8"); sink.put1(imm); @@ -917,7 +918,15 @@ pub(crate) fn emit( match src2 { RegMemImm::Reg { reg } => { - emit_std_reg_reg(sink, prefix, opcode_bytes, 2, dst.to_reg(), *reg, rex); + emit_std_reg_reg( + sink, + prefix, + opcode_bytes, + 2, + dst.to_reg().to_reg(), + reg, + rex, + ); } RegMemImm::Mem { addr } => { let addr = &addr.finalize(state, sink); @@ -928,7 +937,7 @@ pub(crate) fn emit( prefix, opcode_bytes, 2, - dst.to_reg(), + dst.to_reg().to_reg(), addr, rex, ); @@ -1335,7 +1344,12 @@ pub(crate) fn emit( // might be negative; use a sign-extension. let inst = Inst::movsx_rm_r( ExtMode::LQ, - RegMem::mem(Amode::imm_reg_reg_shift(0, tmp1.to_reg(), tmp2.to_reg(), 2)), + RegMem::mem(Amode::imm_reg_reg_shift( + 0, + Gpr::new(tmp1.to_reg()).unwrap(), + Gpr::new(tmp2.to_reg()).unwrap(), + 2, + )), *tmp2, ); inst.emit(sink, info, state); @@ -1424,15 +1438,15 @@ pub(crate) fn emit( _ => unimplemented!("Opcode {:?} not implemented", op), }; - match src_e { + match src_e.clone().to_reg_mem() { RegMem::Reg { reg: reg_e } => { emit_std_reg_reg( sink, prefix, opcode, num_opcodes, - reg_g.to_reg(), - *reg_e, + reg_g.to_reg().to_reg(), + reg_e, rex, ); } @@ -1445,7 +1459,7 @@ pub(crate) fn emit( prefix, opcode, num_opcodes, - reg_g.to_reg(), + reg_g.to_reg().to_reg(), addr, rex, ); @@ -1460,7 +1474,7 @@ pub(crate) fn emit( Avx512Opcode::Vpopcntb => (LegacyPrefixes::_66, OpcodeMap::_0F38, false, 0x54), _ => unimplemented!("Opcode {:?} not implemented", op), }; - match src { + match src.clone().to_reg_mem() { RegMem::Reg { reg: src } => EvexInstruction::new() .length(EvexVectorLength::V128) .prefix(prefix) @@ -1587,9 +1601,17 @@ pub(crate) fn emit( _ => unimplemented!("Opcode {:?} not implemented", op), }; - match src_e { + match src_e.clone().to_reg_mem() { RegMem::Reg { reg: reg_e } => { - emit_std_reg_reg(sink, prefix, opcode, length, reg_g.to_reg(), *reg_e, rex); + emit_std_reg_reg( + sink, + prefix, + opcode, + length, + reg_g.to_reg().to_reg(), + reg_e, + rex, + ); } RegMem::Mem { addr } => { let addr = &addr.finalize(state, sink); @@ -1600,7 +1622,7 @@ pub(crate) fn emit( prefix, opcode, length, - reg_g.to_reg(), + reg_g.to_reg().to_reg(), addr, rex, ); @@ -1619,7 +1641,7 @@ pub(crate) fn emit( Avx512Opcode::Vpmullq => (true, 0x40), _ => unimplemented!("Opcode {:?} not implemented", op), }; - match src1 { + match src1.clone().to_reg_mem() { RegMem::Reg { reg: src } => EvexInstruction::new() .length(EvexVectorLength::V128) .prefix(LegacyPrefixes::_66) @@ -1845,9 +1867,9 @@ pub(crate) fn emit( }; let rex = RexFlags::from(*dst_size); let (src, dst) = if dst_first { - (dst.to_reg(), *src) + (dst.to_reg().to_reg(), src.to_reg()) } else { - (*src, dst.to_reg()) + (src.to_reg(), dst.to_reg().to_reg()) }; emit_std_reg_reg(sink, prefix, opcode, 2, src, dst, rex); @@ -1870,7 +1892,15 @@ pub(crate) fn emit( let rex = RexFlags::from(*src_size); match src_e { RegMem::Reg { reg: reg_e } => { - emit_std_reg_reg(sink, prefix, opcode, 2, reg_g.to_reg(), *reg_e, rex); + emit_std_reg_reg( + sink, + prefix, + opcode, + 2, + reg_g.to_reg().to_reg(), + *reg_e, + rex, + ); } RegMem::Mem { addr } => { let addr = &addr.finalize(state, sink); @@ -1881,7 +1911,7 @@ pub(crate) fn emit( prefix, opcode, 2, - reg_g.to_reg(), + reg_g.to_reg().to_reg(), addr, rex, ); @@ -1950,7 +1980,11 @@ pub(crate) fn emit( // If x seen as a signed int64 is not negative, a signed-conversion will do the right // thing. // TODO use tst src, src here. - let inst = Inst::cmp_rmi_r(OperandSize::Size64, RegMemImm::imm(0), src.to_reg()); + let inst = Inst::cmp_rmi_r( + OperandSize::Size64, + RegMemImm::imm(0), + src.to_reg().to_reg(), + ); inst.emit(sink, info, state); one_way_jmp(sink, CC::L, handle_negative); @@ -1961,8 +1995,8 @@ pub(crate) fn emit( sink, info, state, - src.to_reg(), - *dst, + src.to_reg().to_reg(), + dst.to_writable_reg(), *dst_size == OperandSize::Size64, ); @@ -1973,7 +2007,11 @@ pub(crate) fn emit( // Divide x by two to get it in range for the signed conversion, keep the LSB, and // scale it back up on the FP side. - let inst = Inst::gen_move(*tmp_gpr1, src.to_reg(), types::I64); + let inst = Inst::gen_move( + tmp_gpr1.to_writable_reg(), + src.to_reg().to_reg(), + types::I64, + ); inst.emit(sink, info, state); // tmp_gpr1 := src >> 1 @@ -1981,26 +2019,30 @@ pub(crate) fn emit( OperandSize::Size64, ShiftKind::ShiftRightLogical, Some(1), - *tmp_gpr1, + tmp_gpr1.to_writable_reg(), ); inst.emit(sink, info, state); - let inst = Inst::gen_move(*tmp_gpr2, src.to_reg(), types::I64); + let inst = Inst::gen_move( + tmp_gpr2.to_writable_reg(), + src.to_reg().to_reg(), + types::I64, + ); inst.emit(sink, info, state); let inst = Inst::alu_rmi_r( OperandSize::Size64, AluRmiROpcode::And, RegMemImm::imm(1), - *tmp_gpr2, + tmp_gpr2.to_writable_reg(), ); inst.emit(sink, info, state); let inst = Inst::alu_rmi_r( OperandSize::Size64, AluRmiROpcode::Or, - RegMemImm::reg(tmp_gpr1.to_reg()), - *tmp_gpr2, + RegMemImm::reg(tmp_gpr1.to_reg().to_reg()), + tmp_gpr2.to_writable_reg(), ); inst.emit(sink, info, state); @@ -2008,8 +2050,8 @@ pub(crate) fn emit( sink, info, state, - tmp_gpr2.to_reg(), - *dst, + tmp_gpr2.to_reg().to_reg(), + dst.to_writable_reg(), *dst_size == OperandSize::Size64, ); @@ -2018,7 +2060,11 @@ pub(crate) fn emit( } else { SseOpcode::Addss }; - let inst = Inst::xmm_rm_r(add_op, RegMem::reg(dst.to_reg()), *dst); + let inst = Inst::xmm_rm_r( + add_op, + RegMem::reg(dst.to_reg().to_reg()), + dst.to_writable_reg(), + ); inst.emit(sink, info, state); sink.bind_label(done); @@ -2091,18 +2137,18 @@ pub(crate) fn emit( let not_nan = sink.get_label(); // The truncation. - let inst = Inst::xmm_to_gpr(trunc_op, src, *dst, *dst_size); + let inst = Inst::xmm_to_gpr(trunc_op, src.to_reg(), dst.to_writable_reg(), *dst_size); inst.emit(sink, info, state); // Compare against 1, in case of overflow the dst operand was INT_MIN. - let inst = Inst::cmp_rmi_r(*dst_size, RegMemImm::imm(1), dst.to_reg()); + let inst = Inst::cmp_rmi_r(*dst_size, RegMemImm::imm(1), dst.to_reg().to_reg()); inst.emit(sink, info, state); one_way_jmp(sink, CC::NO, done); // no overflow => done // Check for NaN. - let inst = Inst::xmm_cmp_rm_r(cmp_op, RegMem::reg(src), src); + let inst = Inst::xmm_cmp_rm_r(cmp_op, RegMem::reg(src.to_reg()), src.to_reg()); inst.emit(sink, info, state); one_way_jmp(sink, CC::NP, not_nan); // go to not_nan if not a NaN @@ -2112,8 +2158,8 @@ pub(crate) fn emit( let inst = Inst::alu_rmi_r( *dst_size, AluRmiROpcode::Xor, - RegMemImm::reg(dst.to_reg()), - *dst, + RegMemImm::reg(dst.to_reg().to_reg()), + dst.to_writable_reg(), ); inst.emit(sink, info, state); @@ -2125,11 +2171,18 @@ pub(crate) fn emit( // If the input was positive, saturate to INT_MAX. // Zero out tmp_xmm. - let inst = - Inst::xmm_rm_r(SseOpcode::Xorpd, RegMem::reg(tmp_xmm.to_reg()), *tmp_xmm); + let inst = Inst::xmm_rm_r( + SseOpcode::Xorpd, + RegMem::reg(tmp_xmm.to_reg().to_reg()), + tmp_xmm.to_writable_reg(), + ); inst.emit(sink, info, state); - let inst = Inst::xmm_cmp_rm_r(cmp_op, RegMem::reg(src), tmp_xmm.to_reg()); + let inst = Inst::xmm_cmp_rm_r( + cmp_op, + RegMem::reg(src.to_reg()), + tmp_xmm.to_reg().to_reg(), + ); inst.emit(sink, info, state); // Jump if >= to done. @@ -2137,10 +2190,14 @@ pub(crate) fn emit( // Otherwise, put INT_MAX. if *dst_size == OperandSize::Size64 { - let inst = Inst::imm(OperandSize::Size64, 0x7fffffffffffffff, *dst); + let inst = Inst::imm( + OperandSize::Size64, + 0x7fffffffffffffff, + dst.to_writable_reg(), + ); inst.emit(sink, info, state); } else { - let inst = Inst::imm(OperandSize::Size32, 0x7fffffff, *dst); + let inst = Inst::imm(OperandSize::Size32, 0x7fffffff, dst.to_writable_reg()); inst.emit(sink, info, state); } } else { @@ -2162,7 +2219,8 @@ pub(crate) fn emit( match *src_size { OperandSize::Size32 => { let cst = Ieee32::pow2(output_bits - 1).neg().bits(); - let inst = Inst::imm(OperandSize::Size32, cst as u64, *tmp_gpr); + let inst = + Inst::imm(OperandSize::Size32, cst as u64, tmp_gpr.to_writable_reg()); inst.emit(sink, info, state); } OperandSize::Size64 => { @@ -2174,17 +2232,26 @@ pub(crate) fn emit( } else { Ieee64::pow2(output_bits - 1).neg() }; - let inst = Inst::imm(OperandSize::Size64, cst.bits(), *tmp_gpr); + let inst = + Inst::imm(OperandSize::Size64, cst.bits(), tmp_gpr.to_writable_reg()); inst.emit(sink, info, state); } _ => unreachable!(), } - let inst = - Inst::gpr_to_xmm(cast_op, RegMem::reg(tmp_gpr.to_reg()), *src_size, *tmp_xmm); + let inst = Inst::gpr_to_xmm( + cast_op, + RegMem::reg(tmp_gpr.to_reg().to_reg()), + *src_size, + tmp_xmm.to_writable_reg(), + ); inst.emit(sink, info, state); - let inst = Inst::xmm_cmp_rm_r(cmp_op, RegMem::reg(tmp_xmm.to_reg()), src); + let inst = Inst::xmm_cmp_rm_r( + cmp_op, + RegMem::reg(tmp_xmm.to_reg().to_reg()), + src.to_reg(), + ); inst.emit(sink, info, state); // jump over trap if src >= or > threshold @@ -2198,11 +2265,18 @@ pub(crate) fn emit( sink.bind_label(check_positive); // Zero out the tmp_xmm register. - let inst = - Inst::xmm_rm_r(SseOpcode::Xorpd, RegMem::reg(tmp_xmm.to_reg()), *tmp_xmm); + let inst = Inst::xmm_rm_r( + SseOpcode::Xorpd, + RegMem::reg(tmp_xmm.to_reg().to_reg()), + tmp_xmm.to_writable_reg(), + ); inst.emit(sink, info, state); - let inst = Inst::xmm_cmp_rm_r(cmp_op, RegMem::reg(src), tmp_xmm.to_reg()); + let inst = Inst::xmm_cmp_rm_r( + cmp_op, + RegMem::reg(src.to_reg()), + tmp_xmm.to_reg().to_reg(), + ); inst.emit(sink, info, state); one_way_jmp(sink, CC::NB, done); // jump over trap if 0 >= src @@ -2282,14 +2356,22 @@ pub(crate) fn emit( _ => unreachable!(), }; - let inst = Inst::imm(*src_size, cst, *tmp_gpr); + let inst = Inst::imm(*src_size, cst, tmp_gpr.to_writable_reg()); inst.emit(sink, info, state); - let inst = - Inst::gpr_to_xmm(cast_op, RegMem::reg(tmp_gpr.to_reg()), *src_size, *tmp_xmm); + let inst = Inst::gpr_to_xmm( + cast_op, + RegMem::reg(tmp_gpr.to_reg().to_reg()), + *src_size, + tmp_xmm.to_writable_reg(), + ); inst.emit(sink, info, state); - let inst = Inst::xmm_cmp_rm_r(cmp_op, RegMem::reg(tmp_xmm.to_reg()), src.to_reg()); + let inst = Inst::xmm_cmp_rm_r( + cmp_op, + RegMem::reg(tmp_xmm.to_reg().to_reg()), + src.to_reg().to_reg(), + ); inst.emit(sink, info, state); let handle_large = sink.get_label(); @@ -2303,8 +2385,8 @@ pub(crate) fn emit( let inst = Inst::alu_rmi_r( *dst_size, AluRmiROpcode::Xor, - RegMemImm::reg(dst.to_reg()), - *dst, + RegMemImm::reg(dst.to_reg().to_reg()), + dst.to_writable_reg(), ); inst.emit(sink, info, state); @@ -2321,10 +2403,15 @@ pub(crate) fn emit( // Actual truncation for small inputs: if the result is not positive, then we had an // overflow. - let inst = Inst::xmm_to_gpr(trunc_op, src.to_reg(), *dst, *dst_size); + let inst = Inst::xmm_to_gpr( + trunc_op, + src.to_reg().to_reg(), + dst.to_writable_reg(), + *dst_size, + ); inst.emit(sink, info, state); - let inst = Inst::cmp_rmi_r(*dst_size, RegMemImm::imm(0), dst.to_reg()); + let inst = Inst::cmp_rmi_r(*dst_size, RegMemImm::imm(0), dst.to_reg().to_reg()); inst.emit(sink, info, state); one_way_jmp(sink, CC::NL, done); // if dst >= 0, jump to done @@ -2335,8 +2422,8 @@ pub(crate) fn emit( let inst = Inst::alu_rmi_r( *dst_size, AluRmiROpcode::Xor, - RegMemImm::reg(dst.to_reg()), - *dst, + RegMemImm::reg(dst.to_reg().to_reg()), + dst.to_writable_reg(), ); inst.emit(sink, info, state); @@ -2352,13 +2439,22 @@ pub(crate) fn emit( sink.bind_label(handle_large); - let inst = Inst::xmm_rm_r(sub_op, RegMem::reg(tmp_xmm.to_reg()), *src); + let inst = Inst::xmm_rm_r( + sub_op, + RegMem::reg(tmp_xmm.to_reg().to_reg()), + src.to_writable_reg(), + ); inst.emit(sink, info, state); - let inst = Inst::xmm_to_gpr(trunc_op, src.to_reg(), *dst, *dst_size); + let inst = Inst::xmm_to_gpr( + trunc_op, + src.to_reg().to_reg(), + dst.to_writable_reg(), + *dst_size, + ); inst.emit(sink, info, state); - let inst = Inst::cmp_rmi_r(*dst_size, RegMemImm::imm(0), dst.to_reg()); + let inst = Inst::cmp_rmi_r(*dst_size, RegMemImm::imm(0), dst.to_reg().to_reg()); inst.emit(sink, info, state); let next_is_large = sink.get_label(); @@ -2374,7 +2470,7 @@ pub(crate) fn emit( } else { u32::max_value() as u64 }, - *dst, + dst.to_writable_reg(), ); inst.emit(sink, info, state); @@ -2388,14 +2484,14 @@ pub(crate) fn emit( sink.bind_label(next_is_large); if *dst_size == OperandSize::Size64 { - let inst = Inst::imm(OperandSize::Size64, 1 << 63, *tmp_gpr); + let inst = Inst::imm(OperandSize::Size64, 1 << 63, tmp_gpr.to_writable_reg()); inst.emit(sink, info, state); let inst = Inst::alu_rmi_r( OperandSize::Size64, AluRmiROpcode::Add, - RegMemImm::reg(tmp_gpr.to_reg()), - *dst, + RegMemImm::reg(tmp_gpr.to_reg().to_reg()), + dst.to_writable_reg(), ); inst.emit(sink, info, state); } else { @@ -2403,7 +2499,7 @@ pub(crate) fn emit( OperandSize::Size32, AluRmiROpcode::Add, RegMemImm::imm(1 << 31), - *dst, + dst.to_writable_reg(), ); inst.emit(sink, info, state); } diff --git a/cranelift/codegen/src/isa/x64/inst/emit_tests.rs b/cranelift/codegen/src/isa/x64/inst/emit_tests.rs index 1c71187fdeb8..154992f0fbb3 100644 --- a/cranelift/codegen/src/isa/x64/inst/emit_tests.rs +++ b/cranelift/codegen/src/isa/x64/inst/emit_tests.rs @@ -14,6 +14,7 @@ use super::*; use crate::isa::x64; +use alloc::boxed::Box; use alloc::vec::Vec; impl Inst { @@ -21,8 +22,8 @@ impl Inst { debug_assert_eq!(src.to_reg().get_class(), RegClass::I64); Inst::Neg { size, - src: src.to_reg(), - dst: src, + src: Gpr::new(src.to_reg()).unwrap(), + dst: WritableGpr::from_writable_reg(src).unwrap(), } } } @@ -693,42 +694,66 @@ fn test_x64_emit() { // // Addr_IRRS, offset max simm8 insns.push(( - Inst::mov64_m_r(Amode::imm_reg_reg_shift(127, rax, rax, 0), w_r11), + Inst::mov64_m_r( + Amode::imm_reg_reg_shift(127, Gpr::new(rax).unwrap(), Gpr::new(rax).unwrap(), 0), + w_r11, + ), "4C8B5C007F", "movq 127(%rax,%rax,1), %r11", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg_reg_shift(127, rdi, rax, 1), w_r11), + Inst::mov64_m_r( + Amode::imm_reg_reg_shift(127, Gpr::new(rdi).unwrap(), Gpr::new(rax).unwrap(), 1), + w_r11, + ), "4C8B5C477F", "movq 127(%rdi,%rax,2), %r11", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg_reg_shift(127, r8, rax, 2), w_r11), + Inst::mov64_m_r( + Amode::imm_reg_reg_shift(127, Gpr::new(r8).unwrap(), Gpr::new(rax).unwrap(), 2), + w_r11, + ), "4D8B5C807F", "movq 127(%r8,%rax,4), %r11", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg_reg_shift(127, r15, rax, 3), w_r11), + Inst::mov64_m_r( + Amode::imm_reg_reg_shift(127, Gpr::new(r15).unwrap(), Gpr::new(rax).unwrap(), 3), + w_r11, + ), "4D8B5CC77F", "movq 127(%r15,%rax,8), %r11", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg_reg_shift(127, rax, rdi, 3), w_r11), + Inst::mov64_m_r( + Amode::imm_reg_reg_shift(127, Gpr::new(rax).unwrap(), Gpr::new(rdi).unwrap(), 3), + w_r11, + ), "4C8B5CF87F", "movq 127(%rax,%rdi,8), %r11", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg_reg_shift(127, rdi, rdi, 2), w_r11), + Inst::mov64_m_r( + Amode::imm_reg_reg_shift(127, Gpr::new(rdi).unwrap(), Gpr::new(rdi).unwrap(), 2), + w_r11, + ), "4C8B5CBF7F", "movq 127(%rdi,%rdi,4), %r11", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg_reg_shift(127, r8, rdi, 1), w_r11), + Inst::mov64_m_r( + Amode::imm_reg_reg_shift(127, Gpr::new(r8).unwrap(), Gpr::new(rdi).unwrap(), 1), + w_r11, + ), "4D8B5C787F", "movq 127(%r8,%rdi,2), %r11", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg_reg_shift(127, r15, rdi, 0), w_r11), + Inst::mov64_m_r( + Amode::imm_reg_reg_shift(127, Gpr::new(r15).unwrap(), Gpr::new(rdi).unwrap(), 0), + w_r11, + ), "4D8B5C3F7F", "movq 127(%r15,%rdi,1), %r11", )); @@ -736,42 +761,106 @@ fn test_x64_emit() { // ======================================================== // Addr_IRRS, offset min simm8 insns.push(( - Inst::mov64_m_r(Amode::imm_reg_reg_shift(-128i32 as u32, rax, r8, 2), w_r11), + Inst::mov64_m_r( + Amode::imm_reg_reg_shift( + -128i32 as u32, + Gpr::new(rax).unwrap(), + Gpr::new(r8).unwrap(), + 2, + ), + w_r11, + ), "4E8B5C8080", "movq -128(%rax,%r8,4), %r11", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg_reg_shift(-128i32 as u32, rdi, r8, 3), w_r11), + Inst::mov64_m_r( + Amode::imm_reg_reg_shift( + -128i32 as u32, + Gpr::new(rdi).unwrap(), + Gpr::new(r8).unwrap(), + 3, + ), + w_r11, + ), "4E8B5CC780", "movq -128(%rdi,%r8,8), %r11", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg_reg_shift(-128i32 as u32, r8, r8, 0), w_r11), + Inst::mov64_m_r( + Amode::imm_reg_reg_shift( + -128i32 as u32, + Gpr::new(r8).unwrap(), + Gpr::new(r8).unwrap(), + 0, + ), + w_r11, + ), "4F8B5C0080", "movq -128(%r8,%r8,1), %r11", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg_reg_shift(-128i32 as u32, r15, r8, 1), w_r11), + Inst::mov64_m_r( + Amode::imm_reg_reg_shift( + -128i32 as u32, + Gpr::new(r15).unwrap(), + Gpr::new(r8).unwrap(), + 1, + ), + w_r11, + ), "4F8B5C4780", "movq -128(%r15,%r8,2), %r11", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg_reg_shift(-128i32 as u32, rax, r15, 1), w_r11), + Inst::mov64_m_r( + Amode::imm_reg_reg_shift( + -128i32 as u32, + Gpr::new(rax).unwrap(), + Gpr::new(r15).unwrap(), + 1, + ), + w_r11, + ), "4E8B5C7880", "movq -128(%rax,%r15,2), %r11", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg_reg_shift(-128i32 as u32, rdi, r15, 0), w_r11), + Inst::mov64_m_r( + Amode::imm_reg_reg_shift( + -128i32 as u32, + Gpr::new(rdi).unwrap(), + Gpr::new(r15).unwrap(), + 0, + ), + w_r11, + ), "4E8B5C3F80", "movq -128(%rdi,%r15,1), %r11", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg_reg_shift(-128i32 as u32, r8, r15, 3), w_r11), + Inst::mov64_m_r( + Amode::imm_reg_reg_shift( + -128i32 as u32, + Gpr::new(r8).unwrap(), + Gpr::new(r15).unwrap(), + 3, + ), + w_r11, + ), "4F8B5CF880", "movq -128(%r8,%r15,8), %r11", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg_reg_shift(-128i32 as u32, r15, r15, 2), w_r11), + Inst::mov64_m_r( + Amode::imm_reg_reg_shift( + -128i32 as u32, + Gpr::new(r15).unwrap(), + Gpr::new(r15).unwrap(), + 2, + ), + w_r11, + ), "4F8B5CBF80", "movq -128(%r15,%r15,4), %r11", )); @@ -779,42 +868,96 @@ fn test_x64_emit() { // ======================================================== // Addr_IRRS, offset large positive simm32 insns.push(( - Inst::mov64_m_r(Amode::imm_reg_reg_shift(0x4f6625be, rax, rax, 0), w_r11), + Inst::mov64_m_r( + Amode::imm_reg_reg_shift( + 0x4f6625be, + Gpr::new(rax).unwrap(), + Gpr::new(rax).unwrap(), + 0, + ), + w_r11, + ), "4C8B9C00BE25664F", "movq 1332094398(%rax,%rax,1), %r11", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg_reg_shift(0x4f6625be, rdi, rax, 1), w_r11), + Inst::mov64_m_r( + Amode::imm_reg_reg_shift( + 0x4f6625be, + Gpr::new(rdi).unwrap(), + Gpr::new(rax).unwrap(), + 1, + ), + w_r11, + ), "4C8B9C47BE25664F", "movq 1332094398(%rdi,%rax,2), %r11", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg_reg_shift(0x4f6625be, r8, rax, 2), w_r11), + Inst::mov64_m_r( + Amode::imm_reg_reg_shift(0x4f6625be, Gpr::new(r8).unwrap(), Gpr::new(rax).unwrap(), 2), + w_r11, + ), "4D8B9C80BE25664F", "movq 1332094398(%r8,%rax,4), %r11", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg_reg_shift(0x4f6625be, r15, rax, 3), w_r11), + Inst::mov64_m_r( + Amode::imm_reg_reg_shift( + 0x4f6625be, + Gpr::new(r15).unwrap(), + Gpr::new(rax).unwrap(), + 3, + ), + w_r11, + ), "4D8B9CC7BE25664F", "movq 1332094398(%r15,%rax,8), %r11", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg_reg_shift(0x4f6625be, rax, rdi, 3), w_r11), + Inst::mov64_m_r( + Amode::imm_reg_reg_shift( + 0x4f6625be, + Gpr::new(rax).unwrap(), + Gpr::new(rdi).unwrap(), + 3, + ), + w_r11, + ), "4C8B9CF8BE25664F", "movq 1332094398(%rax,%rdi,8), %r11", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg_reg_shift(0x4f6625be, rdi, rdi, 2), w_r11), + Inst::mov64_m_r( + Amode::imm_reg_reg_shift( + 0x4f6625be, + Gpr::new(rdi).unwrap(), + Gpr::new(rdi).unwrap(), + 2, + ), + w_r11, + ), "4C8B9CBFBE25664F", "movq 1332094398(%rdi,%rdi,4), %r11", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg_reg_shift(0x4f6625be, r8, rdi, 1), w_r11), + Inst::mov64_m_r( + Amode::imm_reg_reg_shift(0x4f6625be, Gpr::new(r8).unwrap(), Gpr::new(rdi).unwrap(), 1), + w_r11, + ), "4D8B9C78BE25664F", "movq 1332094398(%r8,%rdi,2), %r11", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg_reg_shift(0x4f6625be, r15, rdi, 0), w_r11), + Inst::mov64_m_r( + Amode::imm_reg_reg_shift( + 0x4f6625be, + Gpr::new(r15).unwrap(), + Gpr::new(rdi).unwrap(), + 0, + ), + w_r11, + ), "4D8B9C3FBE25664F", "movq 1332094398(%r15,%rdi,1), %r11", )); @@ -823,7 +966,12 @@ fn test_x64_emit() { // Addr_IRRS, offset large negative simm32 insns.push(( Inst::mov64_m_r( - Amode::imm_reg_reg_shift(-0x264d1690i32 as u32, rax, r8, 2), + Amode::imm_reg_reg_shift( + -0x264d1690i32 as u32, + Gpr::new(rax).unwrap(), + Gpr::new(r8).unwrap(), + 2, + ), w_r11, ), "4E8B9C8070E9B2D9", @@ -831,7 +979,12 @@ fn test_x64_emit() { )); insns.push(( Inst::mov64_m_r( - Amode::imm_reg_reg_shift(-0x264d1690i32 as u32, rdi, r8, 3), + Amode::imm_reg_reg_shift( + -0x264d1690i32 as u32, + Gpr::new(rdi).unwrap(), + Gpr::new(r8).unwrap(), + 3, + ), w_r11, ), "4E8B9CC770E9B2D9", @@ -839,7 +992,12 @@ fn test_x64_emit() { )); insns.push(( Inst::mov64_m_r( - Amode::imm_reg_reg_shift(-0x264d1690i32 as u32, r8, r8, 0), + Amode::imm_reg_reg_shift( + -0x264d1690i32 as u32, + Gpr::new(r8).unwrap(), + Gpr::new(r8).unwrap(), + 0, + ), w_r11, ), "4F8B9C0070E9B2D9", @@ -847,7 +1005,12 @@ fn test_x64_emit() { )); insns.push(( Inst::mov64_m_r( - Amode::imm_reg_reg_shift(-0x264d1690i32 as u32, r15, r8, 1), + Amode::imm_reg_reg_shift( + -0x264d1690i32 as u32, + Gpr::new(r15).unwrap(), + Gpr::new(r8).unwrap(), + 1, + ), w_r11, ), "4F8B9C4770E9B2D9", @@ -855,7 +1018,12 @@ fn test_x64_emit() { )); insns.push(( Inst::mov64_m_r( - Amode::imm_reg_reg_shift(-0x264d1690i32 as u32, rax, r15, 1), + Amode::imm_reg_reg_shift( + -0x264d1690i32 as u32, + Gpr::new(rax).unwrap(), + Gpr::new(r15).unwrap(), + 1, + ), w_r11, ), "4E8B9C7870E9B2D9", @@ -863,7 +1031,12 @@ fn test_x64_emit() { )); insns.push(( Inst::mov64_m_r( - Amode::imm_reg_reg_shift(-0x264d1690i32 as u32, rdi, r15, 0), + Amode::imm_reg_reg_shift( + -0x264d1690i32 as u32, + Gpr::new(rdi).unwrap(), + Gpr::new(r15).unwrap(), + 0, + ), w_r11, ), "4E8B9C3F70E9B2D9", @@ -871,7 +1044,12 @@ fn test_x64_emit() { )); insns.push(( Inst::mov64_m_r( - Amode::imm_reg_reg_shift(-0x264d1690i32 as u32, r8, r15, 3), + Amode::imm_reg_reg_shift( + -0x264d1690i32 as u32, + Gpr::new(r8).unwrap(), + Gpr::new(r15).unwrap(), + 3, + ), w_r11, ), "4F8B9CF870E9B2D9", @@ -879,7 +1057,12 @@ fn test_x64_emit() { )); insns.push(( Inst::mov64_m_r( - Amode::imm_reg_reg_shift(-0x264d1690i32 as u32, r15, r15, 2), + Amode::imm_reg_reg_shift( + -0x264d1690i32 as u32, + Gpr::new(r15).unwrap(), + Gpr::new(r15).unwrap(), + 2, + ), w_r11, ), "4F8B9CBF70E9B2D9", @@ -1828,42 +2011,66 @@ fn test_x64_emit() { // ======================================================== // Mov64_M_R insns.push(( - Inst::mov64_m_r(Amode::imm_reg_reg_shift(179, rax, rbx, 0), w_rcx), + Inst::mov64_m_r( + Amode::imm_reg_reg_shift(179, Gpr::new(rax).unwrap(), Gpr::new(rbx).unwrap(), 0), + w_rcx, + ), "488B8C18B3000000", "movq 179(%rax,%rbx,1), %rcx", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg_reg_shift(179, rax, rbx, 0), w_r8), + Inst::mov64_m_r( + Amode::imm_reg_reg_shift(179, Gpr::new(rax).unwrap(), Gpr::new(rbx).unwrap(), 0), + w_r8, + ), "4C8B8418B3000000", "movq 179(%rax,%rbx,1), %r8", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg_reg_shift(179, rax, r9, 0), w_rcx), + Inst::mov64_m_r( + Amode::imm_reg_reg_shift(179, Gpr::new(rax).unwrap(), Gpr::new(r9).unwrap(), 0), + w_rcx, + ), "4A8B8C08B3000000", "movq 179(%rax,%r9,1), %rcx", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg_reg_shift(179, rax, r9, 0), w_r8), + Inst::mov64_m_r( + Amode::imm_reg_reg_shift(179, Gpr::new(rax).unwrap(), Gpr::new(r9).unwrap(), 0), + w_r8, + ), "4E8B8408B3000000", "movq 179(%rax,%r9,1), %r8", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg_reg_shift(179, r10, rbx, 0), w_rcx), + Inst::mov64_m_r( + Amode::imm_reg_reg_shift(179, Gpr::new(r10).unwrap(), Gpr::new(rbx).unwrap(), 0), + w_rcx, + ), "498B8C1AB3000000", "movq 179(%r10,%rbx,1), %rcx", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg_reg_shift(179, r10, rbx, 0), w_r8), + Inst::mov64_m_r( + Amode::imm_reg_reg_shift(179, Gpr::new(r10).unwrap(), Gpr::new(rbx).unwrap(), 0), + w_r8, + ), "4D8B841AB3000000", "movq 179(%r10,%rbx,1), %r8", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg_reg_shift(179, r10, r9, 0), w_rcx), + Inst::mov64_m_r( + Amode::imm_reg_reg_shift(179, Gpr::new(r10).unwrap(), Gpr::new(r9).unwrap(), 0), + w_rcx, + ), "4B8B8C0AB3000000", "movq 179(%r10,%r9,1), %rcx", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg_reg_shift(179, r10, r9, 0), w_r8), + Inst::mov64_m_r( + Amode::imm_reg_reg_shift(179, Gpr::new(r10).unwrap(), Gpr::new(r9).unwrap(), 0), + w_r8, + ), "4F8B840AB3000000", "movq 179(%r10,%r9,1), %r8", )); @@ -1881,7 +2088,10 @@ fn test_x64_emit() { "lea 42(%r10), %r15", )); insns.push(( - Inst::lea(Amode::imm_reg_reg_shift(179, r10, r9, 0), w_r8), + Inst::lea( + Amode::imm_reg_reg_shift(179, Gpr::new(r10).unwrap(), Gpr::new(r9).unwrap(), 0), + w_r8, + ), "4F8D840AB3000000", "lea 179(%r10,%r9,1), %r8", )); @@ -3115,7 +3325,12 @@ fn test_x64_emit() { Inst::cmove( OperandSize::Size16, CC::NO, - RegMem::mem(Amode::imm_reg_reg_shift(37, rdi, rsi, 2)), + RegMem::mem(Amode::imm_reg_reg_shift( + 37, + Gpr::new(rdi).unwrap(), + Gpr::new(rsi).unwrap(), + 2, + )), w_r15, ), "66440F417CB725", @@ -3157,12 +3372,22 @@ fn test_x64_emit() { insns.push((Inst::push64(RegMemImm::reg(rdi)), "57", "pushq %rdi")); insns.push((Inst::push64(RegMemImm::reg(r8)), "4150", "pushq %r8")); insns.push(( - Inst::push64(RegMemImm::mem(Amode::imm_reg_reg_shift(321, rsi, rcx, 3))), + Inst::push64(RegMemImm::mem(Amode::imm_reg_reg_shift( + 321, + Gpr::new(rsi).unwrap(), + Gpr::new(rcx).unwrap(), + 3, + ))), "FFB4CE41010000", "pushq 321(%rsi,%rcx,8)", )); insns.push(( - Inst::push64(RegMemImm::mem(Amode::imm_reg_reg_shift(321, r9, rbx, 2))), + Inst::push64(RegMemImm::mem(Amode::imm_reg_reg_shift( + 321, + Gpr::new(r9).unwrap(), + Gpr::new(rbx).unwrap(), + 2, + ))), "41FFB49941010000", "pushq 321(%r9,%rbx,4)", )); @@ -3226,12 +3451,22 @@ fn test_x64_emit() { insns.push((call_unknown(RegMem::reg(rbp)), "FFD5", "call *%rbp")); insns.push((call_unknown(RegMem::reg(r11)), "41FFD3", "call *%r11")); insns.push(( - call_unknown(RegMem::mem(Amode::imm_reg_reg_shift(321, rsi, rcx, 3))), + call_unknown(RegMem::mem(Amode::imm_reg_reg_shift( + 321, + Gpr::new(rsi).unwrap(), + Gpr::new(rcx).unwrap(), + 3, + ))), "FF94CE41010000", "call *321(%rsi,%rcx,8)", )); insns.push(( - call_unknown(RegMem::mem(Amode::imm_reg_reg_shift(321, r10, rdx, 2))), + call_unknown(RegMem::mem(Amode::imm_reg_reg_shift( + 321, + Gpr::new(r10).unwrap(), + Gpr::new(rdx).unwrap(), + 2, + ))), "41FF949241010000", "call *321(%r10,%rdx,4)", )); @@ -3301,12 +3536,22 @@ fn test_x64_emit() { "jmp *%r11", )); insns.push(( - Inst::jmp_unknown(RegMem::mem(Amode::imm_reg_reg_shift(321, rsi, rcx, 3))), + Inst::jmp_unknown(RegMem::mem(Amode::imm_reg_reg_shift( + 321, + Gpr::new(rsi).unwrap(), + Gpr::new(rcx).unwrap(), + 3, + ))), "FFA4CE41010000", "jmp *321(%rsi,%rcx,8)", )); insns.push(( - Inst::jmp_unknown(RegMem::mem(Amode::imm_reg_reg_shift(321, r10, rdx, 2))), + Inst::jmp_unknown(RegMem::mem(Amode::imm_reg_reg_shift( + 321, + Gpr::new(r10).unwrap(), + Gpr::new(rdx).unwrap(), + 2, + ))), "41FFA49241010000", "jmp *321(%r10,%rdx,4)", )); @@ -3354,7 +3599,12 @@ fn test_x64_emit() { insns.push(( Inst::xmm_rm_r( SseOpcode::Addss, - RegMem::mem(Amode::imm_reg_reg_shift(123, r10, rdx, 2)), + RegMem::mem(Amode::imm_reg_reg_shift( + 123, + Gpr::new(r10).unwrap(), + Gpr::new(rdx).unwrap(), + 2, + )), w_xmm0, ), "F3410F5844927B", @@ -3379,7 +3629,12 @@ fn test_x64_emit() { insns.push(( Inst::xmm_rm_r( SseOpcode::Subss, - RegMem::mem(Amode::imm_reg_reg_shift(321, r10, rax, 3)), + RegMem::mem(Amode::imm_reg_reg_shift( + 321, + Gpr::new(r10).unwrap(), + Gpr::new(rax).unwrap(), + 3, + )), w_xmm10, ), "F3450F5C94C241010000", @@ -4200,10 +4455,17 @@ fn test_x64_emit() { // ======================================================== // Pertaining to atomics. - let am1: SyntheticAmode = Amode::imm_reg_reg_shift(321, r10, rdx, 2).into(); + let am1: SyntheticAmode = + Amode::imm_reg_reg_shift(321, Gpr::new(r10).unwrap(), Gpr::new(rdx).unwrap(), 2).into(); // `am2` doesn't contribute any 1 bits to the rex prefix, so we must use it when testing // for retention of the apparently-redundant rex prefix in the 8-bit case. - let am2: SyntheticAmode = Amode::imm_reg_reg_shift(-12345i32 as u32, rcx, rsi, 3).into(); + let am2: SyntheticAmode = Amode::imm_reg_reg_shift( + -12345i32 as u32, + Gpr::new(rcx).unwrap(), + Gpr::new(rsi).unwrap(), + 3, + ) + .into(); // A general 8-bit case. insns.push(( diff --git a/cranelift/codegen/src/isa/x64/inst/mod.rs b/cranelift/codegen/src/isa/x64/inst/mod.rs index 6f00370ea09d..bd7c039b9436 100644 --- a/cranelift/codegen/src/isa/x64/inst/mod.rs +++ b/cranelift/codegen/src/isa/x64/inst/mod.rs @@ -2,13 +2,11 @@ use crate::binemit::{Addend, CodeOffset, Reloc, StackMap}; use crate::ir::{types, ExternalName, Opcode, SourceLoc, TrapCode, Type, ValueLabel}; -use crate::isa::unwind::UnwindInst; use crate::isa::x64::abi::X64ABIMachineSpec; use crate::isa::x64::settings as x64_settings; use crate::isa::CallConv; use crate::machinst::*; use crate::{settings, CodegenError, CodegenResult}; -use alloc::boxed::Box; use alloc::vec::Vec; use regalloc::{ PrettyPrint, PrettyPrintSized, RealRegUniverse, Reg, RegClass, RegUsageCollector, SpillSlot, @@ -31,510 +29,8 @@ use regs::show_ireg_sized; //============================================================================= // Instructions (top level): definition -// Don't build these directly. Instead use the Inst:: functions to create them. - -/// Instructions. -#[derive(Clone)] -pub enum Inst { - /// Nops of various sizes, including zero. - Nop { len: u8 }, - - // ===================================== - // Integer instructions. - /// Integer arithmetic/bit-twiddling: (add sub and or xor mul adc? sbb?) (32 64) (reg addr imm) reg - AluRmiR { - size: OperandSize, // 4 or 8 - op: AluRmiROpcode, - src1: Reg, - src2: RegMemImm, - dst: Writable, - }, - - /// Instructions on GPR that only read src and defines dst (dst is not modified): bsr, etc. - UnaryRmR { - size: OperandSize, // 2, 4 or 8 - op: UnaryRmROpcode, - src: RegMem, - dst: Writable, - }, - - /// Bitwise not - Not { - size: OperandSize, // 1, 2, 4 or 8 - src: Reg, - dst: Writable, - }, - - /// Integer negation - Neg { - size: OperandSize, // 1, 2, 4 or 8 - src: Reg, - dst: Writable, - }, - - /// Integer quotient and remainder: (div idiv) $rax $rdx (reg addr) - Div { - size: OperandSize, // 1, 2, 4 or 8 - signed: bool, - divisor: RegMem, - dividend: Reg, - dst_quotient: Writable, - dst_remainder: Writable, - }, - - /// The high bits (RDX) of a (un)signed multiply: RDX:RAX := RAX * rhs. - MulHi { - size: OperandSize, // 2, 4, or 8 - signed: bool, - src1: Reg, - src2: RegMem, - dst_lo: Writable, - dst_hi: Writable, - }, - - /// A synthetic sequence to implement the right inline checks for remainder and division, - /// assuming the dividend is in %rax. - /// - /// Puts the result back into %rax if is_div, %rdx if !is_div, to mimic what the div - /// instruction does. - /// - /// The generated code sequence is described in the emit's function match arm for this - /// instruction. - /// - /// Note: %rdx is marked as modified by this instruction, to avoid an early clobber problem - /// with the temporary and divisor registers. Make sure to zero %rdx right before this - /// instruction, or you might run into regalloc failures where %rdx is live before its first - /// def! - CheckedDivOrRemSeq { - kind: DivOrRemKind, - size: OperandSize, - dividend: Reg, - /// The divisor operand. Note it's marked as modified so that it gets assigned a register - /// different from the temporary. - divisor: Writable, - dst_quotient: Writable, - dst_remainder: Writable, - tmp: Option>, - }, - - /// Do a sign-extend based on the sign of the value in rax into rdx: (cwd cdq cqo) - /// or al into ah: (cbw) - SignExtendData { - size: OperandSize, // 1, 2, 4 or 8 - src: Reg, - dst: Writable, - }, - - /// Constant materialization: (imm32 imm64) reg. - /// - /// Either: movl $imm32, %reg32 or movabsq $imm64, %reg32. - Imm { - dst_size: OperandSize, // 4 or 8 - simm64: u64, - dst: Writable, - }, - - /// GPR to GPR move: mov (64 32) reg reg. - MovRR { - size: OperandSize, // 4 or 8 - src: Reg, - dst: Writable, - }, - - /// Zero-extended loads, except for 64 bits: movz (bl bq wl wq lq) addr reg. - /// Note that the lq variant doesn't really exist since the default zero-extend rule makes it - /// unnecessary. For that case we emit the equivalent "movl AM, reg32". - MovzxRmR { - ext_mode: ExtMode, - src: RegMem, - dst: Writable, - }, - - /// A plain 64-bit integer load, since MovZX_RM_R can't represent that. - Mov64MR { - src: SyntheticAmode, - dst: Writable, - }, - - /// Loads the memory address of addr into dst. - LoadEffectiveAddress { - addr: SyntheticAmode, - dst: Writable, - }, - - /// Sign-extended loads and moves: movs (bl bq wl wq lq) addr reg. - MovsxRmR { - ext_mode: ExtMode, - src: RegMem, - dst: Writable, - }, - - /// Integer stores: mov (b w l q) reg addr. - MovRM { - size: OperandSize, // 1, 2, 4 or 8. - src: Reg, - dst: SyntheticAmode, - }, - - /// Arithmetic shifts: (shl shr sar) (b w l q) imm reg. - ShiftR { - size: OperandSize, // 1, 2, 4 or 8 - kind: ShiftKind, - src: Reg, - /// shift count: Some(0 .. #bits-in-type - 1), or None to mean "%cl". - num_bits: Imm8Reg, - dst: Writable, - }, - - /// Arithmetic SIMD shifts. - XmmRmiReg { - opcode: SseOpcode, - src1: Reg, - src2: RegMemImm, - dst: Writable, - }, - - /// Integer comparisons/tests: cmp or test (b w l q) (reg addr imm) reg. - CmpRmiR { - size: OperandSize, // 1, 2, 4 or 8 - opcode: CmpOpcode, - src: RegMemImm, - dst: Reg, - }, - - /// Materializes the requested condition code in the destination reg. - Setcc { cc: CC, dst: Writable }, - - /// Integer conditional move. - /// Overwrites the destination register. - Cmove { - size: OperandSize, // 2, 4, or 8 - cc: CC, - consequent: RegMem, - alternative: Reg, - dst: Writable, - }, - - // ===================================== - // Stack manipulation. - /// pushq (reg addr imm) - Push64 { src: RegMemImm }, - - /// popq reg - Pop64 { dst: Writable }, - - // ===================================== - // Floating-point operations. - /// XMM (scalar or vector) binary op: (add sub and or xor mul adc? sbb?) (32 64) (reg addr) reg - XmmRmR { - op: SseOpcode, - src1: Reg, - src2: RegMem, - dst: Writable, - }, - - XmmRmREvex { - op: Avx512Opcode, - src1: RegMem, - src2: Reg, - dst: Writable, - }, - - /// XMM (scalar or vector) unary op: mov between XMM registers (32 64) (reg addr) reg, sqrt, - /// etc. - /// - /// This differs from XMM_RM_R in that the dst register of XmmUnaryRmR is not used in the - /// computation of the instruction dst value and so does not have to be a previously valid - /// value. This is characteristic of mov instructions. - XmmUnaryRmR { - op: SseOpcode, - src: RegMem, - dst: Writable, - }, - - XmmUnaryRmREvex { - op: Avx512Opcode, - src: RegMem, - dst: Writable, - }, - - /// XMM (scalar or vector) unary op (from xmm to reg/mem): stores, movd, movq - XmmMovRM { - op: SseOpcode, - src: Reg, - dst: SyntheticAmode, - }, - - /// XMM (vector) unary op (to move a constant value into an xmm register): movups - XmmLoadConst { - src: VCodeConstant, - dst: Writable, - ty: Type, - }, - - /// XMM (scalar) unary op (from xmm to integer reg): movd, movq, cvtts{s,d}2si - XmmToGpr { - op: SseOpcode, - src: Reg, - dst: Writable, - dst_size: OperandSize, - }, - - /// XMM (scalar) unary op (from integer to float reg): movd, movq, cvtsi2s{s,d} - GprToXmm { - op: SseOpcode, - src: RegMem, - dst: Writable, - src_size: OperandSize, - }, - - /// Converts an unsigned int64 to a float32/float64. - CvtUint64ToFloatSeq { - dst_size: OperandSize, // 4 or 8 - /// A copy of the source register, fed by lowering. It is marked as modified during - /// register allocation to make sure that the temporary registers differ from the src - /// register, since both registers are live at the same time in the generated code - /// sequence. - src: Writable, - dst: Writable, - tmp_gpr1: Writable, - tmp_gpr2: Writable, - }, - - /// Converts a scalar xmm to a signed int32/int64. - CvtFloatToSintSeq { - dst_size: OperandSize, - src_size: OperandSize, - is_saturating: bool, - /// A copy of the source register, fed by lowering. It is marked as modified during - /// register allocation to make sure that the temporary xmm register differs from the src - /// register, since both registers are live at the same time in the generated code - /// sequence. - src: Writable, - dst: Writable, - tmp_gpr: Writable, - tmp_xmm: Writable, - }, - - /// Converts a scalar xmm to an unsigned int32/int64. - CvtFloatToUintSeq { - src_size: OperandSize, - dst_size: OperandSize, - is_saturating: bool, - /// A copy of the source register, fed by lowering, reused as a temporary. It is marked as - /// modified during register allocation to make sure that the temporary xmm register - /// differs from the src register, since both registers are live at the same time in the - /// generated code sequence. - src: Writable, - dst: Writable, - tmp_gpr: Writable, - tmp_xmm: Writable, - }, - - /// A sequence to compute min/max with the proper NaN semantics for xmm registers. - XmmMinMaxSeq { - size: OperandSize, - is_min: bool, - lhs: Reg, - rhs_dst: Writable, - }, - - /// XMM (scalar) conditional move. - /// Overwrites the destination register if cc is set. - XmmCmove { - size: OperandSize, // 4 or 8 - cc: CC, - src: RegMem, - dst: Writable, - }, - - /// Float comparisons/tests: cmp (b w l q) (reg addr imm) reg. - XmmCmpRmR { - op: SseOpcode, - src: RegMem, - dst: Reg, - }, - - /// A binary XMM instruction with an 8-bit immediate: e.g. cmp (ps pd) imm (reg addr) reg - XmmRmRImm { - op: SseOpcode, - src1: Reg, - src2: RegMem, - dst: Writable, - imm: u8, - size: OperandSize, // 4 or 8 - }, - - // ===================================== - // Control flow instructions. - /// Direct call: call simm32. - CallKnown { - dest: ExternalName, - uses: Vec, - defs: Vec>, - opcode: Opcode, - }, - - /// Indirect call: callq (reg mem). - CallUnknown { - dest: RegMem, - uses: Vec, - defs: Vec>, - opcode: Opcode, - }, - - /// Return. - Ret, - - /// A placeholder instruction, generating no code, meaning that a function epilogue must be - /// inserted there. - EpiloguePlaceholder, - - /// Jump to a known target: jmp simm32. - JmpKnown { dst: MachLabel }, - - /// One-way conditional branch: jcond cond target. - /// - /// This instruction is useful when we have conditional jumps depending on more than two - /// conditions, see for instance the lowering of Brz/brnz with Fcmp inputs. - /// - /// A note of caution: in contexts where the branch target is another block, this has to be the - /// same successor as the one specified in the terminator branch of the current block. - /// Otherwise, this might confuse register allocation by creating new invisible edges. - JmpIf { cc: CC, taken: MachLabel }, - - /// Two-way conditional branch: jcond cond target target. - /// Emitted as a compound sequence; the MachBuffer will shrink it as appropriate. - JmpCond { - cc: CC, - taken: MachLabel, - not_taken: MachLabel, - }, - - /// Jump-table sequence, as one compound instruction (see note in lower.rs for rationale). - /// The generated code sequence is described in the emit's function match arm for this - /// instruction. - /// See comment in lowering about the temporaries signedness. - JmpTableSeq { - idx: Reg, - tmp1: Writable, - tmp2: Writable, - default_target: MachLabel, - targets: Vec, - targets_for_term: Vec, - }, - - /// Indirect jump: jmpq (reg mem). - JmpUnknown { target: RegMem }, - - /// Traps if the condition code is set. - TrapIf { cc: CC, trap_code: TrapCode }, - - /// A debug trap. - Hlt, - - /// An instruction that will always trigger the illegal instruction exception. - Ud2 { trap_code: TrapCode }, - - /// Loads an external symbol in a register, with a relocation: - /// - /// movq $name@GOTPCREL(%rip), dst if PIC is enabled, or - /// movabsq $name, dst otherwise. - LoadExtName { - dst: Writable, - name: Box, - offset: i64, - }, - - // ===================================== - // Instructions pertaining to atomic memory accesses. - /// A standard (native) `lock cmpxchg src, (amode)`, with register conventions: - /// - /// `mem` (read) address - /// `replacement` (read) replacement value - /// %rax (modified) in: expected value, out: value that was actually at `dst` - /// %rflags is written. Do not assume anything about it after the instruction. - /// - /// The instruction "succeeded" iff the lowest `ty` bits of %rax afterwards are the same as - /// they were before. - LockCmpxchg { - ty: Type, // I8, I16, I32 or I64 - replacement: Reg, - expected: Reg, - mem: SyntheticAmode, - dst_old: Writable, - }, - - /// A synthetic instruction, based on a loop around a native `lock cmpxchg` instruction. - /// This atomically modifies a value in memory and returns the old value. The sequence - /// consists of an initial "normal" load from `dst`, followed by a loop which computes the - /// new value and tries to compare-and-swap ("CAS") it into `dst`, using the native - /// instruction `lock cmpxchg{b,w,l,q}` . The loop iterates until the CAS is successful. - /// If there is no contention, there will be only one pass through the loop body. The - /// sequence does *not* perform any explicit memory fence instructions - /// (mfence/sfence/lfence). - /// - /// Note that the transaction is atomic in the sense that, as observed by some other thread, - /// `dst` either has the initial or final value, but no other. It isn't atomic in the sense - /// of guaranteeing that no other thread writes to `dst` in between the initial load and the - /// CAS -- but that would cause the CAS to fail unless the other thread's last write before - /// the CAS wrote the same value that was already there. In other words, this - /// implementation suffers (unavoidably) from the A-B-A problem. - /// - /// This instruction sequence has fixed register uses as follows: - /// - /// %r9 (read) address - /// %r10 (read) second operand for `op` - /// %r11 (written) scratch reg; value afterwards has no meaning - /// %rax (written) the old value at %r9 - /// %rflags is written. Do not assume anything about it after the instruction. - AtomicRmwSeq { - ty: Type, // I8, I16, I32 or I64 - op: inst_common::AtomicRmwOp, - address: Reg, - operand: Reg, - temp: Writable, - dst_old: Writable, - }, - - /// A memory fence (mfence, lfence or sfence). - Fence { kind: FenceKind }, - - // ===================================== - // Meta-instructions generating no code. - /// Marker, no-op in generated code: SP "virtual offset" is adjusted. This - /// controls how MemArg::NominalSPOffset args are lowered. - VirtualSPOffsetAdj { offset: i64 }, - - /// Provides a way to tell the register allocator that the upcoming sequence of instructions - /// will overwrite `dst` so it should be considered as a `def`; use this with care. - /// - /// This is useful when we have a sequence of instructions whose register usages are nominally - /// `mod`s, but such that the combination of operations creates a result that is independent of - /// the initial register value. It's thus semantically a `def`, not a `mod`, when all the - /// instructions are taken together, so we want to ensure the register is defined (its - /// live-range starts) prior to the sequence to keep analyses happy. - /// - /// One alternative would be a compound instruction that somehow encapsulates the others and - /// reports its own `def`s/`use`s/`mod`s; this adds complexity (the instruction list is no - /// longer flat) and requires knowledge about semantics and initial-value independence anyway. - XmmUninitializedValue { dst: Writable }, - - /// A call to the `ElfTlsGetAddr` libcall. Returns address - /// of TLS symbol in rax. - ElfTlsGetAddr { symbol: ExternalName }, - - /// A Mach-O TLS symbol access. Returns address of the TLS - /// symbol in rax. - MachOTlsGetAddr { symbol: ExternalName }, - - /// A definition of a value label. - ValueLabelMarker { reg: Reg, label: ValueLabel }, - - /// An unwind pseudoinstruction describing the state of the - /// machine at this program point. - Unwind { inst: UnwindInst }, -} +// `Inst` is defined inside ISLE as `MInst`. We publicly re-export it here. +pub use super::lower::isle::generated_code::MInst as Inst; pub(crate) fn low32_will_sign_extend_to_64(x: u64) -> bool { let xs = x as i64; @@ -656,15 +152,20 @@ impl Inst { OperandSize::Size32, OperandSize::Size64 ])); - Self::UnaryRmR { size, op, src, dst } + Self::UnaryRmR { + size, + op, + src: GprMem::new(src).unwrap(), + dst: WritableGpr::from_writable_reg(dst).unwrap(), + } } pub(crate) fn not(size: OperandSize, src: Writable) -> Inst { debug_assert_eq!(src.to_reg().get_class(), RegClass::I64); Inst::Not { size, - src: src.to_reg(), - dst: src, + src: Gpr::new(src.to_reg()).unwrap(), + dst: WritableGpr::from_writable_reg(src).unwrap(), } } @@ -753,7 +254,11 @@ impl Inst { pub(crate) fn xmm_mov(op: SseOpcode, src: RegMem, dst: Writable) -> Inst { src.assert_regclass_is(RegClass::V128); debug_assert!(dst.to_reg().get_class() == RegClass::V128); - Inst::XmmUnaryRmR { op, src, dst } + Inst::XmmUnaryRmR { + op, + src: XmmMem::new(src).unwrap(), + dst: WritableXmm::from_writable_reg(dst).unwrap(), + } } pub(crate) fn xmm_load_const(src: VCodeConstant, dst: Writable, ty: Type) -> Inst { @@ -766,13 +271,21 @@ impl Inst { pub(crate) fn xmm_unary_rm_r(op: SseOpcode, src: RegMem, dst: Writable) -> Inst { src.assert_regclass_is(RegClass::V128); debug_assert!(dst.to_reg().get_class() == RegClass::V128); - Inst::XmmUnaryRmR { op, src, dst } + Inst::XmmUnaryRmR { + op, + src: XmmMem::new(src).unwrap(), + dst: WritableXmm::from_writable_reg(dst).unwrap(), + } } pub(crate) fn xmm_unary_rm_r_evex(op: Avx512Opcode, src: RegMem, dst: Writable) -> Inst { src.assert_regclass_is(RegClass::V128); debug_assert!(dst.to_reg().get_class() == RegClass::V128); - Inst::XmmUnaryRmREvex { op, src, dst } + Inst::XmmUnaryRmREvex { + op, + src: XmmMem::new(src).unwrap(), + dst: WritableXmm::from_writable_reg(dst).unwrap(), + } } pub(crate) fn xmm_rm_r(op: SseOpcode, src: RegMem, dst: Writable) -> Self { @@ -780,9 +293,9 @@ impl Inst { debug_assert!(dst.to_reg().get_class() == RegClass::V128); Inst::XmmRmR { op, - src1: dst.to_reg(), - src2: src, - dst, + src1: Xmm::new(dst.to_reg()).unwrap(), + src2: XmmMem::new(src).unwrap(), + dst: WritableXmm::from_writable_reg(dst).unwrap(), } } @@ -797,15 +310,17 @@ impl Inst { debug_assert!(dst.to_reg().get_class() == RegClass::V128); Inst::XmmRmREvex { op, - src1, - src2, - dst, + src1: XmmMem::new(src1).unwrap(), + src2: Xmm::new(src2).unwrap(), + dst: WritableXmm::from_writable_reg(dst).unwrap(), } } pub(crate) fn xmm_uninit_value(dst: Writable) -> Self { debug_assert!(dst.to_reg().get_class() == RegClass::V128); - Inst::XmmUninitializedValue { dst } + Inst::XmmUninitializedValue { + dst: WritableXmm::from_writable_reg(dst).unwrap(), + } } pub(crate) fn xmm_mov_r_m(op: SseOpcode, src: Reg, dst: impl Into) -> Inst { @@ -828,8 +343,8 @@ impl Inst { debug_assert!(dst_size.is_one_of(&[OperandSize::Size32, OperandSize::Size64])); Inst::XmmToGpr { op, - src, - dst, + src: Xmm::new(src).unwrap(), + dst: WritableGpr::from_writable_reg(dst).unwrap(), dst_size, } } @@ -846,7 +361,7 @@ impl Inst { Inst::GprToXmm { op, src, - dst, + dst: WritableXmm::from_writable_reg(dst).unwrap(), src_size, } } @@ -870,10 +385,10 @@ impl Inst { debug_assert!(tmp_gpr2.to_reg().get_class() == RegClass::I64); debug_assert!(dst.to_reg().get_class() == RegClass::V128); Inst::CvtUint64ToFloatSeq { - src, - dst, - tmp_gpr1, - tmp_gpr2, + src: WritableGpr::from_writable_reg(src).unwrap(), + dst: WritableXmm::from_writable_reg(dst).unwrap(), + tmp_gpr1: WritableGpr::from_writable_reg(tmp_gpr1).unwrap(), + tmp_gpr2: WritableGpr::from_writable_reg(tmp_gpr2).unwrap(), dst_size, } } @@ -897,10 +412,10 @@ impl Inst { src_size, dst_size, is_saturating, - src, - dst, - tmp_gpr, - tmp_xmm, + src: WritableXmm::from_writable_reg(src).unwrap(), + dst: WritableGpr::from_writable_reg(dst).unwrap(), + tmp_gpr: WritableGpr::from_writable_reg(tmp_gpr).unwrap(), + tmp_xmm: WritableXmm::from_writable_reg(tmp_xmm).unwrap(), } } @@ -923,10 +438,10 @@ impl Inst { src_size, dst_size, is_saturating, - src, - dst, - tmp_gpr, - tmp_xmm, + src: WritableXmm::from_writable_reg(src).unwrap(), + dst: WritableGpr::from_writable_reg(dst).unwrap(), + tmp_gpr: WritableGpr::from_writable_reg(tmp_gpr).unwrap(), + tmp_xmm: WritableXmm::from_writable_reg(tmp_xmm).unwrap(), } } @@ -976,9 +491,9 @@ impl Inst { debug_assert!(dst.to_reg().get_class() == RegClass::V128); Inst::XmmRmiReg { opcode, - src1: dst.to_reg(), - src2: src, - dst, + src1: Xmm::new(dst.to_reg()).unwrap(), + src2: XmmMemImm::new(src).unwrap(), + dst: WritableXmm::from_writable_reg(dst).unwrap(), } } @@ -1018,7 +533,7 @@ impl Inst { debug_assert!(dst.to_reg().get_class() == RegClass::I64); Inst::LoadEffectiveAddress { addr: addr.into(), - dst, + dst: WritableGpr::from_writable_reg(dst).unwrap(), } } @@ -1270,7 +785,7 @@ impl Inst { } Self::XmmRmR { op, src2, dst, .. } => { - src2.to_reg() == Some(dst.to_reg()) + src2.clone().to_reg_mem().to_reg() == Some(dst.to_reg().to_reg()) && (*op == SseOpcode::Xorps || *op == SseOpcode::Xorpd || *op == SseOpcode::Pxor @@ -1331,7 +846,11 @@ impl Inst { Inst::XmmRmiReg { src1, dst, .. } => { if *src1 != dst.to_reg() { debug_assert!(src1.is_virtual()); - insts.push(Self::gen_move(*dst, *src1, types::I8X16)); + insts.push(Self::gen_move( + dst.to_writable_reg(), + src1.to_reg(), + types::I8X16, + )); *src1 = dst.to_reg(); } insts.push(self); @@ -1339,7 +858,11 @@ impl Inst { Inst::XmmRmR { src1, dst, .. } => { if *src1 != dst.to_reg() { debug_assert!(src1.is_virtual()); - insts.push(Self::gen_move(*dst, *src1, types::I8X16)); + insts.push(Self::gen_move( + dst.to_writable_reg(), + src1.to_reg(), + types::I8X16, + )); *src1 = dst.to_reg(); } insts.push(self); @@ -1368,7 +891,11 @@ impl Inst { Inst::Not { src, dst, .. } | Inst::Neg { src, dst, .. } => { if *src != dst.to_reg() { debug_assert!(src.is_virtual()); - insts.push(Self::gen_move(*dst, *src, types::I64)); + insts.push(Self::gen_move( + dst.to_writable_reg(), + src.to_reg(), + types::I64, + )); *src = dst.to_reg(); } insts.push(self); @@ -1626,19 +1153,19 @@ impl PrettyPrint for Inst { "{} {}, {}", ljustify2(op.to_string(), suffix_bwlq(*size)), src.show_rru_sized(mb_rru, size.to_bytes()), - show_ireg_sized(dst.to_reg(), mb_rru, size.to_bytes()), + show_ireg_sized(dst.to_reg().to_reg(), mb_rru, size.to_bytes()), ), Inst::Not { size, src: _, dst } => format!( "{} {}", ljustify2("not".to_string(), suffix_bwlq(*size)), - show_ireg_sized(dst.to_reg(), mb_rru, size.to_bytes()) + show_ireg_sized(dst.to_reg().to_reg(), mb_rru, size.to_bytes()) ), Inst::Neg { size, src: _, dst } => format!( "{} {}", ljustify2("neg".to_string(), suffix_bwlq(*size)), - show_ireg_sized(dst.to_reg(), mb_rru, size.to_bytes()) + show_ireg_sized(dst.to_reg().to_reg(), mb_rru, size.to_bytes()) ), Inst::Div { @@ -1696,14 +1223,14 @@ impl PrettyPrint for Inst { "{} {}, {}", ljustify(op.to_string()), src.show_rru_sized(mb_rru, op.src_size()), - show_ireg_sized(dst.to_reg(), mb_rru, 8), + show_ireg_sized(dst.to_reg().to_reg(), mb_rru, 8), ), Inst::XmmUnaryRmREvex { op, src, dst, .. } => format!( "{} {}, {}", ljustify(op.to_string()), src.show_rru_sized(mb_rru, 8), - show_ireg_sized(dst.to_reg(), mb_rru, 8), + show_ireg_sized(dst.to_reg().to_reg(), mb_rru, 8), ), Inst::XmmMovRM { op, src, dst, .. } => format!( @@ -1717,7 +1244,7 @@ impl PrettyPrint for Inst { "{} {}, {}", ljustify(op.to_string()), src2.show_rru_sized(mb_rru, 8), - show_ireg_sized(dst.to_reg(), mb_rru, 8), + show_ireg_sized(dst.to_reg().to_reg(), mb_rru, 8), ), Inst::XmmRmREvex { @@ -1730,8 +1257,8 @@ impl PrettyPrint for Inst { "{} {}, {}, {}", ljustify(op.to_string()), src1.show_rru_sized(mb_rru, 8), - show_ireg_sized(*src2, mb_rru, 8), - show_ireg_sized(dst.to_reg(), mb_rru, 8), + show_ireg_sized(src2.to_reg(), mb_rru, 8), + show_ireg_sized(dst.to_reg().to_reg(), mb_rru, 8), ), Inst::XmmMinMaxSeq { @@ -1795,7 +1322,7 @@ impl PrettyPrint for Inst { "{} {}, {}", ljustify(op.to_string()), src.show_rru(mb_rru), - show_ireg_sized(dst.to_reg(), mb_rru, dst_size), + show_ireg_sized(dst.to_reg().to_reg(), mb_rru, dst_size), ) } @@ -1830,7 +1357,7 @@ impl PrettyPrint for Inst { "f32" } )), - show_ireg_sized(src.to_reg(), mb_rru, 8), + show_ireg_sized(src.to_reg().to_reg(), mb_rru, 8), dst.show_rru(mb_rru), ), @@ -1847,8 +1374,8 @@ impl PrettyPrint for Inst { src_size.to_bits(), dst_size.to_bits() )), - show_ireg_sized(src.to_reg(), mb_rru, 8), - show_ireg_sized(dst.to_reg(), mb_rru, dst_size.to_bytes()), + show_ireg_sized(src.to_reg().to_reg(), mb_rru, 8), + show_ireg_sized(dst.to_reg().to_reg(), mb_rru, dst_size.to_bytes()), ), Inst::CvtFloatToUintSeq { @@ -1864,8 +1391,8 @@ impl PrettyPrint for Inst { src_size.to_bits(), dst_size.to_bits() )), - show_ireg_sized(src.to_reg(), mb_rru, 8), - show_ireg_sized(dst.to_reg(), mb_rru, dst_size.to_bytes()), + show_ireg_sized(src.to_reg().to_reg(), mb_rru, 8), + show_ireg_sized(dst.to_reg().to_reg(), mb_rru, dst_size.to_bytes()), ), Inst::Imm { @@ -2174,11 +1701,11 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) { } Inst::Not { src, dst, .. } => { debug_assert_eq!(*src, dst.to_reg()); - collector.add_mod(*dst); + collector.add_mod(dst.to_writable_reg()); } Inst::Neg { src, dst, .. } => { debug_assert_eq!(*src, dst.to_reg()); - collector.add_mod(*dst); + collector.add_mod(dst.to_writable_reg()); } Inst::Div { size, @@ -2249,11 +1776,13 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) { } } } - Inst::UnaryRmR { src, dst, .. } - | Inst::XmmUnaryRmR { src, dst, .. } - | Inst::XmmUnaryRmREvex { src, dst, .. } => { - src.get_regs_as_uses(collector); - collector.add_def(*dst); + Inst::UnaryRmR { src, dst, .. } => { + src.clone().to_reg_mem().get_regs_as_uses(collector); + collector.add_def(dst.to_writable_reg()); + } + Inst::XmmUnaryRmR { src, dst, .. } | Inst::XmmUnaryRmREvex { src, dst, .. } => { + src.clone().to_reg_mem().get_regs_as_uses(collector); + collector.add_def(dst.to_writable_reg()); } Inst::XmmRmR { src1, @@ -2265,10 +1794,10 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) { debug_assert_eq!(*src1, dst.to_reg()); if inst.produces_const() { // No need to account for src, since src == dst. - collector.add_def(*dst); + collector.add_def(dst.to_writable_reg()); } else { - src2.get_regs_as_uses(collector); - collector.add_mod(*dst); + src2.clone().to_reg_mem().get_regs_as_uses(collector); + collector.add_mod(dst.to_writable_reg()); // Some instructions have an implicit use of XMM0. if *op == SseOpcode::Blendvpd || *op == SseOpcode::Blendvps @@ -2285,11 +1814,11 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) { dst, .. } => { - src1.get_regs_as_uses(collector); - collector.add_use(*src2); + src1.clone().to_reg_mem().get_regs_as_uses(collector); + collector.add_use(src2.to_reg()); match *op { - Avx512Opcode::Vpermi2b => collector.add_mod(*dst), - _ => collector.add_def(*dst), + Avx512Opcode::Vpermi2b => collector.add_mod(dst.to_writable_reg()), + _ => collector.add_def(dst.to_writable_reg()), } } Inst::XmmRmRImm { @@ -2320,7 +1849,7 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) { collector.add_mod(*dst); } } - Inst::XmmUninitializedValue { dst } => collector.add_def(*dst), + Inst::XmmUninitializedValue { dst } => collector.add_def(dst.to_writable_reg()), Inst::XmmLoadConst { dst, .. } => collector.add_def(*dst), Inst::XmmMinMaxSeq { lhs, rhs_dst, .. } => { collector.add_use(*lhs); @@ -2330,8 +1859,8 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) { src1, src2, dst, .. } => { debug_assert_eq!(*src1, dst.to_reg()); - src2.get_regs_as_uses(collector); - collector.add_mod(*dst); + src2.clone().to_reg_mem_imm().get_regs_as_uses(collector); + collector.add_mod(dst.to_writable_reg()); } Inst::XmmMovRM { src, dst, .. } => { collector.add_use(*src); @@ -2344,13 +1873,17 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) { Inst::Imm { dst, .. } => { collector.add_def(*dst); } - Inst::MovRR { src, dst, .. } | Inst::XmmToGpr { src, dst, .. } => { + Inst::MovRR { src, dst, .. } => { collector.add_use(*src); collector.add_def(*dst); } + Inst::XmmToGpr { src, dst, .. } => { + collector.add_use(src.to_reg()); + collector.add_def(dst.to_writable_reg()); + } Inst::GprToXmm { src, dst, .. } => { src.get_regs_as_uses(collector); - collector.add_def(*dst); + collector.add_def(dst.to_writable_reg()); } Inst::CvtUint64ToFloatSeq { src, @@ -2359,10 +1892,10 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) { tmp_gpr2, .. } => { - collector.add_mod(*src); - collector.add_def(*dst); - collector.add_def(*tmp_gpr1); - collector.add_def(*tmp_gpr2); + collector.add_mod(src.to_writable_reg()); + collector.add_def(dst.to_writable_reg()); + collector.add_def(tmp_gpr1.to_writable_reg()); + collector.add_def(tmp_gpr2.to_writable_reg()); } Inst::CvtFloatToSintSeq { src, @@ -2378,19 +1911,23 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) { tmp_xmm, .. } => { - collector.add_mod(*src); - collector.add_def(*dst); - collector.add_def(*tmp_gpr); - collector.add_def(*tmp_xmm); + collector.add_mod(src.to_writable_reg()); + collector.add_def(dst.to_writable_reg()); + collector.add_def(tmp_gpr.to_writable_reg()); + collector.add_def(tmp_xmm.to_writable_reg()); } Inst::MovzxRmR { src, dst, .. } => { src.get_regs_as_uses(collector); collector.add_def(*dst); } - Inst::Mov64MR { src, dst, .. } | Inst::LoadEffectiveAddress { addr: src, dst } => { + Inst::Mov64MR { src, dst, .. } => { src.get_regs_as_uses(collector); collector.add_def(*dst) } + Inst::LoadEffectiveAddress { addr: src, dst } => { + src.get_regs_as_uses(collector); + collector.add_def(dst.to_writable_reg()) + } Inst::MovsxRmR { src, dst, .. } => { src.get_regs_as_uses(collector); collector.add_def(*dst); @@ -2535,8 +2072,8 @@ impl Amode { ref mut index, .. } => { - map.map_use(base); - map.map_use(index); + base.map_use(map); + index.map_use(map); } Amode::RipRelative { .. } => { // RIP isn't involved in regalloc. @@ -2622,7 +2159,7 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { } Inst::Not { src, dst, .. } | Inst::Neg { src, dst, .. } => { debug_assert_eq!(*src, dst.to_reg()); - mapper.map_mod(dst); + dst.map_mod(mapper); *src = dst.to_reg(); } Inst::Div { divisor, .. } => divisor.map_uses(mapper), @@ -2643,14 +2180,17 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { ref mut src, ref mut dst, .. + } => { + src.map_uses(mapper); + dst.map_def(mapper); } - | Inst::UnaryRmR { + Inst::UnaryRmR { ref mut src, ref mut dst, .. } => { src.map_uses(mapper); - mapper.map_def(dst); + dst.map_def(mapper); } Inst::XmmRmRImm { ref op, @@ -2691,11 +2231,11 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { debug_assert_eq!(*src1, dst.to_reg()); if produces_const { src2.map_as_def(mapper); - mapper.map_def(dst); + dst.map_def(mapper); *src1 = dst.to_reg(); } else { src2.map_uses(mapper); - mapper.map_mod(dst); + dst.map_mod(mapper); *src1 = dst.to_reg(); } } @@ -2707,10 +2247,10 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { .. } => { src1.map_uses(mapper); - mapper.map_use(src2); + src2.map_use(mapper); match *op { - Avx512Opcode::Vpermi2b => mapper.map_mod(dst), - _ => mapper.map_def(dst), + Avx512Opcode::Vpermi2b => dst.map_mod(mapper), + _ => dst.map_def(mapper), } } Inst::XmmRmiReg { @@ -2721,11 +2261,11 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { } => { debug_assert_eq!(*src1, dst.to_reg()); src2.map_uses(mapper); - mapper.map_mod(dst); + dst.map_mod(mapper); *src1 = dst.to_reg(); } Inst::XmmUninitializedValue { ref mut dst, .. } => { - mapper.map_def(dst); + dst.map_def(mapper); } Inst::XmmLoadConst { ref mut dst, .. } => { mapper.map_def(dst); @@ -2759,14 +2299,17 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { ref mut src, ref mut dst, .. + } => { + mapper.map_use(src); + mapper.map_def(dst); } - | Inst::XmmToGpr { + Inst::XmmToGpr { ref mut src, ref mut dst, .. } => { - mapper.map_use(src); - mapper.map_def(dst); + src.map_use(mapper); + dst.map_def(mapper); } Inst::GprToXmm { ref mut src, @@ -2774,7 +2317,7 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { .. } => { src.map_uses(mapper); - mapper.map_def(dst); + dst.map_def(mapper); } Inst::CvtUint64ToFloatSeq { ref mut src, @@ -2783,10 +2326,10 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { ref mut tmp_gpr2, .. } => { - mapper.map_mod(src); - mapper.map_def(dst); - mapper.map_def(tmp_gpr1); - mapper.map_def(tmp_gpr2); + src.map_mod(mapper); + dst.map_def(mapper); + tmp_gpr1.map_def(mapper); + tmp_gpr2.map_def(mapper); } Inst::CvtFloatToSintSeq { ref mut src, @@ -2802,10 +2345,10 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { ref mut tmp_xmm, .. } => { - mapper.map_mod(src); - mapper.map_def(dst); - mapper.map_def(tmp_gpr); - mapper.map_def(tmp_xmm); + src.map_mod(mapper); + dst.map_def(mapper); + tmp_gpr.map_def(mapper); + tmp_xmm.map_def(mapper); } Inst::MovzxRmR { ref mut src, @@ -2815,10 +2358,14 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { src.map_uses(mapper); mapper.map_def(dst); } - Inst::Mov64MR { src, dst, .. } | Inst::LoadEffectiveAddress { addr: src, dst } => { + Inst::Mov64MR { src, dst, .. } => { src.map_uses(mapper); mapper.map_def(dst); } + Inst::LoadEffectiveAddress { addr: src, dst } => { + src.map_uses(mapper); + dst.map_def(mapper); + } Inst::MovsxRmR { ref mut src, ref mut dst, @@ -2989,8 +2536,8 @@ impl MachInst for Inst { || *op == SseOpcode::Movdqa || *op == SseOpcode::Movdqu => { - if let RegMem::Reg { reg } = src { - Some((*dst, *reg)) + if let RegMem::Reg { reg } = src.clone().to_reg_mem() { + Some((dst.to_writable_reg(), reg)) } else { None } diff --git a/cranelift/codegen/src/isa/x64/lower.isle b/cranelift/codegen/src/isa/x64/lower.isle index 9b881b5f691c..8de9110bbdfe 100644 --- a/cranelift/codegen/src/isa/x64/lower.isle +++ b/cranelift/codegen/src/isa/x64/lower.isle @@ -95,23 +95,23 @@ (rule (lower (has_type (multi_lane 8 16) (iadd x y))) - (value_reg (paddb (put_in_reg x) - (put_in_reg_mem y)))) + (value_xmm (paddb (put_in_xmm x) + (put_in_xmm_mem y)))) (rule (lower (has_type (multi_lane 16 8) (iadd x y))) - (value_reg (paddw (put_in_reg x) - (put_in_reg_mem y)))) + (value_xmm (paddw (put_in_xmm x) + (put_in_xmm_mem y)))) (rule (lower (has_type (multi_lane 32 4) (iadd x y))) - (value_reg (paddd (put_in_reg x) - (put_in_reg_mem y)))) + (value_xmm (paddd (put_in_xmm x) + (put_in_xmm_mem y)))) (rule (lower (has_type (multi_lane 64 2) (iadd x y))) - (value_reg (paddq (put_in_reg x) - (put_in_reg_mem y)))) + (value_xmm (paddq (put_in_xmm x) + (put_in_xmm_mem y)))) ;; `i128` (rule (lower (has_type $I128 (iadd x y))) @@ -131,25 +131,25 @@ (rule (lower (has_type (multi_lane 8 16) (sadd_sat x y))) - (value_reg (paddsb (put_in_reg x) - (put_in_reg_mem y)))) + (value_xmm (paddsb (put_in_xmm x) + (put_in_xmm_mem y)))) (rule (lower (has_type (multi_lane 16 8) (sadd_sat x y))) - (value_reg (paddsw (put_in_reg x) - (put_in_reg_mem y)))) + (value_xmm (paddsw (put_in_xmm x) + (put_in_xmm_mem y)))) ;;;; Rules for `uadd_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (rule (lower (has_type (multi_lane 8 16) (uadd_sat x y))) - (value_reg (paddusb (put_in_reg x) - (put_in_reg_mem y)))) + (value_xmm (paddusb (put_in_xmm x) + (put_in_xmm_mem y)))) (rule (lower (has_type (multi_lane 16 8) (uadd_sat x y))) - (value_reg (paddusw (put_in_reg x) - (put_in_reg_mem y)))) + (value_xmm (paddusw (put_in_xmm x) + (put_in_xmm_mem y)))) ;;;; Rules for `iadd_ifcout` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -264,23 +264,23 @@ (rule (lower (has_type (multi_lane 8 16) (isub x y))) - (value_reg (psubb (put_in_reg x) - (put_in_reg_mem y)))) + (value_xmm (psubb (put_in_xmm x) + (put_in_xmm_mem y)))) (rule (lower (has_type (multi_lane 16 8) (isub x y))) - (value_reg (psubw (put_in_reg x) - (put_in_reg_mem y)))) + (value_xmm (psubw (put_in_xmm x) + (put_in_xmm_mem y)))) (rule (lower (has_type (multi_lane 32 4) (isub x y))) - (value_reg (psubd (put_in_reg x) - (put_in_reg_mem y)))) + (value_xmm (psubd (put_in_xmm x) + (put_in_xmm_mem y)))) (rule (lower (has_type (multi_lane 64 2) (isub x y))) - (value_reg (psubq (put_in_reg x) - (put_in_reg_mem y)))) + (value_xmm (psubq (put_in_xmm x) + (put_in_xmm_mem y)))) ;; `i128` (rule (lower (has_type $I128 (isub x y))) @@ -300,25 +300,25 @@ (rule (lower (has_type (multi_lane 8 16) (ssub_sat x y))) - (value_reg (psubsb (put_in_reg x) - (put_in_reg_mem y)))) + (value_xmm (psubsb (put_in_xmm x) + (put_in_xmm_mem y)))) (rule (lower (has_type (multi_lane 16 8) (ssub_sat x y))) - (value_reg (psubsw (put_in_reg x) - (put_in_reg_mem y)))) + (value_xmm (psubsw (put_in_xmm x) + (put_in_xmm_mem y)))) ;;;; Rules for `usub_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (rule (lower (has_type (multi_lane 8 16) (usub_sat x y))) - (value_reg (psubusb (put_in_reg x) - (put_in_reg_mem y)))) + (value_xmm (psubusb (put_in_xmm x) + (put_in_xmm_mem y)))) (rule (lower (has_type (multi_lane 16 8) (usub_sat x y))) - (value_reg (psubusw (put_in_reg x) - (put_in_reg_mem y)))) + (value_xmm (psubusw (put_in_xmm x) + (put_in_xmm_mem y)))) ;;;; Rules for `band` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -360,14 +360,16 @@ ;; SSE. -(decl sse_and (Type Reg RegMem) Reg) +(decl sse_and (Type Xmm XmmMem) Xmm) (rule (sse_and $F32X4 x y) (andps x y)) (rule (sse_and $F64X2 x y) (andpd x y)) (rule (sse_and (multi_lane _bits _lanes) x y) (pand x y)) (rule (lower (has_type ty @ (multi_lane _bits _lanes) (band x y))) - (value_reg (sse_and ty (put_in_reg x) (put_in_reg_mem y)))) + (value_xmm (sse_and ty + (put_in_xmm x) + (put_in_xmm_mem y)))) ;; `{i,b}128`. @@ -432,14 +434,16 @@ ;; SSE. -(decl sse_or (Type Reg RegMem) Reg) +(decl sse_or (Type Xmm XmmMem) Xmm) (rule (sse_or $F32X4 x y) (orps x y)) (rule (sse_or $F64X2 x y) (orpd x y)) (rule (sse_or (multi_lane _bits _lanes) x y) (por x y)) (rule (lower (has_type ty @ (multi_lane _bits _lanes) (bor x y))) - (value_reg (sse_or ty (put_in_reg x) (put_in_reg_mem y)))) + (value_xmm (sse_or ty + (put_in_xmm x) + (put_in_xmm_mem y)))) ;; `{i,b}128`. @@ -507,7 +511,7 @@ ;; SSE. (rule (lower (has_type ty @ (multi_lane _bits _lanes) (bxor x y))) - (value_reg (sse_xor ty (put_in_reg x) (put_in_reg_mem y)))) + (value_xmm (sse_xor ty (put_in_xmm x) (put_in_xmm_mem y)))) ;; `{i,b}128`. @@ -578,16 +582,16 @@ ;; instructions. The basic idea, whether the amount to shift by is an immediate ;; or not, is to use a 16x8 shift and then mask off the incorrect bits to 0s. (rule (lower (has_type $I8X16 (ishl src amt))) - (let ((src_ Reg (put_in_reg src)) + (let ((src_ Xmm (put_in_xmm src)) (amt_gpr RegMemImm (put_in_reg_mem_imm amt)) - (amt_xmm RegMemImm (reg_mem_imm_to_xmm amt_gpr)) + (amt_xmm XmmMemImm (mov_rmi_to_xmm amt_gpr)) ;; Shift `src` using 16x8. Unfortunately, a 16x8 shift will only be ;; correct for half of the lanes; the others must be fixed up with ;; the mask below. - (unmasked Reg (psllw src_ amt_xmm)) + (unmasked Xmm (psllw src_ amt_xmm)) (mask_addr SyntheticAmode (ishl_i8x16_mask amt_gpr)) (mask Reg (x64_load $I8X16 mask_addr (ExtKind.None)))) - (value_reg (sse_and $I8X16 unmasked (RegMem.Reg mask))))) + (value_xmm (sse_and $I8X16 unmasked (xmm_mem_new (RegMem.Reg mask)))))) ;; Get the address of the mask to use when fixing up the lanes that weren't ;; correctly generated by the 16x8 shift. @@ -608,25 +612,28 @@ (extern constructor ishl_i8x16_mask_table ishl_i8x16_mask_table) (rule (ishl_i8x16_mask (RegMemImm.Reg amt)) (let ((mask_table SyntheticAmode (ishl_i8x16_mask_table)) - (base_mask_addr Reg (lea mask_table)) + (base_mask_addr Gpr (lea mask_table)) (mask_offset Reg (shl $I64 amt (Imm8Reg.Imm8 4)))) (amode_to_synthetic_amode (amode_imm_reg_reg_shift 0 base_mask_addr - mask_offset + (gpr_new mask_offset) 0)))) (rule (ishl_i8x16_mask (RegMemImm.Mem amt)) (ishl_i8x16_mask (RegMemImm.Reg (x64_load $I64 amt (ExtKind.None))))) ;; 16x8, 32x4, and 64x2 shifts can each use a single instruction. + (rule (lower (has_type $I16X8 (ishl src amt))) - (value_reg (psllw (put_in_reg src) - (reg_mem_imm_to_xmm (put_in_reg_mem_imm amt))))) + (value_xmm (psllw (put_in_xmm src) + (mov_rmi_to_xmm (put_in_reg_mem_imm amt))))) + (rule (lower (has_type $I32X4 (ishl src amt))) - (value_reg (pslld (put_in_reg src) - (reg_mem_imm_to_xmm (put_in_reg_mem_imm amt))))) + (value_xmm (pslld (put_in_xmm src) + (mov_rmi_to_xmm (put_in_reg_mem_imm amt))))) + (rule (lower (has_type $I64X2 (ishl src amt))) - (value_reg (psllq (put_in_reg src) - (reg_mem_imm_to_xmm (put_in_reg_mem_imm amt))))) + (value_xmm (psllq (put_in_xmm src) + (mov_rmi_to_xmm (put_in_reg_mem_imm amt))))) ;;;; Rules for `ushr` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -672,16 +679,18 @@ ;; There are no 8x16 shifts in x64. Do the same 16x8-shift-and-mask thing we do ;; with 8x16 `ishl`. (rule (lower (has_type $I8X16 (ushr src amt))) - (let ((src_ Reg (put_in_reg src)) + (let ((src_ Xmm (put_in_xmm src)) (amt_gpr RegMemImm (put_in_reg_mem_imm amt)) - (amt_xmm RegMemImm (reg_mem_imm_to_xmm amt_gpr)) + (amt_xmm XmmMemImm (mov_rmi_to_xmm amt_gpr)) ;; Shift `src` using 16x8. Unfortunately, a 16x8 shift will only be ;; correct for half of the lanes; the others must be fixed up with ;; the mask below. - (unmasked Reg (psrlw src_ amt_xmm)) + (unmasked Xmm (psrlw src_ amt_xmm)) (mask_addr SyntheticAmode (ushr_i8x16_mask amt_gpr)) (mask Reg (x64_load $I8X16 mask_addr (ExtKind.None)))) - (value_reg (sse_and $I8X16 unmasked (RegMem.Reg mask))))) + (value_xmm (sse_and $I8X16 + unmasked + (xmm_mem_new (RegMem.Reg mask)))))) ;; Get the address of the mask to use when fixing up the lanes that weren't ;; correctly generated by the 16x8 shift. @@ -702,25 +711,28 @@ (extern constructor ushr_i8x16_mask_table ushr_i8x16_mask_table) (rule (ushr_i8x16_mask (RegMemImm.Reg amt)) (let ((mask_table SyntheticAmode (ushr_i8x16_mask_table)) - (base_mask_addr Reg (lea mask_table)) + (base_mask_addr Gpr (lea mask_table)) (mask_offset Reg (shl $I64 amt (Imm8Reg.Imm8 4)))) (amode_to_synthetic_amode (amode_imm_reg_reg_shift 0 base_mask_addr - mask_offset + (gpr_new mask_offset) 0)))) (rule (ushr_i8x16_mask (RegMemImm.Mem amt)) (ushr_i8x16_mask (RegMemImm.Reg (x64_load $I64 amt (ExtKind.None))))) ;; 16x8, 32x4, and 64x2 shifts can each use a single instruction. + (rule (lower (has_type $I16X8 (ushr src amt))) - (value_reg (psrlw (put_in_reg src) - (reg_mem_imm_to_xmm (put_in_reg_mem_imm amt))))) + (value_xmm (psrlw (put_in_xmm src) + (mov_rmi_to_xmm (put_in_reg_mem_imm amt))))) + (rule (lower (has_type $I32X4 (ushr src amt))) - (value_reg (psrld (put_in_reg src) - (reg_mem_imm_to_xmm (put_in_reg_mem_imm amt))))) + (value_xmm (psrld (put_in_xmm src) + (mov_rmi_to_xmm (put_in_reg_mem_imm amt))))) + (rule (lower (has_type $I64X2 (ushr src amt))) - (value_reg (psrlq (put_in_reg src) - (reg_mem_imm_to_xmm (put_in_reg_mem_imm amt))))) + (value_xmm (psrlq (put_in_xmm src) + (mov_rmi_to_xmm (put_in_reg_mem_imm amt))))) ;;;; Rules for `sshr` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -780,33 +792,35 @@ ;; shifted_hi.i16x8 = shift each lane of `high` ;; result = [s0'', s1'', ..., s15''] (rule (lower (has_type $I8X16 (sshr src amt @ (value_type amt_ty)))) - (let ((src_ Reg (put_in_reg src)) + (let ((src_ Xmm (put_in_xmm src)) ;; In order for `packsswb` later to only use the high byte of each ;; 16x8 lane, we shift right an extra 8 bits, relying on `psraw` to ;; fill in the upper bits appropriately. - (lo Reg (punpcklbw src_ (RegMem.Reg src_))) - (hi Reg (punpckhbw src_ (RegMem.Reg src_))) - (amt_ RegMemImm (sshr_i8x16_bigger_shift amt_ty (put_in_reg_mem_imm amt))) - (shifted_lo Reg (psraw lo amt_)) - (shifted_hi Reg (psraw hi amt_))) - (value_reg (packsswb shifted_lo (RegMem.Reg shifted_hi))))) - -(decl sshr_i8x16_bigger_shift (Type RegMemImm) RegMemImm) + (lo Xmm (punpcklbw src_ (xmm_to_xmm_mem src_))) + (hi Xmm (punpckhbw src_ (xmm_to_xmm_mem src_))) + (amt_ XmmMemImm (sshr_i8x16_bigger_shift amt_ty (put_in_reg_mem_imm amt))) + (shifted_lo Xmm (psraw lo amt_)) + (shifted_hi Xmm (psraw hi amt_))) + (value_xmm (packsswb shifted_lo (xmm_to_xmm_mem shifted_hi))))) + +(decl sshr_i8x16_bigger_shift (Type RegMemImm) XmmMemImm) (rule (sshr_i8x16_bigger_shift _ty (RegMemImm.Imm i)) - (RegMemImm.Imm (u32_add i 8))) + (xmm_mem_imm_new (RegMemImm.Imm (u32_add i 8)))) (rule (sshr_i8x16_bigger_shift ty (RegMemImm.Reg r)) - (reg_mem_imm_to_xmm (RegMemImm.Reg (add ty r (RegMemImm.Imm 8))))) + (mov_rmi_to_xmm (RegMemImm.Reg (add ty r (RegMemImm.Imm 8))))) (rule (sshr_i8x16_bigger_shift ty rmi @ (RegMemImm.Mem _m)) - (reg_mem_imm_to_xmm (RegMemImm.Reg (add ty (imm ty 8) rmi)))) + (mov_rmi_to_xmm (RegMemImm.Reg (add ty (imm ty 8) rmi)))) ;; `sshr.{i16x8,i32x4}` can be a simple `psra{w,d}`, we just have to make sure ;; that if the shift amount is in a register, it is in an XMM register. + (rule (lower (has_type $I16X8 (sshr src amt))) - (value_reg (psraw (put_in_reg src) - (reg_mem_imm_to_xmm (put_in_reg_mem_imm amt))))) + (value_xmm (psraw (put_in_xmm src) + (mov_rmi_to_xmm (put_in_reg_mem_imm amt))))) + (rule (lower (has_type $I32X4 (sshr src amt))) - (value_reg (psrad (put_in_reg src) - (reg_mem_imm_to_xmm (put_in_reg_mem_imm amt))))) + (value_xmm (psrad (put_in_xmm src) + (mov_rmi_to_xmm (put_in_reg_mem_imm amt))))) ;; The `sshr.i64x2` CLIF instruction has no single x86 instruction in the older ;; feature sets. Newer ones like AVX512VL + AVX512F include `vpsraq`, a 128-bit @@ -817,14 +831,15 @@ ;; ;; (TODO: when EVEX support is available, add an alternate lowering here). (rule (lower (has_type $I64X2 (sshr src amt))) - (let ((src_ Reg (put_in_reg src)) - (lo Reg (pextrd $I64 src_ 0)) - (hi Reg (pextrd $I64 src_ 1)) + (let ((src_ Xmm (put_in_xmm src)) + (lo Gpr (pextrd $I64 src_ 0)) + (hi Gpr (pextrd $I64 src_ 1)) (amt_ Imm8Reg (put_masked_in_imm8_reg amt $I64)) - (shifted_lo Reg (sar $I64 lo amt_)) - (shifted_hi Reg (sar $I64 hi amt_))) - (value_reg (make_i64x2_from_lanes (RegMem.Reg shifted_lo) - (RegMem.Reg shifted_hi))))) + (shifted_lo Reg (sar $I64 (gpr_to_reg lo) amt_)) + (shifted_hi Reg (sar $I64 (gpr_to_reg hi) amt_))) + (value_xmm (make_i64x2_from_lanes (gpr_mem_new (RegMem.Reg shifted_lo)) + (gpr_mem_new (RegMem.Reg shifted_hi)))))) + ;;;; Rules for `rotl` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; `i16` and `i8`: we need to extend the shift amount, or mask the @@ -910,35 +925,35 @@ ;; `i64` and smaller. (rule (lower (has_type (fits_in_64 ty) (ineg x))) - (value_reg (neg ty (put_in_reg x)))) + (value_gpr (neg ty (put_in_gpr x)))) ;; SSE. (rule (lower (has_type $I8X16 (ineg x))) - (value_reg (psubb (imm $I8X16 0) - (put_in_reg_mem x)))) + (value_xmm (psubb (xmm_new (imm $I8X16 0)) + (put_in_xmm_mem x)))) (rule (lower (has_type $I16X8 (ineg x))) - (value_reg (psubw (imm $I16X8 0) - (put_in_reg_mem x)))) + (value_xmm (psubw (xmm_new (imm $I16X8 0)) + (put_in_xmm_mem x)))) (rule (lower (has_type $I32X4 (ineg x))) - (value_reg (psubd (imm $I32X4 0) - (put_in_reg_mem x)))) + (value_xmm (psubd (xmm_new (imm $I32X4 0)) + (put_in_xmm_mem x)))) (rule (lower (has_type $I64X2 (ineg x))) - (value_reg (psubq (imm $I64X2 0) - (put_in_reg_mem x)))) + (value_xmm (psubq (xmm_new (imm $I64X2 0)) + (put_in_xmm_mem x)))) ;;;; Rules for `avg_round` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (rule (lower (has_type (multi_lane 8 16) (avg_round x y))) - (value_reg (pavgb (put_in_reg x) (put_in_reg_mem y)))) + (value_xmm (pavgb (put_in_xmm x) (put_in_xmm_mem y)))) (rule (lower (has_type (multi_lane 16 8) (avg_round x y))) - (value_reg (pavgw (put_in_reg x) (put_in_reg_mem y)))) + (value_xmm (pavgw (put_in_xmm x) (put_in_xmm_mem y)))) ;;;; Rules for `imul` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -1017,10 +1032,10 @@ ;; (No i8x16 multiply.) (rule (lower (has_type (multi_lane 16 8) (imul x y))) - (value_reg (pmullw (put_in_reg x) (put_in_reg_mem y)))) + (value_xmm (pmullw (put_in_xmm x) (put_in_xmm_mem y)))) (rule (lower (has_type (multi_lane 32 4) (imul x y))) - (value_reg (pmulld (put_in_reg x) (put_in_reg_mem y)))) + (value_xmm (pmulld (put_in_xmm x) (put_in_xmm_mem y)))) ;; With AVX-512 we can implement `i64x2` multiplication with a single ;; instruction. @@ -1028,7 +1043,7 @@ (avx512dq_enabled) (multi_lane 64 2)) (imul x y))) - (value_reg (vpmullq (put_in_reg_mem x) (put_in_reg y)))) + (value_xmm (vpmullq (put_in_xmm_mem x) (put_in_xmm y)))) ;; Otherwise, for i64x2 multiplication we describe a lane A as being composed of ;; a 32-bit upper half "Ah" and a 32-bit lower half "Al". The 32-bit long hand @@ -1052,24 +1067,24 @@ ;; 32-bits when doing calculations, i.e., `Ah == A >> 32`. (rule (lower (has_type (multi_lane 64 2) (imul a b))) - (let ((a0 Reg (put_in_reg a)) - (b0 Reg (put_in_reg b)) + (let ((a0 Xmm (put_in_xmm a)) + (b0 Xmm (put_in_xmm b)) ;; a_hi = A >> 32 - (a_hi Reg (psrlq a0 (RegMemImm.Imm 32))) + (a_hi Xmm (psrlq a0 (xmm_mem_imm_new (RegMemImm.Imm 32)))) ;; ah_bl = Ah * Bl - (ah_bl Reg (pmuludq a_hi (RegMem.Reg b0))) + (ah_bl Xmm (pmuludq a_hi (xmm_to_xmm_mem b0))) ;; b_hi = B >> 32 - (b_hi Reg (psrlq b0 (RegMemImm.Imm 32))) + (b_hi Xmm (psrlq b0 (xmm_mem_imm_new (RegMemImm.Imm 32)))) ;; al_bh = Al * Bh - (al_bh Reg (pmuludq a0 (RegMem.Reg b_hi))) + (al_bh Xmm (pmuludq a0 (xmm_to_xmm_mem b_hi))) ;; aa_bb = ah_bl + al_bh - (aa_bb Reg (paddq ah_bl (RegMem.Reg al_bh))) + (aa_bb Xmm (paddq ah_bl (xmm_to_xmm_mem al_bh))) ;; aa_bb_shifted = aa_bb << 32 - (aa_bb_shifted Reg (psllq aa_bb (RegMemImm.Imm 32))) + (aa_bb_shifted Xmm (psllq aa_bb (xmm_mem_imm_new (RegMemImm.Imm 32)))) ;; al_bl = Al * Bl - (al_bl Reg (pmuludq a0 (RegMem.Reg b0)))) + (al_bl Xmm (pmuludq a0 (xmm_to_xmm_mem b0)))) ;; al_bl + aa_bb_shifted - (value_reg (paddq al_bl (RegMem.Reg aa_bb_shifted))))) + (value_xmm (paddq al_bl (xmm_to_xmm_mem aa_bb_shifted))))) ;; Special case for `i16x8.extmul_high_i8x16_s`. (rule (lower (has_type (multi_lane 16 8) @@ -1077,13 +1092,13 @@ x))) (def_inst (swiden_high (and (value_type (multi_lane 8 16)) y)))))) - (let ((x1 Reg (put_in_reg x)) - (x2 Reg (palignr x1 (RegMem.Reg x1) 8 (OperandSize.Size32))) - (x3 Reg (pmovsxbw (RegMem.Reg x2))) - (y1 Reg (put_in_reg y)) - (y2 Reg (palignr y1 (RegMem.Reg y1) 8 (OperandSize.Size32))) - (y3 Reg (pmovsxbw (RegMem.Reg y2)))) - (value_reg (pmullw x3 (RegMem.Reg y3))))) + (let ((x1 Xmm (put_in_xmm x)) + (x2 Xmm (palignr x1 (xmm_to_xmm_mem x1) 8 (OperandSize.Size32))) + (x3 Xmm (pmovsxbw (xmm_to_xmm_mem x2))) + (y1 Xmm (put_in_xmm y)) + (y2 Xmm (palignr y1 (xmm_to_xmm_mem y1) 8 (OperandSize.Size32))) + (y3 Xmm (pmovsxbw (xmm_to_xmm_mem y2)))) + (value_xmm (pmullw x3 (xmm_to_xmm_mem y3))))) ;; Special case for `i32x4.extmul_high_i16x8_s`. (rule (lower (has_type (multi_lane 32 4) @@ -1091,11 +1106,11 @@ x))) (def_inst (swiden_high (and (value_type (multi_lane 16 8)) y)))))) - (let ((x2 Reg (put_in_reg x)) - (y2 Reg (put_in_reg y)) - (lo Reg (pmullw x2 (RegMem.Reg y2))) - (hi Reg (pmulhw x2 (RegMem.Reg y2)))) - (value_reg (punpckhwd lo (RegMem.Reg hi))))) + (let ((x2 Xmm (put_in_xmm x)) + (y2 Xmm (put_in_xmm y)) + (lo Xmm (pmullw x2 (xmm_to_xmm_mem y2))) + (hi Xmm (pmulhw x2 (xmm_to_xmm_mem y2)))) + (value_xmm (punpckhwd lo (xmm_to_xmm_mem hi))))) ;; Special case for `i64x2.extmul_high_i32x4_s`. (rule (lower (has_type (multi_lane 64 2) @@ -1103,13 +1118,13 @@ x))) (def_inst (swiden_high (and (value_type (multi_lane 32 4)) y)))))) - (let ((x2 Reg (pshufd (put_in_reg_mem x) + (let ((x2 Xmm (pshufd (put_in_xmm_mem x) 0xFA (OperandSize.Size32))) - (y2 Reg (pshufd (put_in_reg_mem y) + (y2 Xmm (pshufd (put_in_xmm_mem y) 0xFA (OperandSize.Size32)))) - (value_reg (pmuldq x2 (RegMem.Reg y2))))) + (value_xmm (pmuldq x2 (xmm_to_xmm_mem y2))))) ;; Special case for `i16x8.extmul_low_i8x16_s`. (rule (lower (has_type (multi_lane 16 8) @@ -1117,9 +1132,9 @@ x))) (def_inst (swiden_low (and (value_type (multi_lane 8 16)) y)))))) - (let ((x2 Reg (pmovsxbw (put_in_reg_mem x))) - (y2 Reg (pmovsxbw (put_in_reg_mem y)))) - (value_reg (pmullw x2 (RegMem.Reg y2))))) + (let ((x2 Xmm (pmovsxbw (put_in_xmm_mem x))) + (y2 Xmm (pmovsxbw (put_in_xmm_mem y)))) + (value_xmm (pmullw x2 (xmm_to_xmm_mem y2))))) ;; Special case for `i32x4.extmul_low_i16x8_s`. (rule (lower (has_type (multi_lane 32 4) @@ -1127,11 +1142,11 @@ x))) (def_inst (swiden_low (and (value_type (multi_lane 16 8)) y)))))) - (let ((x2 Reg (put_in_reg x)) - (y2 Reg (put_in_reg y)) - (lo Reg (pmullw x2 (RegMem.Reg y2))) - (hi Reg (pmulhw x2 (RegMem.Reg y2)))) - (value_reg (punpcklwd lo (RegMem.Reg hi))))) + (let ((x2 Xmm (put_in_xmm x)) + (y2 Xmm (put_in_xmm y)) + (lo Xmm (pmullw x2 (xmm_to_xmm_mem y2))) + (hi Xmm (pmulhw x2 (xmm_to_xmm_mem y2)))) + (value_xmm (punpcklwd lo (xmm_to_xmm_mem hi))))) ;; Special case for `i64x2.extmul_low_i32x4_s`. (rule (lower (has_type (multi_lane 64 2) @@ -1139,13 +1154,13 @@ x))) (def_inst (swiden_low (and (value_type (multi_lane 32 4)) y)))))) - (let ((x2 Reg (pshufd (put_in_reg_mem x) + (let ((x2 Xmm (pshufd (put_in_xmm_mem x) 0x50 (OperandSize.Size32))) - (y2 Reg (pshufd (put_in_reg_mem y) + (y2 Xmm (pshufd (put_in_xmm_mem y) 0x50 (OperandSize.Size32)))) - (value_reg (pmuldq x2 (RegMem.Reg y2))))) + (value_xmm (pmuldq x2 (xmm_to_xmm_mem y2))))) ;; Special case for `i16x8.extmul_high_i8x16_u`. (rule (lower (has_type (multi_lane 16 8) @@ -1153,13 +1168,13 @@ x))) (def_inst (uwiden_high (and (value_type (multi_lane 8 16)) y)))))) - (let ((x1 Reg (put_in_reg x)) - (x2 Reg (palignr x1 (RegMem.Reg x1) 8 (OperandSize.Size32))) - (x3 Reg (pmovzxbw (RegMem.Reg x2))) - (y1 Reg (put_in_reg y)) - (y2 Reg (palignr y1 (RegMem.Reg y1) 8 (OperandSize.Size32))) - (y3 Reg (pmovzxbw (RegMem.Reg y2)))) - (value_reg (pmullw x3 (RegMem.Reg y3))))) + (let ((x1 Xmm (put_in_xmm x)) + (x2 Xmm (palignr x1 (xmm_to_xmm_mem x1) 8 (OperandSize.Size32))) + (x3 Xmm (pmovzxbw (xmm_to_xmm_mem x2))) + (y1 Xmm (put_in_xmm y)) + (y2 Xmm (palignr y1 (xmm_to_xmm_mem y1) 8 (OperandSize.Size32))) + (y3 Xmm (pmovzxbw (xmm_to_xmm_mem y2)))) + (value_xmm (pmullw x3 (xmm_to_xmm_mem y3))))) ;; Special case for `i32x4.extmul_high_i16x8_u`. (rule (lower (has_type (multi_lane 32 4) @@ -1167,11 +1182,11 @@ x))) (def_inst (uwiden_high (and (value_type (multi_lane 16 8)) y)))))) - (let ((x2 Reg (put_in_reg x)) - (y2 Reg (put_in_reg y)) - (lo Reg (pmullw x2 (RegMem.Reg y2))) - (hi Reg (pmulhuw x2 (RegMem.Reg y2)))) - (value_reg (punpckhwd lo (RegMem.Reg hi))))) + (let ((x2 Xmm (put_in_xmm x)) + (y2 Xmm (put_in_xmm y)) + (lo Xmm (pmullw x2 (xmm_to_xmm_mem y2))) + (hi Xmm (pmulhuw x2 (xmm_to_xmm_mem y2)))) + (value_xmm (punpckhwd lo (xmm_to_xmm_mem hi))))) ;; Special case for `i64x2.extmul_high_i32x4_u`. (rule (lower (has_type (multi_lane 64 2) @@ -1179,13 +1194,13 @@ x))) (def_inst (uwiden_high (and (value_type (multi_lane 32 4)) y)))))) - (let ((x2 Reg (pshufd (put_in_reg_mem x) + (let ((x2 Xmm (pshufd (put_in_xmm_mem x) 0xFA (OperandSize.Size32))) - (y2 Reg (pshufd (put_in_reg_mem y) + (y2 Xmm (pshufd (put_in_xmm_mem y) 0xFA (OperandSize.Size32)))) - (value_reg (pmuludq x2 (RegMem.Reg y2))))) + (value_xmm (pmuludq x2 (xmm_to_xmm_mem y2))))) ;; Special case for `i16x8.extmul_low_i8x16_u`. (rule (lower (has_type (multi_lane 16 8) @@ -1193,9 +1208,9 @@ x))) (def_inst (uwiden_low (and (value_type (multi_lane 8 16)) y)))))) - (let ((x2 Reg (pmovzxbw (put_in_reg_mem x))) - (y2 Reg (pmovzxbw (put_in_reg_mem y)))) - (value_reg (pmullw x2 (RegMem.Reg y2))))) + (let ((x2 Xmm (pmovzxbw (put_in_xmm_mem x))) + (y2 Xmm (pmovzxbw (put_in_xmm_mem y)))) + (value_xmm (pmullw x2 (xmm_to_xmm_mem y2))))) ;; Special case for `i32x4.extmul_low_i16x8_u`. (rule (lower (has_type (multi_lane 32 4) @@ -1203,11 +1218,11 @@ x))) (def_inst (uwiden_low (and (value_type (multi_lane 16 8)) y)))))) - (let ((x2 Reg (put_in_reg x)) - (y2 Reg (put_in_reg y)) - (lo Reg (pmullw x2 (RegMem.Reg y2))) - (hi Reg (pmulhuw x2 (RegMem.Reg y2)))) - (value_reg (punpcklwd lo (RegMem.Reg hi))))) + (let ((x2 Xmm (put_in_xmm x)) + (y2 Xmm (put_in_xmm y)) + (lo Xmm (pmullw x2 (xmm_to_xmm_mem y2))) + (hi Xmm (pmulhuw x2 (xmm_to_xmm_mem y2)))) + (value_xmm (punpcklwd lo (xmm_to_xmm_mem hi))))) ;; Special case for `i64x2.extmul_low_i32x4_u`. (rule (lower (has_type (multi_lane 64 2) @@ -1215,17 +1230,17 @@ x))) (def_inst (uwiden_low (and (value_type (multi_lane 32 4)) y)))))) - (let ((x2 Reg (pshufd (put_in_reg_mem x) + (let ((x2 Xmm (pshufd (put_in_xmm_mem x) 0x50 (OperandSize.Size32))) - (y2 Reg (pshufd (put_in_reg_mem y) + (y2 Xmm (pshufd (put_in_xmm_mem y) 0x50 (OperandSize.Size32)))) - (value_reg (pmuludq x2 (RegMem.Reg y2))))) + (value_xmm (pmuludq x2 (xmm_to_xmm_mem y2))))) ;;;; Rules for `band_not` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -(decl sse_and_not (Type Reg RegMem) Reg) +(decl sse_and_not (Type Xmm XmmMem) Xmm) (rule (sse_and_not $F32X4 x y) (andnps x y)) (rule (sse_and_not $F64X2 x y) (andnpd x y)) (rule (sse_and_not (multi_lane _bits _lanes) x y) (pandn x y)) @@ -1238,64 +1253,66 @@ ;; ;; pandn(x, y) = and(not(x), y) (rule (lower (has_type ty (band_not x y))) - (value_reg (sse_and_not ty - (put_in_reg y) - (put_in_reg_mem x)))) + (value_xmm (sse_and_not ty + (put_in_xmm y) + (put_in_xmm_mem x)))) ;;;; Rules for `iabs` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (rule (lower (has_type $I8X16 (iabs x))) - (value_reg (pabsb (put_in_reg_mem x)))) + (value_xmm (pabsb (put_in_xmm_mem x)))) (rule (lower (has_type $I16X8 (iabs x))) - (value_reg (pabsw (put_in_reg_mem x)))) + (value_xmm (pabsw (put_in_xmm_mem x)))) (rule (lower (has_type $I32X4 (iabs x))) - (value_reg (pabsd (put_in_reg_mem x)))) + (value_xmm (pabsd (put_in_xmm_mem x)))) ;; When AVX512 is available, we can use a single `vpabsq` instruction. (rule (lower (has_type (and (avx512vl_enabled) (avx512f_enabled) $I64X2) (iabs x))) - (value_reg (vpabsq (put_in_reg_mem x)))) + (value_xmm (vpabsq (put_in_xmm_mem x)))) -;; Otherwise, we use a separate register, `neg`, to contain the results of `0 - +;; Otherwise, we use a separate xmmister, `neg`, to contain the results of `0 - ;; x` and then blend in those results with `blendvpd` if the MSB of `neg` was ;; set to 1 (i.e. if `neg` was negative or, conversely, if `x` was originally ;; positive). (rule (lower (has_type $I64X2 (iabs x))) - (let ((rx Reg (put_in_reg x)) - (neg Reg (psubq (imm $I64X2 0) (RegMem.Reg rx)))) - (value_reg (blendvpd neg (RegMem.Reg rx) neg)))) + (let ((rx Xmm (put_in_xmm x)) + (neg Xmm (psubq (xmm_new (imm $I64X2 0)) (xmm_to_xmm_mem rx)))) + (value_xmm (blendvpd neg (xmm_to_xmm_mem rx) neg)))) ;;;; Rules for `fabs` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; Special case for `f32x4.abs`. (rule (lower (has_type $F32X4 (fabs x))) - (value_reg (andps (put_in_reg x) - (RegMem.Reg (psrld (vector_all_ones $F32X4) (RegMemImm.Imm 1)))))) + (value_xmm (andps (put_in_xmm x) + (xmm_to_xmm_mem (psrld (vector_all_ones $F32X4) + (xmm_mem_imm_new (RegMemImm.Imm 1))))))) ;; Special case for `f64x2.abs`. (rule (lower (has_type $F64X2 (fabs x))) - (value_reg (andpd (put_in_reg x) - (RegMem.Reg (psrlq (vector_all_ones $F64X2) (RegMemImm.Imm 1)))))) + (value_xmm (andpd (put_in_xmm x) + (xmm_to_xmm_mem (psrlq (vector_all_ones $F64X2) + (xmm_mem_imm_new (RegMemImm.Imm 1))))))) ;;;; Rules for `bnot` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; `i64` and smaller. (rule (lower (has_type (fits_in_64 ty) (bnot x))) - (value_reg (not ty (put_in_reg x)))) + (value_gpr (not ty (put_in_gpr x)))) ;; `i128`. (decl i128_not (Value) ValueRegs) (rule (i128_not x) (let ((x_regs ValueRegs (put_in_regs x)) - (x_lo Reg (value_regs_get x_regs 0)) - (x_hi Reg (value_regs_get x_regs 1))) - (value_regs (not $I64 x_lo) + (x_lo Gpr (gpr_new (value_regs_get x_regs 0))) + (x_hi Gpr (gpr_new (value_regs_get x_regs 1)))) + (value_gprs (not $I64 x_lo) (not $I64 x_hi)))) (rule (lower (has_type $I128 (bnot x))) @@ -1307,7 +1324,7 @@ ;; Special case for vector-types where bit-negation is an xor against an ;; all-one value (rule (lower (has_type ty @ (multi_lane _bits _lanes) (bnot x))) - (value_reg (sse_xor ty (put_in_reg x) (RegMem.Reg (vector_all_ones ty))))) + (value_xmm (sse_xor ty (put_in_xmm x) (xmm_to_xmm_mem (vector_all_ones ty))))) ;;;; Rules for `bitselect` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -1318,48 +1335,53 @@ ;; a = and if_true, condition ;; b = and_not condition, if_false ;; or b, a - (let ((cond_reg Reg (put_in_reg condition)) - (a Reg (sse_and ty (put_in_reg if_true) (RegMem.Reg cond_reg))) - (b Reg (sse_and_not ty cond_reg (put_in_reg_mem if_false)))) - (value_reg (sse_or ty b (RegMem.Reg a))))) + (let ((cond_xmm Xmm (put_in_xmm condition)) + (a Xmm (sse_and ty (put_in_xmm if_true) (xmm_to_xmm_mem cond_xmm))) + (b Xmm (sse_and_not ty cond_xmm (put_in_xmm_mem if_false)))) + (value_xmm (sse_or ty b (xmm_to_xmm_mem a))))) ;;;; Rules for `vselect` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (rule (lower (has_type ty @ (multi_lane _bits _lanes) (vselect condition if_true if_false))) - (value_reg (sse_blend ty - (put_in_reg_mem condition) - (put_in_reg_mem if_true) - (put_in_reg if_false)))) + (value_xmm (sse_blend ty + (put_in_xmm_mem condition) + (put_in_xmm_mem if_true) + (put_in_xmm if_false)))) ;;;; Rules for `insertlane` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (rule (lower (insertlane vec @ (value_type ty) val (u8_from_uimm8 idx))) - (value_reg (vec_insert_lane ty (put_in_reg vec) (put_in_reg_mem val) idx))) + (value_xmm (vec_insert_lane ty (put_in_xmm vec) (put_in_reg_mem val) idx))) ;; Helper function used below for `insertlane` but also here for other ;; lowerings. ;; ;; Note that the `Type` used here is the type of vector the insertion is ;; happening into, or the type of the first `Reg` argument. -(decl vec_insert_lane (Type Reg RegMem u8) Reg) +(decl vec_insert_lane (Type Xmm RegMem u8) Xmm) ;; i8x16.replace_lane -(rule (vec_insert_lane $I8X16 vec val idx) (pinsrb vec val idx)) +(rule (vec_insert_lane $I8X16 vec val idx) + (pinsrb vec (gpr_mem_new val) idx)) ;; i16x8.replace_lane -(rule (vec_insert_lane $I16X8 vec val idx) (pinsrw vec val idx)) +(rule (vec_insert_lane $I16X8 vec val idx) + (pinsrw vec (gpr_mem_new val) idx)) ;; i32x4.replace_lane -(rule (vec_insert_lane $I32X4 vec val idx) (pinsrd vec val idx (OperandSize.Size32))) +(rule (vec_insert_lane $I32X4 vec val idx) + (pinsrd vec (gpr_mem_new val) idx (OperandSize.Size32))) ;; i64x2.replace_lane -(rule (vec_insert_lane $I64X2 vec val idx) (pinsrd vec val idx (OperandSize.Size64))) +(rule (vec_insert_lane $I64X2 vec val idx) + (pinsrd vec (gpr_mem_new val) idx (OperandSize.Size64))) ;; f32x4.replace_lane -(rule (vec_insert_lane $F32X4 vec val idx) (insertps vec val (sse_insertps_lane_imm idx))) +(rule (vec_insert_lane $F32X4 vec val idx) + (insertps vec (xmm_mem_new val) (sse_insertps_lane_imm idx))) -;; external rust code used to calculate the immediate value to `insertps` +;; External rust code used to calculate the immediate value to `insertps`. (decl sse_insertps_lane_imm (u8) u8) (extern constructor sse_insertps_lane_imm sse_insertps_lane_imm) @@ -1378,60 +1400,63 @@ ;; load from memory into a temp register and then the second `movsd` (modeled ;; internally as `xmm_rm_r` will merge the temp register into our `vec` ;; register. -(rule (vec_insert_lane $F64X2 vec (RegMem.Reg val) 0) (movsd vec (RegMem.Reg val))) +(rule (vec_insert_lane $F64X2 vec (RegMem.Reg val) 0) + (movsd vec (xmm_mem_new (RegMem.Reg val)))) (rule (vec_insert_lane $F64X2 vec mem 0) - (movsd vec (RegMem.Reg (xmm_unary_rm_r (SseOpcode.Movsd) mem)))) + (movsd vec (xmm_to_xmm_mem (xmm_unary_rm_r (SseOpcode.Movsd) + (xmm_mem_new mem))))) ;; f64x2.replace_lane 1 ;; ;; Here the `movlhps` instruction is used specifically to specialize moving ;; into the second lane where unlike above cases we're not using the lane ;; immediate as an immediate to the instruction itself. -(rule (vec_insert_lane $F64X2 vec val 1) (movlhps vec val)) +(rule (vec_insert_lane $F64X2 vec val 1) + (movlhps vec (xmm_mem_new val))) ;;;; Rules for `imax` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (rule (lower (has_type $I8X16 (imax x y))) - (value_reg (pmaxsb (put_in_reg x) (put_in_reg_mem y)))) + (value_xmm (pmaxsb (put_in_xmm x) (put_in_xmm_mem y)))) (rule (lower (has_type $I16X8 (imax x y))) - (value_reg (pmaxsw (put_in_reg x) (put_in_reg_mem y)))) + (value_xmm (pmaxsw (put_in_xmm x) (put_in_xmm_mem y)))) (rule (lower (has_type $I32X4 (imax x y))) - (value_reg (pmaxsd (put_in_reg x) (put_in_reg_mem y)))) + (value_xmm (pmaxsd (put_in_xmm x) (put_in_xmm_mem y)))) ;;;; Rules for `imin` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (rule (lower (has_type $I8X16 (imin x y))) - (value_reg (pminsb (put_in_reg x) (put_in_reg_mem y)))) + (value_xmm (pminsb (put_in_xmm x) (put_in_xmm_mem y)))) (rule (lower (has_type $I16X8 (imin x y))) - (value_reg (pminsw (put_in_reg x) (put_in_reg_mem y)))) + (value_xmm (pminsw (put_in_xmm x) (put_in_xmm_mem y)))) (rule (lower (has_type $I32X4 (imin x y))) - (value_reg (pminsd (put_in_reg x) (put_in_reg_mem y)))) + (value_xmm (pminsd (put_in_xmm x) (put_in_xmm_mem y)))) ;;;; Rules for `umax` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (rule (lower (has_type $I8X16 (umax x y))) - (value_reg (pmaxub (put_in_reg x) (put_in_reg_mem y)))) + (value_xmm (pmaxub (put_in_xmm x) (put_in_xmm_mem y)))) (rule (lower (has_type $I16X8 (umax x y))) - (value_reg (pmaxuw (put_in_reg x) (put_in_reg_mem y)))) + (value_xmm (pmaxuw (put_in_xmm x) (put_in_xmm_mem y)))) (rule (lower (has_type $I32X4 (umax x y))) - (value_reg (pmaxud (put_in_reg x) (put_in_reg_mem y)))) + (value_xmm (pmaxud (put_in_xmm x) (put_in_xmm_mem y)))) ;;;; Rules for `umin` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (rule (lower (has_type $I8X16 (umin x y))) - (value_reg (pminub (put_in_reg x) (put_in_reg_mem y)))) + (value_xmm (pminub (put_in_xmm x) (put_in_xmm_mem y)))) (rule (lower (has_type $I16X8 (umin x y))) - (value_reg (pminuw (put_in_reg x) (put_in_reg_mem y)))) + (value_xmm (pminuw (put_in_xmm x) (put_in_xmm_mem y)))) (rule (lower (has_type $I32X4 (umin x y))) - (value_reg (pminud (put_in_reg x) (put_in_reg_mem y)))) + (value_xmm (pminud (put_in_xmm x) (put_in_xmm_mem y)))) ;;;; Rules for `trap` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; diff --git a/cranelift/codegen/src/isa/x64/lower.rs b/cranelift/codegen/src/isa/x64/lower.rs index d5503416de9f..3e74dddda93d 100644 --- a/cranelift/codegen/src/isa/x64/lower.rs +++ b/cranelift/codegen/src/isa/x64/lower.rs @@ -1,7 +1,7 @@ //! Lowering rules for X64. // ISLE integration glue. -mod isle; +pub(super) mod isle; use crate::data_value::DataValue; use crate::ir::{ @@ -1057,7 +1057,13 @@ fn lower_to_amode>(ctx: &mut C, spec: InsnInput, offset: i ) }; - return Amode::imm_reg_reg_shift(offset as u32, base, index, shift).with_flags(flags); + return Amode::imm_reg_reg_shift( + offset as u32, + Gpr::new(base).unwrap(), + Gpr::new(index).unwrap(), + shift, + ) + .with_flags(flags); } let input = put_input_in_reg(ctx, spec); @@ -3950,7 +3956,13 @@ fn lower_insn_to_regs>( let index = put_input_in_reg(ctx, inputs[1]); let shift = 0; let flags = ctx.memflags(insn).expect("load should have memflags"); - Amode::imm_reg_reg_shift(offset as u32, base, index, shift).with_flags(flags) + Amode::imm_reg_reg_shift( + offset as u32, + Gpr::new(base).unwrap(), + Gpr::new(index).unwrap(), + shift, + ) + .with_flags(flags) } _ => unreachable!(), }; @@ -4054,7 +4066,13 @@ fn lower_insn_to_regs>( let index = put_input_in_reg(ctx, inputs[2]); let shift = 0; let flags = ctx.memflags(insn).expect("store should have memflags"); - Amode::imm_reg_reg_shift(offset as u32, base, index, shift).with_flags(flags) + Amode::imm_reg_reg_shift( + offset as u32, + Gpr::new(base).unwrap(), + Gpr::new(index).unwrap(), + shift, + ) + .with_flags(flags) } _ => unreachable!(), diff --git a/cranelift/codegen/src/isa/x64/lower/isle.rs b/cranelift/codegen/src/isa/x64/lower/isle.rs index 613b6832d792..87a5ea9f6ece 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle.rs +++ b/cranelift/codegen/src/isa/x64/lower/isle.rs @@ -1,26 +1,28 @@ //! ISLE integration glue code for x64 lowering. // Pull in the ISLE generated code. -mod generated_code; +pub(crate) mod generated_code; +use generated_code::MInst; +use regalloc::Writable; // Types that the generated ISLE code uses via `use super::*`. -use super::{ - is_mergeable_load, lower_to_amode, AluRmiROpcode, Inst as MInst, OperandSize, Reg, RegMemImm, -}; +use super::{is_mergeable_load, lower_to_amode, Reg}; use crate::{ - ir::{immediates::*, types::*, Inst, InstructionData, Opcode, TrapCode, Value, ValueList}, - isa::x64::{ - inst::{ - args::{ - Amode, Avx512Opcode, CmpOpcode, ExtKind, ExtMode, FcmpImm, Imm8Reg, RegMem, - ShiftKind, SseOpcode, SyntheticAmode, CC, - }, - regs, x64_map_regs, + ir::{ + immediates::*, types::*, Inst, InstructionData, Opcode, TrapCode, Value, ValueLabel, + ValueList, + }, + isa::{ + settings::Flags, + unwind::UnwindInst, + x64::{ + inst::{args::*, regs, x64_map_regs}, + settings::Flags as IsaFlags, }, - settings::Flags as IsaFlags, }, - machinst::{isle::*, InsnInput, InsnOutput, LowerCtx, VCodeConstantData}, - settings::Flags, + machinst::{ + isle::*, AtomicRmwOp, InsnInput, InsnOutput, LowerCtx, VCodeConstant, VCodeConstantData, + }, }; use std::convert::TryFrom; @@ -252,8 +254,8 @@ where } #[inline] - fn xmm0(&mut self) -> WritableReg { - WritableReg::from_reg(regs::xmm0()) + fn xmm0(&mut self) -> WritableXmm { + WritableXmm::from_reg(Xmm::new(regs::xmm0()).unwrap()) } #[inline] @@ -262,7 +264,7 @@ where } #[inline] - fn amode_imm_reg_reg_shift(&mut self, simm32: u32, base: Reg, index: Reg, shift: u8) -> Amode { + fn amode_imm_reg_reg_shift(&mut self, simm32: u32, base: Gpr, index: Gpr, shift: u8) -> Amode { Amode::imm_reg_reg_shift(simm32, base, index, shift) } @@ -271,6 +273,16 @@ where amode.clone().into() } + #[inline] + fn writable_gpr_to_reg(&mut self, r: WritableGpr) -> WritableReg { + r.to_writable_reg() + } + + #[inline] + fn writable_xmm_to_reg(&mut self, r: WritableXmm) -> WritableReg { + r.to_writable_reg() + } + fn ishl_i8x16_mask_for_const(&mut self, amt: u32) -> SyntheticAmode { // When the shift amount is known, we can statically (i.e. at compile // time) determine the mask to use and only emit that. @@ -306,6 +318,96 @@ where .use_constant(VCodeConstantData::WellKnown(&I8X16_USHR_MASKS)); SyntheticAmode::ConstantOffset(mask_table) } + + #[inline] + fn writable_reg_to_xmm(&mut self, r: WritableReg) -> WritableXmm { + Writable::from_reg(Xmm::new(r.to_reg()).unwrap()) + } + + #[inline] + fn writable_xmm_to_xmm(&mut self, r: WritableXmm) -> Xmm { + r.to_reg() + } + + #[inline] + fn writable_gpr_to_gpr(&mut self, r: WritableGpr) -> Gpr { + r.to_reg() + } + + #[inline] + fn gpr_to_reg(&mut self, r: Gpr) -> Reg { + r.into() + } + + #[inline] + fn xmm_to_reg(&mut self, r: Xmm) -> Reg { + r.into() + } + + #[inline] + fn xmm_to_xmm_mem_imm(&mut self, r: Xmm) -> XmmMemImm { + r.into() + } + + #[inline] + fn temp_writable_gpr(&mut self) -> WritableGpr { + Writable::from_reg(Gpr::new(self.temp_writable_reg(I64).to_reg()).unwrap()) + } + + #[inline] + fn temp_writable_xmm(&mut self) -> WritableXmm { + Writable::from_reg(Xmm::new(self.temp_writable_reg(I8X16).to_reg()).unwrap()) + } + + #[inline] + fn xmm_mem_new(&mut self, rm: &RegMem) -> XmmMem { + XmmMem::new(rm.clone()).unwrap() + } + + #[inline] + fn gpr_mem_imm_new(&mut self, rmi: &RegMemImm) -> GprMemImm { + GprMemImm::new(rmi.clone()).unwrap() + } + + #[inline] + fn xmm_mem_imm_new(&mut self, rmi: &RegMemImm) -> XmmMemImm { + XmmMemImm::new(rmi.clone()).unwrap() + } + + #[inline] + fn xmm_to_xmm_mem(&mut self, r: Xmm) -> XmmMem { + r.into() + } + + #[inline] + fn xmm_mem_to_reg_mem(&mut self, xm: &XmmMem) -> RegMem { + xm.clone().into() + } + + #[inline] + fn gpr_mem_to_reg_mem(&mut self, gm: &GprMem) -> RegMem { + gm.clone().into() + } + + #[inline] + fn xmm_new(&mut self, r: Reg) -> Xmm { + Xmm::new(r).unwrap() + } + + #[inline] + fn gpr_new(&mut self, r: Reg) -> Gpr { + Gpr::new(r).unwrap() + } + + #[inline] + fn gpr_mem_new(&mut self, rm: &RegMem) -> GprMem { + GprMem::new(rm.clone()).unwrap() + } + + #[inline] + fn reg_to_gpr_mem(&mut self, r: Reg) -> GprMem { + GprMem::new(RegMem::reg(r)).unwrap() + } } // Since x64 doesn't have 8x16 shifts and we must use a 16x8 shift instead, we diff --git a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest index 98720addfbd4..2c8fb4b29805 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest +++ b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest @@ -1,4 +1,4 @@ src/clif.isle 9ea75a6f790b5c03 -src/prelude.isle 2bfcafbef6b29358 -src/isa/x64/inst.isle bbb6a3d201200cc8 -src/isa/x64/lower.isle 82db7f7d47ac7809 +src/prelude.isle 6aaf8ce0f5a5c2ec +src/isa/x64/inst.isle 2f76eb1f9ecf0c5e +src/isa/x64/lower.isle 144c33c4e64a17a7 diff --git a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs index 89e9f0f9a27e..31aaea3e4844 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs +++ b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs @@ -70,11 +70,31 @@ pub trait Context { fn put_in_reg_mem_imm(&mut self, arg0: Value) -> RegMemImm; fn put_in_reg_mem(&mut self, arg0: Value) -> RegMem; fn synthetic_amode_to_reg_mem(&mut self, arg0: &SyntheticAmode) -> RegMem; - fn amode_imm_reg_reg_shift(&mut self, arg0: u32, arg1: Reg, arg2: Reg, arg3: u8) -> Amode; + fn amode_imm_reg_reg_shift(&mut self, arg0: u32, arg1: Gpr, arg2: Gpr, arg3: u8) -> Amode; fn amode_to_synthetic_amode(&mut self, arg0: &Amode) -> SyntheticAmode; fn put_masked_in_imm8_reg(&mut self, arg0: Value, arg1: Type) -> Imm8Reg; fn encode_fcmp_imm(&mut self, arg0: &FcmpImm) -> u8; - fn xmm0(&mut self) -> WritableReg; + fn writable_gpr_to_reg(&mut self, arg0: WritableGpr) -> WritableReg; + fn writable_xmm_to_reg(&mut self, arg0: WritableXmm) -> WritableReg; + fn writable_reg_to_xmm(&mut self, arg0: WritableReg) -> WritableXmm; + fn writable_xmm_to_xmm(&mut self, arg0: WritableXmm) -> Xmm; + fn writable_gpr_to_gpr(&mut self, arg0: WritableGpr) -> Gpr; + fn gpr_to_reg(&mut self, arg0: Gpr) -> Reg; + fn xmm_to_reg(&mut self, arg0: Xmm) -> Reg; + fn xmm_to_xmm_mem_imm(&mut self, arg0: Xmm) -> XmmMemImm; + fn temp_writable_gpr(&mut self) -> WritableGpr; + fn temp_writable_xmm(&mut self) -> WritableXmm; + fn xmm_mem_new(&mut self, arg0: &RegMem) -> XmmMem; + fn gpr_mem_imm_new(&mut self, arg0: &RegMemImm) -> GprMemImm; + fn xmm_mem_imm_new(&mut self, arg0: &RegMemImm) -> XmmMemImm; + fn xmm_to_xmm_mem(&mut self, arg0: Xmm) -> XmmMem; + fn xmm_mem_to_reg_mem(&mut self, arg0: &XmmMem) -> RegMem; + fn gpr_mem_to_reg_mem(&mut self, arg0: &GprMem) -> RegMem; + fn xmm_new(&mut self, arg0: Reg) -> Xmm; + fn gpr_new(&mut self, arg0: Reg) -> Gpr; + fn gpr_mem_new(&mut self, arg0: &RegMem) -> GprMem; + fn reg_to_gpr_mem(&mut self, arg0: Reg) -> GprMem; + fn xmm0(&mut self) -> WritableXmm; fn avx512vl_enabled(&mut self, arg0: Type) -> Option<()>; fn avx512dq_enabled(&mut self, arg0: Type) -> Option<()>; fn avx512f_enabled(&mut self, arg0: Type) -> Option<()>; @@ -93,26 +113,337 @@ pub trait Context { fn sse_insertps_lane_imm(&mut self, arg0: u8) -> u8; } -/// Internal type SideEffectNoResult: defined at src/prelude.isle line 295. +/// Internal type SideEffectNoResult: defined at src/prelude.isle line 307. #[derive(Clone, Debug)] pub enum SideEffectNoResult { Inst { inst: MInst }, } -/// Internal type ProducesFlags: defined at src/prelude.isle line 314. +/// Internal type ProducesFlags: defined at src/prelude.isle line 326. #[derive(Clone, Debug)] pub enum ProducesFlags { ProducesFlags { inst: MInst, result: Reg }, } -/// Internal type ConsumesFlags: defined at src/prelude.isle line 317. +/// Internal type ConsumesFlags: defined at src/prelude.isle line 329. #[derive(Clone, Debug)] pub enum ConsumesFlags { ConsumesFlags { inst: MInst, result: Reg }, } -/// Internal type ExtendKind: defined at src/isa/x64/inst.isle line 464. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Internal type MInst: defined at src/isa/x64/inst.isle line 8. +#[derive(Clone)] +pub enum MInst { + Nop { + len: u8, + }, + AluRmiR { + size: OperandSize, + op: AluRmiROpcode, + src1: Reg, + src2: RegMemImm, + dst: WritableReg, + }, + UnaryRmR { + size: OperandSize, + op: UnaryRmROpcode, + src: GprMem, + dst: WritableGpr, + }, + Not { + size: OperandSize, + src: Gpr, + dst: WritableGpr, + }, + Neg { + size: OperandSize, + src: Gpr, + dst: WritableGpr, + }, + Div { + size: OperandSize, + signed: bool, + divisor: RegMem, + dividend: Reg, + dst_quotient: WritableReg, + dst_remainder: WritableReg, + }, + MulHi { + size: OperandSize, + signed: bool, + src1: Reg, + src2: RegMem, + dst_lo: WritableReg, + dst_hi: WritableReg, + }, + CheckedDivOrRemSeq { + kind: DivOrRemKind, + size: OperandSize, + dividend: Reg, + divisor: WritableReg, + dst_quotient: WritableReg, + dst_remainder: WritableReg, + tmp: OptionWritableReg, + }, + SignExtendData { + size: OperandSize, + src: Reg, + dst: WritableReg, + }, + Imm { + dst_size: OperandSize, + simm64: u64, + dst: WritableReg, + }, + MovRR { + size: OperandSize, + src: Reg, + dst: WritableReg, + }, + MovzxRmR { + ext_mode: ExtMode, + src: RegMem, + dst: WritableReg, + }, + Mov64MR { + src: SyntheticAmode, + dst: WritableReg, + }, + LoadEffectiveAddress { + addr: SyntheticAmode, + dst: WritableGpr, + }, + MovsxRmR { + ext_mode: ExtMode, + src: RegMem, + dst: WritableReg, + }, + MovRM { + size: OperandSize, + src: Reg, + dst: SyntheticAmode, + }, + ShiftR { + size: OperandSize, + kind: ShiftKind, + src: Reg, + num_bits: Imm8Reg, + dst: WritableReg, + }, + XmmRmiReg { + opcode: SseOpcode, + src1: Xmm, + src2: XmmMemImm, + dst: WritableXmm, + }, + CmpRmiR { + size: OperandSize, + opcode: CmpOpcode, + src: RegMemImm, + dst: Reg, + }, + Setcc { + cc: CC, + dst: WritableReg, + }, + Cmove { + size: OperandSize, + cc: CC, + consequent: RegMem, + alternative: Reg, + dst: WritableReg, + }, + Push64 { + src: RegMemImm, + }, + Pop64 { + dst: WritableReg, + }, + XmmRmR { + op: SseOpcode, + src1: Xmm, + src2: XmmMem, + dst: WritableXmm, + }, + XmmRmREvex { + op: Avx512Opcode, + src1: XmmMem, + src2: Xmm, + dst: WritableXmm, + }, + XmmUnaryRmR { + op: SseOpcode, + src: XmmMem, + dst: WritableXmm, + }, + XmmUnaryRmREvex { + op: Avx512Opcode, + src: XmmMem, + dst: WritableXmm, + }, + XmmMovRM { + op: SseOpcode, + src: Reg, + dst: SyntheticAmode, + }, + XmmLoadConst { + src: VCodeConstant, + dst: WritableReg, + ty: Type, + }, + XmmToGpr { + op: SseOpcode, + src: Xmm, + dst: WritableGpr, + dst_size: OperandSize, + }, + GprToXmm { + op: SseOpcode, + src: RegMem, + dst: WritableXmm, + src_size: OperandSize, + }, + CvtUint64ToFloatSeq { + dst_size: OperandSize, + src: WritableGpr, + dst: WritableXmm, + tmp_gpr1: WritableGpr, + tmp_gpr2: WritableGpr, + }, + CvtFloatToSintSeq { + dst_size: OperandSize, + src_size: OperandSize, + is_saturating: bool, + src: WritableXmm, + dst: WritableGpr, + tmp_gpr: WritableGpr, + tmp_xmm: WritableXmm, + }, + CvtFloatToUintSeq { + dst_size: OperandSize, + src_size: OperandSize, + is_saturating: bool, + src: WritableXmm, + dst: WritableGpr, + tmp_gpr: WritableGpr, + tmp_xmm: WritableXmm, + }, + XmmMinMaxSeq { + size: OperandSize, + is_min: bool, + lhs: Reg, + rhs_dst: WritableReg, + }, + XmmCmove { + size: OperandSize, + cc: CC, + src: RegMem, + dst: WritableReg, + }, + XmmCmpRmR { + op: SseOpcode, + src: RegMem, + dst: Reg, + }, + XmmRmRImm { + op: SseOpcode, + src1: Reg, + src2: RegMem, + dst: WritableReg, + imm: u8, + size: OperandSize, + }, + CallKnown { + dest: ExternalName, + uses: VecReg, + defs: VecWritableReg, + opcode: Opcode, + }, + CallUnknown { + dest: RegMem, + uses: VecReg, + defs: VecWritableReg, + opcode: Opcode, + }, + Ret, + EpiloguePlaceholder, + JmpKnown { + dst: MachLabel, + }, + JmpIf { + cc: CC, + taken: MachLabel, + }, + JmpCond { + cc: CC, + taken: MachLabel, + not_taken: MachLabel, + }, + JmpTableSeq { + idx: Reg, + tmp1: WritableReg, + tmp2: WritableReg, + default_target: MachLabel, + targets: VecMachLabel, + targets_for_term: VecMachLabel, + }, + JmpUnknown { + target: RegMem, + }, + TrapIf { + cc: CC, + trap_code: TrapCode, + }, + Hlt, + Ud2 { + trap_code: TrapCode, + }, + LoadExtName { + dst: WritableReg, + name: BoxExternalName, + offset: i64, + }, + LockCmpxchg { + ty: Type, + replacement: Reg, + expected: Reg, + mem: SyntheticAmode, + dst_old: WritableReg, + }, + AtomicRmwSeq { + ty: Type, + op: AtomicRmwOp, + address: Reg, + operand: Reg, + temp: WritableReg, + dst_old: WritableReg, + }, + Fence { + kind: FenceKind, + }, + VirtualSPOffsetAdj { + offset: i64, + }, + XmmUninitializedValue { + dst: WritableXmm, + }, + ElfTlsGetAddr { + symbol: ExternalName, + }, + MachOTlsGetAddr { + symbol: ExternalName, + }, + ValueLabelMarker { + reg: Reg, + label: ValueLabel, + }, + Unwind { + inst: UnwindInst, + }, +} + +/// Internal type ExtendKind: defined at src/isa/x64/inst.isle line 1051. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum ExtendKind { Sign, Zero, @@ -121,7 +452,7 @@ pub enum ExtendKind { // Generated as internal constructor for term temp_reg. pub fn constructor_temp_reg(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; - // Rule at src/prelude.isle line 70. + // Rule at src/prelude.isle line 73. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); return Some(expr1_0); @@ -130,7 +461,7 @@ pub fn constructor_temp_reg(ctx: &mut C, arg0: Type) -> Option // Generated as internal constructor for term lo_reg. pub fn constructor_lo_reg(ctx: &mut C, arg0: Value) -> Option { let pattern0_0 = arg0; - // Rule at src/prelude.isle line 105. + // Rule at src/prelude.isle line 108. let expr0_0 = C::put_in_regs(ctx, pattern0_0); let expr1_0: usize = 0; let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); @@ -147,7 +478,7 @@ pub fn constructor_value_regs_none( inst: ref pattern1_0, } = pattern0_0 { - // Rule at src/prelude.isle line 300. + // Rule at src/prelude.isle line 312. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::value_regs_invalid(ctx); return Some(expr1_0); @@ -165,7 +496,7 @@ pub fn constructor_safepoint( inst: ref pattern1_0, } = pattern0_0 { - // Rule at src/prelude.isle line 306. + // Rule at src/prelude.isle line 318. let expr0_0 = C::emit_safepoint(ctx, &pattern1_0); let expr1_0 = C::value_regs_invalid(ctx); return Some(expr1_0); @@ -191,7 +522,7 @@ pub fn constructor_with_flags( result: pattern3_1, } = pattern2_0 { - // Rule at src/prelude.isle line 327. + // Rule at src/prelude.isle line 339. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern3_0); let expr2_0 = C::value_regs(ctx, pattern1_1, pattern3_1); @@ -219,7 +550,7 @@ pub fn constructor_with_flags_1( result: pattern3_1, } = pattern2_0 { - // Rule at src/prelude.isle line 335. + // Rule at src/prelude.isle line 347. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern3_0); return Some(pattern3_1); @@ -253,7 +584,7 @@ pub fn constructor_with_flags_2( result: pattern5_1, } = pattern4_0 { - // Rule at src/prelude.isle line 345. + // Rule at src/prelude.isle line 357. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern5_0); let expr2_0 = C::emit(ctx, &pattern3_0); @@ -270,22 +601,22 @@ pub fn constructor_operand_size_bits(ctx: &mut C, arg0: &OperandSize let pattern0_0 = arg0; match pattern0_0 { &OperandSize::Size8 => { - // Rule at src/isa/x64/inst.isle line 98. + // Rule at src/isa/x64/inst.isle line 509. let expr0_0: u16 = 8; return Some(expr0_0); } &OperandSize::Size16 => { - // Rule at src/isa/x64/inst.isle line 99. + // Rule at src/isa/x64/inst.isle line 510. let expr0_0: u16 = 16; return Some(expr0_0); } &OperandSize::Size32 => { - // Rule at src/isa/x64/inst.isle line 100. + // Rule at src/isa/x64/inst.isle line 511. let expr0_0: u16 = 32; return Some(expr0_0); } &OperandSize::Size64 => { - // Rule at src/isa/x64/inst.isle line 101. + // Rule at src/isa/x64/inst.isle line 512. let expr0_0: u16 = 64; return Some(expr0_0); } @@ -294,6 +625,89 @@ pub fn constructor_operand_size_bits(ctx: &mut C, arg0: &OperandSize return None; } +// Generated as internal constructor for term put_in_gpr. +pub fn constructor_put_in_gpr(ctx: &mut C, arg0: Value) -> Option { + let pattern0_0 = arg0; + // Rule at src/isa/x64/inst.isle line 934. + let expr0_0 = C::put_in_reg(ctx, pattern0_0); + let expr1_0 = C::gpr_new(ctx, expr0_0); + return Some(expr1_0); +} + +// Generated as internal constructor for term put_in_gpr_mem. +pub fn constructor_put_in_gpr_mem(ctx: &mut C, arg0: Value) -> Option { + let pattern0_0 = arg0; + // Rule at src/isa/x64/inst.isle line 941. + let expr0_0 = C::put_in_reg_mem(ctx, pattern0_0); + let expr1_0 = C::gpr_mem_new(ctx, &expr0_0); + return Some(expr1_0); +} + +// Generated as internal constructor for term put_in_gpr_mem_imm. +pub fn constructor_put_in_gpr_mem_imm(ctx: &mut C, arg0: Value) -> Option { + let pattern0_0 = arg0; + // Rule at src/isa/x64/inst.isle line 948. + let expr0_0 = C::put_in_reg_mem_imm(ctx, pattern0_0); + let expr1_0 = C::gpr_mem_imm_new(ctx, &expr0_0); + return Some(expr1_0); +} + +// Generated as internal constructor for term put_in_xmm. +pub fn constructor_put_in_xmm(ctx: &mut C, arg0: Value) -> Option { + let pattern0_0 = arg0; + // Rule at src/isa/x64/inst.isle line 955. + let expr0_0 = C::put_in_reg(ctx, pattern0_0); + let expr1_0 = C::xmm_new(ctx, expr0_0); + return Some(expr1_0); +} + +// Generated as internal constructor for term put_in_xmm_mem. +pub fn constructor_put_in_xmm_mem(ctx: &mut C, arg0: Value) -> Option { + let pattern0_0 = arg0; + // Rule at src/isa/x64/inst.isle line 962. + let expr0_0 = C::put_in_reg_mem(ctx, pattern0_0); + let expr1_0 = C::xmm_mem_new(ctx, &expr0_0); + return Some(expr1_0); +} + +// Generated as internal constructor for term put_in_xmm_mem_imm. +pub fn constructor_put_in_xmm_mem_imm(ctx: &mut C, arg0: Value) -> Option { + let pattern0_0 = arg0; + // Rule at src/isa/x64/inst.isle line 969. + let expr0_0 = C::put_in_reg_mem_imm(ctx, pattern0_0); + let expr1_0 = C::xmm_mem_imm_new(ctx, &expr0_0); + return Some(expr1_0); +} + +// Generated as internal constructor for term value_gpr. +pub fn constructor_value_gpr(ctx: &mut C, arg0: Gpr) -> Option { + let pattern0_0 = arg0; + // Rule at src/isa/x64/inst.isle line 974. + let expr0_0 = C::gpr_to_reg(ctx, pattern0_0); + let expr1_0 = C::value_reg(ctx, expr0_0); + return Some(expr1_0); +} + +// Generated as internal constructor for term value_gprs. +pub fn constructor_value_gprs(ctx: &mut C, arg0: Gpr, arg1: Gpr) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 979. + let expr0_0 = C::gpr_to_reg(ctx, pattern0_0); + let expr1_0 = C::gpr_to_reg(ctx, pattern1_0); + let expr2_0 = C::value_regs(ctx, expr0_0, expr1_0); + return Some(expr2_0); +} + +// Generated as internal constructor for term value_xmm. +pub fn constructor_value_xmm(ctx: &mut C, arg0: Xmm) -> Option { + let pattern0_0 = arg0; + // Rule at src/isa/x64/inst.isle line 984. + let expr0_0 = C::xmm_to_reg(ctx, pattern0_0); + let expr1_0 = C::value_reg(ctx, expr0_0); + return Some(expr1_0); +} + // Generated as internal constructor for term extend_to_reg. pub fn constructor_extend_to_reg( ctx: &mut C, @@ -306,12 +720,12 @@ pub fn constructor_extend_to_reg( let pattern2_0 = arg1; if pattern2_0 == pattern1_0 { let pattern4_0 = arg2; - // Rule at src/isa/x64/inst.isle line 476. + // Rule at src/isa/x64/inst.isle line 1063. let expr0_0 = C::put_in_reg(ctx, pattern0_0); return Some(expr0_0); } let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 479. + // Rule at src/isa/x64/inst.isle line 1066. let expr0_0 = C::ty_bits_u16(ctx, pattern1_0); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern2_0); let expr2_0 = constructor_operand_size_bits(ctx, &expr1_0)?; @@ -335,7 +749,7 @@ pub fn constructor_extend( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/inst.isle line 499. + // Rule at src/isa/x64/inst.isle line 1086. let expr0_0 = constructor_movsx(ctx, pattern2_0, pattern3_0, pattern4_0)?; return Some(expr0_0); } @@ -343,7 +757,7 @@ pub fn constructor_extend( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/inst.isle line 495. + // Rule at src/isa/x64/inst.isle line 1082. let expr0_0 = constructor_movzx(ctx, pattern2_0, pattern3_0, pattern4_0)?; return Some(expr0_0); } @@ -356,17 +770,17 @@ pub fn constructor_extend( pub fn constructor_sse_xor_op(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32X4 { - // Rule at src/isa/x64/inst.isle line 506. + // Rule at src/isa/x64/inst.isle line 1093. let expr0_0 = SseOpcode::Xorps; return Some(expr0_0); } if pattern0_0 == F64X2 { - // Rule at src/isa/x64/inst.isle line 507. + // Rule at src/isa/x64/inst.isle line 1094. let expr0_0 = SseOpcode::Xorpd; return Some(expr0_0); } if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { - // Rule at src/isa/x64/inst.isle line 508. + // Rule at src/isa/x64/inst.isle line 1095. let expr0_0 = SseOpcode::Pxor; return Some(expr0_0); } @@ -377,13 +791,13 @@ pub fn constructor_sse_xor_op(ctx: &mut C, arg0: Type) -> Option( ctx: &mut C, arg0: Type, - arg1: Reg, - arg2: &RegMem, -) -> Option { + arg1: Xmm, + arg2: &XmmMem, +) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 512. + // Rule at src/isa/x64/inst.isle line 1099. let expr0_0 = constructor_sse_xor_op(ctx, pattern0_0)?; let expr1_0 = constructor_xmm_rm_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -393,40 +807,40 @@ pub fn constructor_sse_xor( pub fn constructor_sse_cmp_op(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32X4 { - // Rule at src/isa/x64/inst.isle line 521. + // Rule at src/isa/x64/inst.isle line 1108. let expr0_0 = SseOpcode::Cmpps; return Some(expr0_0); } if pattern0_0 == F64X2 { - // Rule at src/isa/x64/inst.isle line 522. + // Rule at src/isa/x64/inst.isle line 1109. let expr0_0 = SseOpcode::Cmppd; return Some(expr0_0); } if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { if pattern1_0 == 8 { if pattern1_1 == 16 { - // Rule at src/isa/x64/inst.isle line 517. + // Rule at src/isa/x64/inst.isle line 1104. let expr0_0 = SseOpcode::Pcmpeqb; return Some(expr0_0); } } if pattern1_0 == 16 { if pattern1_1 == 8 { - // Rule at src/isa/x64/inst.isle line 518. + // Rule at src/isa/x64/inst.isle line 1105. let expr0_0 = SseOpcode::Pcmpeqw; return Some(expr0_0); } } if pattern1_0 == 32 { if pattern1_1 == 4 { - // Rule at src/isa/x64/inst.isle line 519. + // Rule at src/isa/x64/inst.isle line 1106. let expr0_0 = SseOpcode::Pcmpeqd; return Some(expr0_0); } } if pattern1_0 == 64 { if pattern1_1 == 2 { - // Rule at src/isa/x64/inst.isle line 520. + // Rule at src/isa/x64/inst.isle line 1107. let expr0_0 = SseOpcode::Pcmpeqq; return Some(expr0_0); } @@ -436,14 +850,14 @@ pub fn constructor_sse_cmp_op(ctx: &mut C, arg0: Type) -> Option(ctx: &mut C, arg0: Type) -> Option { +pub fn constructor_vector_all_ones(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 536. - let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); - let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); + // Rule at src/isa/x64/inst.isle line 1123. + let expr0_0 = C::temp_writable_xmm(ctx); + let expr1_0 = C::writable_xmm_to_xmm(ctx, expr0_0); let expr2_0: Type = I32X4; let expr3_0 = constructor_sse_cmp_op(ctx, expr2_0)?; - let expr4_0 = RegMem::Reg { reg: expr1_0 }; + let expr4_0 = C::xmm_to_xmm_mem(ctx, expr1_0); let expr5_0 = MInst::XmmRmR { op: expr3_0, src1: expr1_0, @@ -457,70 +871,71 @@ pub fn constructor_vector_all_ones(ctx: &mut C, arg0: Type) -> Optio // Generated as internal constructor for term make_i64x2_from_lanes. pub fn constructor_make_i64x2_from_lanes( ctx: &mut C, - arg0: &RegMem, - arg1: &RegMem, -) -> Option { + arg0: &GprMem, + arg1: &GprMem, +) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 547. - let expr0_0: Type = I64X2; - let expr1_0 = C::temp_writable_reg(ctx, expr0_0); - let expr2_0 = C::writable_reg_to_reg(ctx, expr1_0); - let expr3_0 = MInst::XmmUninitializedValue { dst: expr1_0 }; - let expr4_0 = C::emit(ctx, &expr3_0); - let expr5_0 = SseOpcode::Pinsrd; - let expr6_0: u8 = 0; - let expr7_0 = OperandSize::Size64; - let expr8_0 = MInst::XmmRmRImm { - op: expr5_0, - src1: expr2_0, - src2: pattern0_0.clone(), + // Rule at src/isa/x64/inst.isle line 1134. + let expr0_0 = C::temp_writable_xmm(ctx); + let expr1_0 = C::writable_xmm_to_reg(ctx, expr0_0); + let expr2_0 = C::writable_xmm_to_xmm(ctx, expr0_0); + let expr3_0 = C::xmm_to_reg(ctx, expr2_0); + let expr4_0 = MInst::XmmUninitializedValue { dst: expr0_0 }; + let expr5_0 = C::emit(ctx, &expr4_0); + let expr6_0 = SseOpcode::Pinsrd; + let expr7_0 = C::gpr_mem_to_reg_mem(ctx, pattern0_0); + let expr8_0: u8 = 0; + let expr9_0 = OperandSize::Size64; + let expr10_0 = MInst::XmmRmRImm { + op: expr6_0, + src1: expr3_0, + src2: expr7_0, dst: expr1_0, - imm: expr6_0, - size: expr7_0, + imm: expr8_0, + size: expr9_0, }; - let expr9_0 = C::emit(ctx, &expr8_0); - let expr10_0 = SseOpcode::Pinsrd; - let expr11_0: u8 = 1; - let expr12_0 = OperandSize::Size64; - let expr13_0 = MInst::XmmRmRImm { - op: expr10_0, - src1: expr2_0, - src2: pattern1_0.clone(), + let expr11_0 = C::emit(ctx, &expr10_0); + let expr12_0 = SseOpcode::Pinsrd; + let expr13_0 = C::gpr_mem_to_reg_mem(ctx, pattern1_0); + let expr14_0: u8 = 1; + let expr15_0 = OperandSize::Size64; + let expr16_0 = MInst::XmmRmRImm { + op: expr12_0, + src1: expr3_0, + src2: expr13_0, dst: expr1_0, - imm: expr11_0, - size: expr12_0, + imm: expr14_0, + size: expr15_0, }; - let expr14_0 = C::emit(ctx, &expr13_0); + let expr17_0 = C::emit(ctx, &expr16_0); return Some(expr2_0); } -// Generated as internal constructor for term reg_mem_imm_to_xmm. -pub fn constructor_reg_mem_imm_to_xmm( - ctx: &mut C, - arg0: &RegMemImm, -) -> Option { +// Generated as internal constructor for term mov_rmi_to_xmm. +pub fn constructor_mov_rmi_to_xmm(ctx: &mut C, arg0: &RegMemImm) -> Option { let pattern0_0 = arg0; match pattern0_0 { &RegMemImm::Imm { simm32: pattern1_0 } => { - // Rule at src/isa/x64/inst.isle line 568. - return Some(pattern0_0.clone()); + // Rule at src/isa/x64/inst.isle line 1157. + let expr0_0 = C::xmm_mem_imm_new(ctx, pattern0_0); + return Some(expr0_0); } &RegMemImm::Reg { reg: pattern1_0 } => { - // Rule at src/isa/x64/inst.isle line 569. - let expr0_0: Type = I8X16; - let expr1_0 = SseOpcode::Movd; - let expr2_0 = RegMem::Reg { reg: pattern1_0 }; - let expr3_0 = OperandSize::Size32; - let expr4_0 = constructor_gpr_to_xmm(ctx, expr0_0, &expr1_0, &expr2_0, &expr3_0)?; - let expr5_0 = RegMemImm::Reg { reg: expr4_0 }; - return Some(expr5_0); + // Rule at src/isa/x64/inst.isle line 1158. + let expr0_0 = SseOpcode::Movd; + let expr1_0 = C::reg_to_gpr_mem(ctx, pattern1_0); + let expr2_0 = OperandSize::Size32; + let expr3_0 = constructor_gpr_to_xmm(ctx, &expr0_0, &expr1_0, &expr2_0)?; + let expr4_0 = C::xmm_to_xmm_mem_imm(ctx, expr3_0); + return Some(expr4_0); } &RegMemImm::Mem { addr: ref pattern1_0, } => { - // Rule at src/isa/x64/inst.isle line 567. - return Some(pattern0_0.clone()); + // Rule at src/isa/x64/inst.isle line 1156. + let expr0_0 = C::xmm_mem_imm_new(ctx, pattern0_0); + return Some(expr0_0); } _ => {} } @@ -538,7 +953,7 @@ pub fn constructor_x64_load( if pattern0_0 == I64 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 585. + // Rule at src/isa/x64/inst.isle line 1173. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::Mov64MR { @@ -552,53 +967,63 @@ pub fn constructor_x64_load( if pattern0_0 == F32 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 590. + // Rule at src/isa/x64/inst.isle line 1178. let expr0_0 = SseOpcode::Movss; let expr1_0 = C::synthetic_amode_to_reg_mem(ctx, pattern2_0); - let expr2_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, &expr1_0)?; - return Some(expr2_0); + let expr2_0 = C::xmm_mem_new(ctx, &expr1_0); + let expr3_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, &expr2_0)?; + let expr4_0 = C::xmm_to_reg(ctx, expr3_0); + return Some(expr4_0); } if pattern0_0 == F64 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 594. + // Rule at src/isa/x64/inst.isle line 1182. let expr0_0 = SseOpcode::Movsd; let expr1_0 = C::synthetic_amode_to_reg_mem(ctx, pattern2_0); - let expr2_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, &expr1_0)?; - return Some(expr2_0); + let expr2_0 = C::xmm_mem_new(ctx, &expr1_0); + let expr3_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, &expr2_0)?; + let expr4_0 = C::xmm_to_reg(ctx, expr3_0); + return Some(expr4_0); } if pattern0_0 == F32X4 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 598. + // Rule at src/isa/x64/inst.isle line 1186. let expr0_0 = SseOpcode::Movups; let expr1_0 = C::synthetic_amode_to_reg_mem(ctx, pattern2_0); - let expr2_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, &expr1_0)?; - return Some(expr2_0); + let expr2_0 = C::xmm_mem_new(ctx, &expr1_0); + let expr3_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, &expr2_0)?; + let expr4_0 = C::xmm_to_reg(ctx, expr3_0); + return Some(expr4_0); } if pattern0_0 == F64X2 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 602. + // Rule at src/isa/x64/inst.isle line 1190. let expr0_0 = SseOpcode::Movupd; let expr1_0 = C::synthetic_amode_to_reg_mem(ctx, pattern2_0); - let expr2_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, &expr1_0)?; - return Some(expr2_0); + let expr2_0 = C::xmm_mem_new(ctx, &expr1_0); + let expr3_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, &expr2_0)?; + let expr4_0 = C::xmm_to_reg(ctx, expr3_0); + return Some(expr4_0); } if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 606. + // Rule at src/isa/x64/inst.isle line 1194. let expr0_0 = SseOpcode::Movdqu; let expr1_0 = C::synthetic_amode_to_reg_mem(ctx, pattern2_0); - let expr2_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, &expr1_0)?; - return Some(expr2_0); + let expr2_0 = C::xmm_mem_new(ctx, &expr1_0); + let expr3_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, &expr2_0)?; + let expr4_0 = C::xmm_to_reg(ctx, expr3_0); + return Some(expr4_0); } if let Some(pattern1_0) = C::fits_in_32(ctx, pattern0_0) { let pattern2_0 = arg1; let pattern3_0 = arg2; if let &ExtKind::SignExtend = pattern3_0 { - // Rule at src/isa/x64/inst.isle line 580. + // Rule at src/isa/x64/inst.isle line 1168. let expr0_0 = C::ty_bytes(ctx, pattern1_0); let expr1_0: u16 = 8; let expr2_0 = C::ext_mode(ctx, expr0_0, expr1_0); @@ -622,7 +1047,7 @@ pub fn constructor_alu_rmi_r( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 618. + // Rule at src/isa/x64/inst.isle line 1206. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = MInst::AluRmiR { @@ -647,7 +1072,7 @@ pub fn constructor_add( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 626. + // Rule at src/isa/x64/inst.isle line 1214. let expr0_0 = AluRmiROpcode::Add; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -663,7 +1088,7 @@ pub fn constructor_add_with_flags( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 634. + // Rule at src/isa/x64/inst.isle line 1222. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = AluRmiROpcode::Add; @@ -692,7 +1117,7 @@ pub fn constructor_adc( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 645. + // Rule at src/isa/x64/inst.isle line 1233. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = AluRmiROpcode::Adc; @@ -721,7 +1146,7 @@ pub fn constructor_sub( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 656. + // Rule at src/isa/x64/inst.isle line 1244. let expr0_0 = AluRmiROpcode::Sub; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -737,7 +1162,7 @@ pub fn constructor_sub_with_flags( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 664. + // Rule at src/isa/x64/inst.isle line 1252. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = AluRmiROpcode::Sub; @@ -766,7 +1191,7 @@ pub fn constructor_sbb( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 675. + // Rule at src/isa/x64/inst.isle line 1263. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = AluRmiROpcode::Sbb; @@ -795,7 +1220,7 @@ pub fn constructor_mul( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 686. + // Rule at src/isa/x64/inst.isle line 1274. let expr0_0 = AluRmiROpcode::Mul; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -811,7 +1236,7 @@ pub fn constructor_x64_and( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 697. + // Rule at src/isa/x64/inst.isle line 1285. let expr0_0 = AluRmiROpcode::And; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -827,7 +1252,7 @@ pub fn constructor_or( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 705. + // Rule at src/isa/x64/inst.isle line 1293. let expr0_0 = AluRmiROpcode::Or; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -843,7 +1268,7 @@ pub fn constructor_xor( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 713. + // Rule at src/isa/x64/inst.isle line 1301. let expr0_0 = AluRmiROpcode::Xor; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -855,7 +1280,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option if pattern0_0 == I64 { let pattern2_0 = arg1; if let Some(pattern3_0) = C::nonzero_u64_fits_in_u32(ctx, pattern2_0) { - // Rule at src/isa/x64/inst.isle line 742. + // Rule at src/isa/x64/inst.isle line 1334. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = OperandSize::Size32; @@ -872,67 +1297,69 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option if pattern0_0 == F32 { let pattern2_0 = arg1; if pattern2_0 == 0 { - // Rule at src/isa/x64/inst.isle line 771. - let expr0_0: Type = F32; - let expr1_0 = C::temp_writable_reg(ctx, expr0_0); - let expr2_0 = C::writable_reg_to_reg(ctx, expr1_0); - let expr3_0 = SseOpcode::Xorps; - let expr4_0 = RegMem::Reg { reg: expr2_0 }; - let expr5_0 = MInst::XmmRmR { - op: expr3_0, - src1: expr2_0, - src2: expr4_0, - dst: expr1_0, + // Rule at src/isa/x64/inst.isle line 1363. + let expr0_0 = C::temp_writable_xmm(ctx); + let expr1_0 = C::writable_xmm_to_xmm(ctx, expr0_0); + let expr2_0 = SseOpcode::Xorps; + let expr3_0 = C::xmm_to_xmm_mem(ctx, expr1_0); + let expr4_0 = MInst::XmmRmR { + op: expr2_0, + src1: expr1_0, + src2: expr3_0, + dst: expr0_0, }; - let expr6_0 = C::emit(ctx, &expr5_0); - return Some(expr2_0); + let expr5_0 = C::emit(ctx, &expr4_0); + let expr6_0 = C::xmm_to_reg(ctx, expr1_0); + return Some(expr6_0); } - // Rule at src/isa/x64/inst.isle line 730. - let expr0_0: Type = F32; - let expr1_0 = SseOpcode::Movd; - let expr2_0: Type = I32; - let expr3_0 = constructor_imm(ctx, expr2_0, pattern2_0)?; - let expr4_0 = RegMem::Reg { reg: expr3_0 }; + // Rule at src/isa/x64/inst.isle line 1318. + let expr0_0 = SseOpcode::Movd; + let expr1_0: Type = I32; + let expr2_0 = constructor_imm(ctx, expr1_0, pattern2_0)?; + let expr3_0 = RegMem::Reg { reg: expr2_0 }; + let expr4_0 = C::gpr_mem_new(ctx, &expr3_0); let expr5_0 = OperandSize::Size32; - let expr6_0 = constructor_gpr_to_xmm(ctx, expr0_0, &expr1_0, &expr4_0, &expr5_0)?; - return Some(expr6_0); + let expr6_0 = constructor_gpr_to_xmm(ctx, &expr0_0, &expr4_0, &expr5_0)?; + let expr7_0 = C::xmm_to_reg(ctx, expr6_0); + return Some(expr7_0); } if pattern0_0 == F64 { let pattern2_0 = arg1; if pattern2_0 == 0 { - // Rule at src/isa/x64/inst.isle line 783. - let expr0_0: Type = F64; - let expr1_0 = C::temp_writable_reg(ctx, expr0_0); - let expr2_0 = C::writable_reg_to_reg(ctx, expr1_0); - let expr3_0 = SseOpcode::Xorpd; - let expr4_0 = RegMem::Reg { reg: expr2_0 }; - let expr5_0 = MInst::XmmRmR { - op: expr3_0, - src1: expr2_0, - src2: expr4_0, - dst: expr1_0, + // Rule at src/isa/x64/inst.isle line 1375. + let expr0_0 = C::temp_writable_xmm(ctx); + let expr1_0 = C::writable_xmm_to_xmm(ctx, expr0_0); + let expr2_0 = SseOpcode::Xorpd; + let expr3_0 = C::xmm_to_xmm_mem(ctx, expr1_0); + let expr4_0 = MInst::XmmRmR { + op: expr2_0, + src1: expr1_0, + src2: expr3_0, + dst: expr0_0, }; - let expr6_0 = C::emit(ctx, &expr5_0); - return Some(expr2_0); + let expr5_0 = C::emit(ctx, &expr4_0); + let expr6_0 = C::xmm_to_reg(ctx, expr1_0); + return Some(expr6_0); } - // Rule at src/isa/x64/inst.isle line 734. - let expr0_0: Type = F64; - let expr1_0 = SseOpcode::Movq; - let expr2_0: Type = I64; - let expr3_0 = constructor_imm(ctx, expr2_0, pattern2_0)?; - let expr4_0 = RegMem::Reg { reg: expr3_0 }; + // Rule at src/isa/x64/inst.isle line 1324. + let expr0_0 = SseOpcode::Movq; + let expr1_0: Type = I64; + let expr2_0 = constructor_imm(ctx, expr1_0, pattern2_0)?; + let expr3_0 = RegMem::Reg { reg: expr2_0 }; + let expr4_0 = C::gpr_mem_new(ctx, &expr3_0); let expr5_0 = OperandSize::Size64; - let expr6_0 = constructor_gpr_to_xmm(ctx, expr0_0, &expr1_0, &expr4_0, &expr5_0)?; - return Some(expr6_0); + let expr6_0 = constructor_gpr_to_xmm(ctx, &expr0_0, &expr4_0, &expr5_0)?; + let expr7_0 = C::xmm_to_reg(ctx, expr6_0); + return Some(expr7_0); } if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { let pattern2_0 = arg1; if pattern2_0 == 0 { - // Rule at src/isa/x64/inst.isle line 761. - let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); - let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); + // Rule at src/isa/x64/inst.isle line 1353. + let expr0_0 = C::temp_writable_xmm(ctx); + let expr1_0 = C::writable_xmm_to_xmm(ctx, expr0_0); let expr2_0 = constructor_sse_xor_op(ctx, pattern0_0)?; - let expr3_0 = RegMem::Reg { reg: expr1_0 }; + let expr3_0 = C::xmm_to_xmm_mem(ctx, expr1_0); let expr4_0 = MInst::XmmRmR { op: expr2_0, src1: expr1_0, @@ -940,13 +1367,14 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option dst: expr0_0, }; let expr5_0 = C::emit(ctx, &expr4_0); - return Some(expr1_0); + let expr6_0 = C::xmm_to_reg(ctx, expr1_0); + return Some(expr6_0); } } if let Some(pattern1_0) = C::fits_in_64(ctx, pattern0_0) { let pattern2_0 = arg1; if pattern2_0 == 0 { - // Rule at src/isa/x64/inst.isle line 748. + // Rule at src/isa/x64/inst.isle line 1340. let expr0_0 = C::temp_writable_reg(ctx, pattern1_0); let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); let expr2_0 = C::operand_size_of_type_32_64(ctx, pattern1_0); @@ -962,7 +1390,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option let expr6_0 = C::emit(ctx, &expr5_0); return Some(expr1_0); } - // Rule at src/isa/x64/inst.isle line 723. + // Rule at src/isa/x64/inst.isle line 1311. let expr0_0 = C::temp_writable_reg(ctx, pattern1_0); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern1_0); let expr2_0 = MInst::Imm { @@ -989,7 +1417,7 @@ pub fn constructor_shift_r( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 796. + // Rule at src/isa/x64/inst.isle line 1388. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::raw_operand_size_of_type(ctx, pattern0_0); let expr2_0 = MInst::ShiftR { @@ -1014,7 +1442,7 @@ pub fn constructor_x64_rotl( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 807. + // Rule at src/isa/x64/inst.isle line 1399. let expr0_0 = ShiftKind::RotateLeft; let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -1030,7 +1458,7 @@ pub fn constructor_x64_rotr( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 813. + // Rule at src/isa/x64/inst.isle line 1405. let expr0_0 = ShiftKind::RotateRight; let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -1046,7 +1474,7 @@ pub fn constructor_shl( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 818. + // Rule at src/isa/x64/inst.isle line 1410. let expr0_0 = ShiftKind::ShiftLeft; let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -1062,7 +1490,7 @@ pub fn constructor_shr( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 823. + // Rule at src/isa/x64/inst.isle line 1415. let expr0_0 = ShiftKind::ShiftRightLogical; let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -1078,7 +1506,7 @@ pub fn constructor_sar( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 828. + // Rule at src/isa/x64/inst.isle line 1420. let expr0_0 = ShiftKind::ShiftRightArithmetic; let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -1096,7 +1524,7 @@ pub fn constructor_cmp_rmi_r( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 833. + // Rule at src/isa/x64/inst.isle line 1425. let expr0_0 = MInst::CmpRmiR { size: pattern0_0.clone(), opcode: pattern1_0.clone(), @@ -1121,7 +1549,7 @@ pub fn constructor_cmp( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 842. + // Rule at src/isa/x64/inst.isle line 1434. let expr0_0 = CmpOpcode::Cmp; let expr1_0 = constructor_cmp_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -1137,7 +1565,7 @@ pub fn constructor_test( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 847. + // Rule at src/isa/x64/inst.isle line 1439. let expr0_0 = CmpOpcode::Test; let expr1_0 = constructor_cmp_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -1155,7 +1583,7 @@ pub fn constructor_cmove( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 852. + // Rule at src/isa/x64/inst.isle line 1444. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = MInst::Cmove { @@ -1183,7 +1611,7 @@ pub fn constructor_movzx( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 860. + // Rule at src/isa/x64/inst.isle line 1452. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::MovzxRmR { ext_mode: pattern1_0.clone(), @@ -1205,7 +1633,7 @@ pub fn constructor_movsx( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 867. + // Rule at src/isa/x64/inst.isle line 1459. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::MovsxRmR { ext_mode: pattern1_0.clone(), @@ -1222,15 +1650,15 @@ pub fn constructor_xmm_rm_r( ctx: &mut C, arg0: Type, arg1: &SseOpcode, - arg2: Reg, - arg3: &RegMem, -) -> Option { + arg2: Xmm, + arg3: &XmmMem, +) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 874. - let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); + // Rule at src/isa/x64/inst.isle line 1466. + let expr0_0 = C::temp_writable_xmm(ctx); let expr1_0 = MInst::XmmRmR { op: pattern1_0.clone(), src1: pattern2_0, @@ -1238,15 +1666,15 @@ pub fn constructor_xmm_rm_r( dst: expr0_0, }; let expr2_0 = C::emit(ctx, &expr1_0); - let expr3_0 = C::writable_reg_to_reg(ctx, expr0_0); + let expr3_0 = C::writable_xmm_to_xmm(ctx, expr0_0); return Some(expr3_0); } // Generated as internal constructor for term paddb. -pub fn constructor_paddb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_paddb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 881. + // Rule at src/isa/x64/inst.isle line 1473. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Paddb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1254,10 +1682,10 @@ pub fn constructor_paddb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O } // Generated as internal constructor for term paddw. -pub fn constructor_paddw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_paddw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 886. + // Rule at src/isa/x64/inst.isle line 1478. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Paddw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1265,10 +1693,10 @@ pub fn constructor_paddw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O } // Generated as internal constructor for term paddd. -pub fn constructor_paddd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_paddd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 891. + // Rule at src/isa/x64/inst.isle line 1483. let expr0_0: Type = I32X4; let expr1_0 = SseOpcode::Paddd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1276,10 +1704,10 @@ pub fn constructor_paddd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O } // Generated as internal constructor for term paddq. -pub fn constructor_paddq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_paddq(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 896. + // Rule at src/isa/x64/inst.isle line 1488. let expr0_0: Type = I64X2; let expr1_0 = SseOpcode::Paddq; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1287,10 +1715,10 @@ pub fn constructor_paddq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O } // Generated as internal constructor for term paddsb. -pub fn constructor_paddsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_paddsb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 901. + // Rule at src/isa/x64/inst.isle line 1493. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Paddsb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1298,10 +1726,10 @@ pub fn constructor_paddsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term paddsw. -pub fn constructor_paddsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_paddsw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 906. + // Rule at src/isa/x64/inst.isle line 1498. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Paddsw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1309,10 +1737,10 @@ pub fn constructor_paddsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term paddusb. -pub fn constructor_paddusb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_paddusb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 911. + // Rule at src/isa/x64/inst.isle line 1503. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Paddusb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1320,10 +1748,10 @@ pub fn constructor_paddusb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term paddusw. -pub fn constructor_paddusw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_paddusw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 916. + // Rule at src/isa/x64/inst.isle line 1508. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Paddusw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1331,10 +1759,10 @@ pub fn constructor_paddusw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term psubb. -pub fn constructor_psubb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_psubb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 921. + // Rule at src/isa/x64/inst.isle line 1513. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Psubb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1342,10 +1770,10 @@ pub fn constructor_psubb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O } // Generated as internal constructor for term psubw. -pub fn constructor_psubw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_psubw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 926. + // Rule at src/isa/x64/inst.isle line 1518. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Psubw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1353,10 +1781,10 @@ pub fn constructor_psubw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O } // Generated as internal constructor for term psubd. -pub fn constructor_psubd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_psubd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 931. + // Rule at src/isa/x64/inst.isle line 1523. let expr0_0: Type = I32X4; let expr1_0 = SseOpcode::Psubd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1364,10 +1792,10 @@ pub fn constructor_psubd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O } // Generated as internal constructor for term psubq. -pub fn constructor_psubq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_psubq(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 936. + // Rule at src/isa/x64/inst.isle line 1528. let expr0_0: Type = I64X2; let expr1_0 = SseOpcode::Psubq; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1375,10 +1803,10 @@ pub fn constructor_psubq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O } // Generated as internal constructor for term psubsb. -pub fn constructor_psubsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_psubsb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 941. + // Rule at src/isa/x64/inst.isle line 1533. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Psubsb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1386,10 +1814,10 @@ pub fn constructor_psubsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term psubsw. -pub fn constructor_psubsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_psubsw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 946. + // Rule at src/isa/x64/inst.isle line 1538. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Psubsw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1397,10 +1825,10 @@ pub fn constructor_psubsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term psubusb. -pub fn constructor_psubusb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_psubusb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 951. + // Rule at src/isa/x64/inst.isle line 1543. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Psubusb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1408,10 +1836,10 @@ pub fn constructor_psubusb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term psubusw. -pub fn constructor_psubusw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_psubusw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 956. + // Rule at src/isa/x64/inst.isle line 1548. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Psubusw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1419,10 +1847,10 @@ pub fn constructor_psubusw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term pavgb. -pub fn constructor_pavgb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pavgb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 961. + // Rule at src/isa/x64/inst.isle line 1553. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pavgb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1430,10 +1858,10 @@ pub fn constructor_pavgb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O } // Generated as internal constructor for term pavgw. -pub fn constructor_pavgw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pavgw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 966. + // Rule at src/isa/x64/inst.isle line 1558. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pavgw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1441,10 +1869,10 @@ pub fn constructor_pavgw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O } // Generated as internal constructor for term pand. -pub fn constructor_pand(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pand(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 971. + // Rule at src/isa/x64/inst.isle line 1563. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Pand; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1452,10 +1880,10 @@ pub fn constructor_pand(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Op } // Generated as internal constructor for term andps. -pub fn constructor_andps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_andps(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 976. + // Rule at src/isa/x64/inst.isle line 1568. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Andps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1463,10 +1891,10 @@ pub fn constructor_andps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O } // Generated as internal constructor for term andpd. -pub fn constructor_andpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_andpd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 981. + // Rule at src/isa/x64/inst.isle line 1573. let expr0_0: Type = F64X2; let expr1_0 = SseOpcode::Andpd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1474,10 +1902,10 @@ pub fn constructor_andpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O } // Generated as internal constructor for term por. -pub fn constructor_por(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_por(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 986. + // Rule at src/isa/x64/inst.isle line 1578. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Por; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1485,10 +1913,10 @@ pub fn constructor_por(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Opt } // Generated as internal constructor for term orps. -pub fn constructor_orps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_orps(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 991. + // Rule at src/isa/x64/inst.isle line 1583. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Orps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1496,10 +1924,10 @@ pub fn constructor_orps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Op } // Generated as internal constructor for term orpd. -pub fn constructor_orpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_orpd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 996. + // Rule at src/isa/x64/inst.isle line 1588. let expr0_0: Type = F64X2; let expr1_0 = SseOpcode::Orpd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1507,10 +1935,10 @@ pub fn constructor_orpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Op } // Generated as internal constructor for term pxor. -pub fn constructor_pxor(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pxor(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1001. + // Rule at src/isa/x64/inst.isle line 1593. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pxor; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1518,10 +1946,10 @@ pub fn constructor_pxor(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Op } // Generated as internal constructor for term xorps. -pub fn constructor_xorps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_xorps(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1006. + // Rule at src/isa/x64/inst.isle line 1598. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Xorps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1529,10 +1957,10 @@ pub fn constructor_xorps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O } // Generated as internal constructor for term xorpd. -pub fn constructor_xorpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_xorpd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1011. + // Rule at src/isa/x64/inst.isle line 1603. let expr0_0: Type = F64X2; let expr1_0 = SseOpcode::Xorpd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1540,10 +1968,10 @@ pub fn constructor_xorpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O } // Generated as internal constructor for term pmullw. -pub fn constructor_pmullw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pmullw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1016. + // Rule at src/isa/x64/inst.isle line 1608. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pmullw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1551,10 +1979,10 @@ pub fn constructor_pmullw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term pmulld. -pub fn constructor_pmulld(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pmulld(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1021. + // Rule at src/isa/x64/inst.isle line 1613. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pmulld; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1562,10 +1990,10 @@ pub fn constructor_pmulld(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term pmulhw. -pub fn constructor_pmulhw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pmulhw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1026. + // Rule at src/isa/x64/inst.isle line 1618. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pmulhw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1573,10 +2001,10 @@ pub fn constructor_pmulhw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term pmulhuw. -pub fn constructor_pmulhuw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pmulhuw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1031. + // Rule at src/isa/x64/inst.isle line 1623. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pmulhuw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1584,10 +2012,10 @@ pub fn constructor_pmulhuw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term pmuldq. -pub fn constructor_pmuldq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pmuldq(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1036. + // Rule at src/isa/x64/inst.isle line 1628. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pmuldq; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1595,10 +2023,10 @@ pub fn constructor_pmuldq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term pmuludq. -pub fn constructor_pmuludq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pmuludq(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1041. + // Rule at src/isa/x64/inst.isle line 1633. let expr0_0: Type = I64X2; let expr1_0 = SseOpcode::Pmuludq; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1606,10 +2034,10 @@ pub fn constructor_pmuludq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term punpckhwd. -pub fn constructor_punpckhwd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_punpckhwd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1046. + // Rule at src/isa/x64/inst.isle line 1638. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Punpckhwd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1617,10 +2045,10 @@ pub fn constructor_punpckhwd(ctx: &mut C, arg0: Reg, arg1: &RegMem) } // Generated as internal constructor for term punpcklwd. -pub fn constructor_punpcklwd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_punpcklwd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1051. + // Rule at src/isa/x64/inst.isle line 1643. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Punpcklwd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1628,10 +2056,10 @@ pub fn constructor_punpcklwd(ctx: &mut C, arg0: Reg, arg1: &RegMem) } // Generated as internal constructor for term andnps. -pub fn constructor_andnps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_andnps(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1056. + // Rule at src/isa/x64/inst.isle line 1648. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Andnps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1639,10 +2067,10 @@ pub fn constructor_andnps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term andnpd. -pub fn constructor_andnpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_andnpd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1061. + // Rule at src/isa/x64/inst.isle line 1653. let expr0_0: Type = F64X2; let expr1_0 = SseOpcode::Andnpd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1650,10 +2078,10 @@ pub fn constructor_andnpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term pandn. -pub fn constructor_pandn(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pandn(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1066. + // Rule at src/isa/x64/inst.isle line 1658. let expr0_0: Type = F64X2; let expr1_0 = SseOpcode::Pandn; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1664,17 +2092,17 @@ pub fn constructor_pandn(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_sse_blend_op(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32X4 { - // Rule at src/isa/x64/inst.isle line 1070. + // Rule at src/isa/x64/inst.isle line 1662. let expr0_0 = SseOpcode::Blendvps; return Some(expr0_0); } if pattern0_0 == F64X2 { - // Rule at src/isa/x64/inst.isle line 1071. + // Rule at src/isa/x64/inst.isle line 1663. let expr0_0 = SseOpcode::Blendvpd; return Some(expr0_0); } if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { - // Rule at src/isa/x64/inst.isle line 1072. + // Rule at src/isa/x64/inst.isle line 1664. let expr0_0 = SseOpcode::Pblendvb; return Some(expr0_0); } @@ -1685,17 +2113,17 @@ pub fn constructor_sse_blend_op(ctx: &mut C, arg0: Type) -> Option(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32X4 { - // Rule at src/isa/x64/inst.isle line 1075. + // Rule at src/isa/x64/inst.isle line 1667. let expr0_0 = SseOpcode::Movaps; return Some(expr0_0); } if pattern0_0 == F64X2 { - // Rule at src/isa/x64/inst.isle line 1076. + // Rule at src/isa/x64/inst.isle line 1668. let expr0_0 = SseOpcode::Movapd; return Some(expr0_0); } if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { - // Rule at src/isa/x64/inst.isle line 1077. + // Rule at src/isa/x64/inst.isle line 1669. let expr0_0 = SseOpcode::Movdqa; return Some(expr0_0); } @@ -1706,15 +2134,15 @@ pub fn constructor_sse_mov_op(ctx: &mut C, arg0: Type) -> Option( ctx: &mut C, arg0: Type, - arg1: &RegMem, - arg2: &RegMem, - arg3: Reg, -) -> Option { + arg1: &XmmMem, + arg2: &XmmMem, + arg3: Xmm, +) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1081. + // Rule at src/isa/x64/inst.isle line 1673. let expr0_0 = C::xmm0(ctx); let expr1_0 = constructor_sse_mov_op(ctx, pattern0_0)?; let expr2_0 = MInst::XmmUnaryRmR { @@ -1731,17 +2159,17 @@ pub fn constructor_sse_blend( // Generated as internal constructor for term blendvpd. pub fn constructor_blendvpd( ctx: &mut C, - arg0: Reg, - arg1: &RegMem, - arg2: Reg, -) -> Option { + arg0: Xmm, + arg1: &XmmMem, + arg2: Xmm, +) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1093. + // Rule at src/isa/x64/inst.isle line 1687. let expr0_0 = C::xmm0(ctx); let expr1_0 = SseOpcode::Movapd; - let expr2_0 = RegMem::Reg { reg: pattern2_0 }; + let expr2_0 = C::xmm_to_xmm_mem(ctx, pattern2_0); let expr3_0 = MInst::XmmUnaryRmR { op: expr1_0, src: expr2_0, @@ -1755,10 +2183,10 @@ pub fn constructor_blendvpd( } // Generated as internal constructor for term movsd. -pub fn constructor_movsd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_movsd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1105. + // Rule at src/isa/x64/inst.isle line 1701. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Movsd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1766,10 +2194,10 @@ pub fn constructor_movsd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O } // Generated as internal constructor for term movlhps. -pub fn constructor_movlhps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_movlhps(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1110. + // Rule at src/isa/x64/inst.isle line 1706. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Movlhps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1777,10 +2205,10 @@ pub fn constructor_movlhps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term pmaxsb. -pub fn constructor_pmaxsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pmaxsb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1115. + // Rule at src/isa/x64/inst.isle line 1711. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxsb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1788,10 +2216,10 @@ pub fn constructor_pmaxsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term pmaxsw. -pub fn constructor_pmaxsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pmaxsw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1120. + // Rule at src/isa/x64/inst.isle line 1716. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxsw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1799,10 +2227,10 @@ pub fn constructor_pmaxsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term pmaxsd. -pub fn constructor_pmaxsd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pmaxsd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1125. + // Rule at src/isa/x64/inst.isle line 1721. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxsd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1810,10 +2238,10 @@ pub fn constructor_pmaxsd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term pminsb. -pub fn constructor_pminsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pminsb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1130. + // Rule at src/isa/x64/inst.isle line 1726. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pminsb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1821,10 +2249,10 @@ pub fn constructor_pminsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term pminsw. -pub fn constructor_pminsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pminsw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1135. + // Rule at src/isa/x64/inst.isle line 1731. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pminsw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1832,10 +2260,10 @@ pub fn constructor_pminsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term pminsd. -pub fn constructor_pminsd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pminsd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1140. + // Rule at src/isa/x64/inst.isle line 1736. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pminsd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1843,10 +2271,10 @@ pub fn constructor_pminsd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term pmaxub. -pub fn constructor_pmaxub(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pmaxub(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1145. + // Rule at src/isa/x64/inst.isle line 1741. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxub; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1854,10 +2282,10 @@ pub fn constructor_pmaxub(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term pmaxuw. -pub fn constructor_pmaxuw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pmaxuw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1150. + // Rule at src/isa/x64/inst.isle line 1746. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxuw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1865,10 +2293,10 @@ pub fn constructor_pmaxuw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term pmaxud. -pub fn constructor_pmaxud(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pmaxud(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1155. + // Rule at src/isa/x64/inst.isle line 1751. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxud; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1876,10 +2304,10 @@ pub fn constructor_pmaxud(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term pminub. -pub fn constructor_pminub(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pminub(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1160. + // Rule at src/isa/x64/inst.isle line 1756. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pminub; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1887,10 +2315,10 @@ pub fn constructor_pminub(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term pminuw. -pub fn constructor_pminuw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pminuw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1165. + // Rule at src/isa/x64/inst.isle line 1761. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pminuw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1898,10 +2326,10 @@ pub fn constructor_pminuw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term pminud. -pub fn constructor_pminud(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_pminud(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1170. + // Rule at src/isa/x64/inst.isle line 1766. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pminud; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1909,10 +2337,10 @@ pub fn constructor_pminud(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> } // Generated as internal constructor for term punpcklbw. -pub fn constructor_punpcklbw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_punpcklbw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1175. + // Rule at src/isa/x64/inst.isle line 1771. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Punpcklbw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1920,10 +2348,10 @@ pub fn constructor_punpcklbw(ctx: &mut C, arg0: Reg, arg1: &RegMem) } // Generated as internal constructor for term punpckhbw. -pub fn constructor_punpckhbw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_punpckhbw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1180. + // Rule at src/isa/x64/inst.isle line 1776. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Punpckhbw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1931,10 +2359,10 @@ pub fn constructor_punpckhbw(ctx: &mut C, arg0: Reg, arg1: &RegMem) } // Generated as internal constructor for term packsswb. -pub fn constructor_packsswb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { +pub fn constructor_packsswb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1185. + // Rule at src/isa/x64/inst.isle line 1781. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Packsswb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1949,15 +2377,15 @@ pub fn constructor_xmm_rm_r_imm( arg2: &RegMem, arg3: u8, arg4: &OperandSize, -) -> Option { +) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; let pattern4_0 = arg4; - // Rule at src/isa/x64/inst.isle line 1190. - let expr0_0: Type = I8X16; - let expr1_0 = C::temp_writable_reg(ctx, expr0_0); + // Rule at src/isa/x64/inst.isle line 1786. + let expr0_0 = C::temp_writable_xmm(ctx); + let expr1_0 = C::writable_xmm_to_reg(ctx, expr0_0); let expr2_0 = MInst::XmmRmRImm { op: pattern0_0.clone(), src1: pattern1_0, @@ -1967,118 +2395,218 @@ pub fn constructor_xmm_rm_r_imm( size: pattern4_0.clone(), }; let expr3_0 = C::emit(ctx, &expr2_0); - let expr4_0 = C::writable_reg_to_reg(ctx, expr1_0); + let expr4_0 = C::writable_xmm_to_xmm(ctx, expr0_0); return Some(expr4_0); } // Generated as internal constructor for term palignr. pub fn constructor_palignr( ctx: &mut C, - arg0: Reg, - arg1: &RegMem, + arg0: Xmm, + arg1: &XmmMem, arg2: u8, arg3: &OperandSize, -) -> Option { +) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1202. + // Rule at src/isa/x64/inst.isle line 1798. let expr0_0 = SseOpcode::Palignr; - let expr1_0 = constructor_xmm_rm_r_imm( - ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0, pattern3_0, - )?; - return Some(expr1_0); + let expr1_0 = C::xmm_to_reg(ctx, pattern0_0); + let expr2_0 = C::xmm_mem_to_reg_mem(ctx, pattern1_0); + let expr3_0 = + constructor_xmm_rm_r_imm(ctx, &expr0_0, expr1_0, &expr2_0, pattern2_0, pattern3_0)?; + return Some(expr3_0); +} + +// Generated as internal constructor for term cmpps. +pub fn constructor_cmpps( + ctx: &mut C, + arg0: Xmm, + arg1: &XmmMem, + arg2: &FcmpImm, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + // Rule at src/isa/x64/inst.isle line 1807. + let expr0_0 = SseOpcode::Cmpps; + let expr1_0 = C::xmm_to_reg(ctx, pattern0_0); + let expr2_0 = C::xmm_mem_to_reg_mem(ctx, pattern1_0); + let expr3_0 = C::encode_fcmp_imm(ctx, pattern2_0); + let expr4_0 = OperandSize::Size32; + let expr5_0 = constructor_xmm_rm_r_imm(ctx, &expr0_0, expr1_0, &expr2_0, expr3_0, &expr4_0)?; + return Some(expr5_0); +} + +// Generated as internal constructor for term pinsrb. +pub fn constructor_pinsrb( + ctx: &mut C, + arg0: Xmm, + arg1: &GprMem, + arg2: u8, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + // Rule at src/isa/x64/inst.isle line 1816. + let expr0_0 = SseOpcode::Pinsrb; + let expr1_0 = C::xmm_to_reg(ctx, pattern0_0); + let expr2_0 = C::gpr_mem_to_reg_mem(ctx, pattern1_0); + let expr3_0 = OperandSize::Size32; + let expr4_0 = constructor_xmm_rm_r_imm(ctx, &expr0_0, expr1_0, &expr2_0, pattern2_0, &expr3_0)?; + return Some(expr4_0); +} + +// Generated as internal constructor for term pinsrw. +pub fn constructor_pinsrw( + ctx: &mut C, + arg0: Xmm, + arg1: &GprMem, + arg2: u8, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + // Rule at src/isa/x64/inst.isle line 1825. + let expr0_0 = SseOpcode::Pinsrw; + let expr1_0 = C::xmm_to_reg(ctx, pattern0_0); + let expr2_0 = C::gpr_mem_to_reg_mem(ctx, pattern1_0); + let expr3_0 = OperandSize::Size32; + let expr4_0 = constructor_xmm_rm_r_imm(ctx, &expr0_0, expr1_0, &expr2_0, pattern2_0, &expr3_0)?; + return Some(expr4_0); +} + +// Generated as internal constructor for term pinsrd. +pub fn constructor_pinsrd( + ctx: &mut C, + arg0: Xmm, + arg1: &GprMem, + arg2: u8, + arg3: &OperandSize, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + let pattern3_0 = arg3; + // Rule at src/isa/x64/inst.isle line 1834. + let expr0_0 = SseOpcode::Pinsrd; + let expr1_0 = C::xmm_to_reg(ctx, pattern0_0); + let expr2_0 = C::gpr_mem_to_reg_mem(ctx, pattern1_0); + let expr3_0 = + constructor_xmm_rm_r_imm(ctx, &expr0_0, expr1_0, &expr2_0, pattern2_0, pattern3_0)?; + return Some(expr3_0); +} + +// Generated as internal constructor for term insertps. +pub fn constructor_insertps( + ctx: &mut C, + arg0: Xmm, + arg1: &XmmMem, + arg2: u8, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + // Rule at src/isa/x64/inst.isle line 1843. + let expr0_0 = SseOpcode::Insertps; + let expr1_0 = C::xmm_to_reg(ctx, pattern0_0); + let expr2_0 = C::xmm_mem_to_reg_mem(ctx, pattern1_0); + let expr3_0 = OperandSize::Size32; + let expr4_0 = constructor_xmm_rm_r_imm(ctx, &expr0_0, expr1_0, &expr2_0, pattern2_0, &expr3_0)?; + return Some(expr4_0); } // Generated as internal constructor for term pshufd. pub fn constructor_pshufd( ctx: &mut C, - arg0: &RegMem, + arg0: &XmmMem, arg1: u8, arg2: &OperandSize, -) -> Option { +) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1211. - let expr0_0: Type = I8X16; - let expr1_0 = C::temp_writable_reg(ctx, expr0_0); - let expr2_0 = C::writable_reg_to_reg(ctx, expr1_0); - let expr3_0 = SseOpcode::Pshufd; - let expr4_0 = MInst::XmmRmRImm { - op: expr3_0, - src1: expr2_0, - src2: pattern0_0.clone(), - dst: expr1_0, + // Rule at src/isa/x64/inst.isle line 1852. + let expr0_0 = C::temp_writable_xmm(ctx); + let expr1_0 = C::writable_xmm_to_xmm(ctx, expr0_0); + let expr2_0 = SseOpcode::Pshufd; + let expr3_0 = C::xmm_to_reg(ctx, expr1_0); + let expr4_0 = C::xmm_mem_to_reg_mem(ctx, pattern0_0); + let expr5_0 = C::writable_xmm_to_reg(ctx, expr0_0); + let expr6_0 = MInst::XmmRmRImm { + op: expr2_0, + src1: expr3_0, + src2: expr4_0, + dst: expr5_0, imm: pattern1_0, size: pattern2_0.clone(), }; - let expr5_0 = C::emit(ctx, &expr4_0); - return Some(expr2_0); + let expr7_0 = C::emit(ctx, &expr6_0); + return Some(expr1_0); } // Generated as internal constructor for term xmm_unary_rm_r. pub fn constructor_xmm_unary_rm_r( ctx: &mut C, arg0: &SseOpcode, - arg1: &RegMem, -) -> Option { + arg1: &XmmMem, +) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1224. - let expr0_0: Type = I8X16; - let expr1_0 = C::temp_writable_reg(ctx, expr0_0); - let expr2_0 = MInst::XmmUnaryRmR { + // Rule at src/isa/x64/inst.isle line 1865. + let expr0_0 = C::temp_writable_xmm(ctx); + let expr1_0 = MInst::XmmUnaryRmR { op: pattern0_0.clone(), src: pattern1_0.clone(), - dst: expr1_0, + dst: expr0_0, }; - let expr3_0 = C::emit(ctx, &expr2_0); - let expr4_0 = C::writable_reg_to_reg(ctx, expr1_0); - return Some(expr4_0); + let expr2_0 = C::emit(ctx, &expr1_0); + let expr3_0 = C::writable_xmm_to_xmm(ctx, expr0_0); + return Some(expr3_0); } // Generated as internal constructor for term pmovsxbw. -pub fn constructor_pmovsxbw(ctx: &mut C, arg0: &RegMem) -> Option { +pub fn constructor_pmovsxbw(ctx: &mut C, arg0: &XmmMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1231. + // Rule at src/isa/x64/inst.isle line 1872. let expr0_0 = SseOpcode::Pmovsxbw; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); } // Generated as internal constructor for term pmovzxbw. -pub fn constructor_pmovzxbw(ctx: &mut C, arg0: &RegMem) -> Option { +pub fn constructor_pmovzxbw(ctx: &mut C, arg0: &XmmMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1236. + // Rule at src/isa/x64/inst.isle line 1877. let expr0_0 = SseOpcode::Pmovzxbw; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); } // Generated as internal constructor for term pabsb. -pub fn constructor_pabsb(ctx: &mut C, arg0: &RegMem) -> Option { +pub fn constructor_pabsb(ctx: &mut C, arg0: &XmmMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1241. + // Rule at src/isa/x64/inst.isle line 1882. let expr0_0 = SseOpcode::Pabsb; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); } // Generated as internal constructor for term pabsw. -pub fn constructor_pabsw(ctx: &mut C, arg0: &RegMem) -> Option { +pub fn constructor_pabsw(ctx: &mut C, arg0: &XmmMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1246. + // Rule at src/isa/x64/inst.isle line 1887. let expr0_0 = SseOpcode::Pabsw; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); } // Generated as internal constructor for term pabsd. -pub fn constructor_pabsd(ctx: &mut C, arg0: &RegMem) -> Option { +pub fn constructor_pabsd(ctx: &mut C, arg0: &XmmMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1251. + // Rule at src/isa/x64/inst.isle line 1892. let expr0_0 = SseOpcode::Pabsd; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -2088,27 +2616,26 @@ pub fn constructor_pabsd(ctx: &mut C, arg0: &RegMem) -> Option pub fn constructor_xmm_unary_rm_r_evex( ctx: &mut C, arg0: &Avx512Opcode, - arg1: &RegMem, -) -> Option { + arg1: &XmmMem, +) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1256. - let expr0_0: Type = I8X16; - let expr1_0 = C::temp_writable_reg(ctx, expr0_0); - let expr2_0 = MInst::XmmUnaryRmREvex { + // Rule at src/isa/x64/inst.isle line 1897. + let expr0_0 = C::temp_writable_xmm(ctx); + let expr1_0 = MInst::XmmUnaryRmREvex { op: pattern0_0.clone(), src: pattern1_0.clone(), - dst: expr1_0, + dst: expr0_0, }; - let expr3_0 = C::emit(ctx, &expr2_0); - let expr4_0 = C::writable_reg_to_reg(ctx, expr1_0); - return Some(expr4_0); + let expr2_0 = C::emit(ctx, &expr1_0); + let expr3_0 = C::writable_xmm_to_xmm(ctx, expr0_0); + return Some(expr3_0); } // Generated as internal constructor for term vpabsq. -pub fn constructor_vpabsq(ctx: &mut C, arg0: &RegMem) -> Option { +pub fn constructor_vpabsq(ctx: &mut C, arg0: &XmmMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1263. + // Rule at src/isa/x64/inst.isle line 1904. let expr0_0 = Avx512Opcode::Vpabsq; let expr1_0 = constructor_xmm_unary_rm_r_evex(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -2118,140 +2645,35 @@ pub fn constructor_vpabsq(ctx: &mut C, arg0: &RegMem) -> Option pub fn constructor_xmm_rm_r_evex( ctx: &mut C, arg0: &Avx512Opcode, - arg1: &RegMem, - arg2: Reg, -) -> Option { + arg1: &XmmMem, + arg2: Xmm, +) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1268. - let expr0_0: Type = I8X16; - let expr1_0 = C::temp_writable_reg(ctx, expr0_0); - let expr2_0 = MInst::XmmRmREvex { + // Rule at src/isa/x64/inst.isle line 1909. + let expr0_0 = C::temp_writable_xmm(ctx); + let expr1_0 = MInst::XmmRmREvex { op: pattern0_0.clone(), src1: pattern1_0.clone(), src2: pattern2_0, - dst: expr1_0, + dst: expr0_0, }; - let expr3_0 = C::emit(ctx, &expr2_0); - let expr4_0 = C::writable_reg_to_reg(ctx, expr1_0); - return Some(expr4_0); + let expr2_0 = C::emit(ctx, &expr1_0); + let expr3_0 = C::writable_xmm_to_xmm(ctx, expr0_0); + return Some(expr3_0); } // Generated as internal constructor for term vpmullq. -pub fn constructor_vpmullq(ctx: &mut C, arg0: &RegMem, arg1: Reg) -> Option { +pub fn constructor_vpmullq(ctx: &mut C, arg0: &XmmMem, arg1: Xmm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1280. + // Rule at src/isa/x64/inst.isle line 1921. let expr0_0 = Avx512Opcode::Vpmullq; let expr1_0 = constructor_xmm_rm_r_evex(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); } -// Generated as internal constructor for term xmm_rmi_reg. -pub fn constructor_xmm_rmi_reg( - ctx: &mut C, - arg0: &SseOpcode, - arg1: Reg, - arg2: &RegMemImm, -) -> Option { - let pattern0_0 = arg0; - let pattern1_0 = arg1; - let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1287. - let expr0_0: Type = I8X16; - let expr1_0 = C::temp_writable_reg(ctx, expr0_0); - let expr2_0 = MInst::XmmRmiReg { - opcode: pattern0_0.clone(), - src1: pattern1_0, - src2: pattern2_0.clone(), - dst: expr1_0, - }; - let expr3_0 = C::emit(ctx, &expr2_0); - let expr4_0 = C::writable_reg_to_reg(ctx, expr1_0); - return Some(expr4_0); -} - -// Generated as internal constructor for term psllw. -pub fn constructor_psllw(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option { - let pattern0_0 = arg0; - let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1297. - let expr0_0 = SseOpcode::Psllw; - let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; - return Some(expr1_0); -} - -// Generated as internal constructor for term pslld. -pub fn constructor_pslld(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option { - let pattern0_0 = arg0; - let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1302. - let expr0_0 = SseOpcode::Pslld; - let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; - return Some(expr1_0); -} - -// Generated as internal constructor for term psllq. -pub fn constructor_psllq(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option { - let pattern0_0 = arg0; - let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1307. - let expr0_0 = SseOpcode::Psllq; - let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; - return Some(expr1_0); -} - -// Generated as internal constructor for term psrlw. -pub fn constructor_psrlw(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option { - let pattern0_0 = arg0; - let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1312. - let expr0_0 = SseOpcode::Psrlw; - let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; - return Some(expr1_0); -} - -// Generated as internal constructor for term psrld. -pub fn constructor_psrld(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option { - let pattern0_0 = arg0; - let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1317. - let expr0_0 = SseOpcode::Psrld; - let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; - return Some(expr1_0); -} - -// Generated as internal constructor for term psrlq. -pub fn constructor_psrlq(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option { - let pattern0_0 = arg0; - let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1322. - let expr0_0 = SseOpcode::Psrlq; - let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; - return Some(expr1_0); -} - -// Generated as internal constructor for term psraw. -pub fn constructor_psraw(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option { - let pattern0_0 = arg0; - let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1327. - let expr0_0 = SseOpcode::Psraw; - let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; - return Some(expr1_0); -} - -// Generated as internal constructor for term psrad. -pub fn constructor_psrad(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option { - let pattern0_0 = arg0; - let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1332. - let expr0_0 = SseOpcode::Psrad; - let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; - return Some(expr1_0); -} - // Generated as internal constructor for term mul_hi. pub fn constructor_mul_hi( ctx: &mut C, @@ -2264,7 +2686,7 @@ pub fn constructor_mul_hi( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1339. + // Rule at src/isa/x64/inst.isle line 1930. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::temp_writable_reg(ctx, pattern0_0); let expr2_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); @@ -2293,179 +2715,192 @@ pub fn constructor_mulhi_u( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1355. + // Rule at src/isa/x64/inst.isle line 1946. let expr0_0: bool = false; let expr1_0 = constructor_mul_hi(ctx, pattern0_0, expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); } -// Generated as internal constructor for term cmpps. -pub fn constructor_cmpps( +// Generated as internal constructor for term xmm_rmi_xmm. +pub fn constructor_xmm_rmi_xmm( ctx: &mut C, - arg0: Reg, - arg1: &RegMem, - arg2: &FcmpImm, -) -> Option { + arg0: &SseOpcode, + arg1: Xmm, + arg2: &XmmMemImm, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + // Rule at src/isa/x64/inst.isle line 1951. + let expr0_0 = C::temp_writable_xmm(ctx); + let expr1_0 = MInst::XmmRmiReg { + opcode: pattern0_0.clone(), + src1: pattern1_0, + src2: pattern2_0.clone(), + dst: expr0_0, + }; + let expr2_0 = C::emit(ctx, &expr1_0); + let expr3_0 = C::writable_xmm_to_xmm(ctx, expr0_0); + return Some(expr3_0); +} + +// Generated as internal constructor for term psllw. +pub fn constructor_psllw(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 1961. + let expr0_0 = SseOpcode::Psllw; + let expr1_0 = constructor_xmm_rmi_xmm(ctx, &expr0_0, pattern0_0, pattern1_0)?; + return Some(expr1_0); +} + +// Generated as internal constructor for term pslld. +pub fn constructor_pslld(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 1966. + let expr0_0 = SseOpcode::Pslld; + let expr1_0 = constructor_xmm_rmi_xmm(ctx, &expr0_0, pattern0_0, pattern1_0)?; + return Some(expr1_0); +} + +// Generated as internal constructor for term psllq. +pub fn constructor_psllq(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 1971. + let expr0_0 = SseOpcode::Psllq; + let expr1_0 = constructor_xmm_rmi_xmm(ctx, &expr0_0, pattern0_0, pattern1_0)?; + return Some(expr1_0); +} + +// Generated as internal constructor for term psrlw. +pub fn constructor_psrlw(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1360. - let expr0_0 = SseOpcode::Cmpps; - let expr1_0 = C::encode_fcmp_imm(ctx, pattern2_0); - let expr2_0 = OperandSize::Size32; - let expr3_0 = - constructor_xmm_rm_r_imm(ctx, &expr0_0, pattern0_0, pattern1_0, expr1_0, &expr2_0)?; - return Some(expr3_0); + // Rule at src/isa/x64/inst.isle line 1976. + let expr0_0 = SseOpcode::Psrlw; + let expr1_0 = constructor_xmm_rmi_xmm(ctx, &expr0_0, pattern0_0, pattern1_0)?; + return Some(expr1_0); } -// Generated as internal constructor for term cmppd. -pub fn constructor_cmppd( - ctx: &mut C, - arg0: Reg, - arg1: &RegMem, - arg2: &FcmpImm, -) -> Option { +// Generated as internal constructor for term psrld. +pub fn constructor_psrld(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1373. - let expr0_0 = SseOpcode::Cmppd; - let expr1_0 = C::encode_fcmp_imm(ctx, pattern2_0); - let expr2_0 = OperandSize::Size32; - let expr3_0 = - constructor_xmm_rm_r_imm(ctx, &expr0_0, pattern0_0, pattern1_0, expr1_0, &expr2_0)?; - return Some(expr3_0); + // Rule at src/isa/x64/inst.isle line 1981. + let expr0_0 = SseOpcode::Psrld; + let expr1_0 = constructor_xmm_rmi_xmm(ctx, &expr0_0, pattern0_0, pattern1_0)?; + return Some(expr1_0); } -// Generated as internal constructor for term gpr_to_xmm. -pub fn constructor_gpr_to_xmm( - ctx: &mut C, - arg0: Type, - arg1: &SseOpcode, - arg2: &RegMem, - arg3: &OperandSize, -) -> Option { +// Generated as internal constructor for term psrlq. +pub fn constructor_psrlq(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - let pattern2_0 = arg2; - let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1382. - let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); - let expr1_0 = MInst::GprToXmm { - op: pattern1_0.clone(), - src: pattern2_0.clone(), - dst: expr0_0, - src_size: pattern3_0.clone(), - }; - let expr2_0 = C::emit(ctx, &expr1_0); - let expr3_0 = C::writable_reg_to_reg(ctx, expr0_0); - return Some(expr3_0); + // Rule at src/isa/x64/inst.isle line 1986. + let expr0_0 = SseOpcode::Psrlq; + let expr1_0 = constructor_xmm_rmi_xmm(ctx, &expr0_0, pattern0_0, pattern1_0)?; + return Some(expr1_0); } -// Generated as internal constructor for term pinsrb. -pub fn constructor_pinsrb( - ctx: &mut C, - arg0: Reg, - arg1: &RegMem, - arg2: u8, -) -> Option { +// Generated as internal constructor for term psraw. +pub fn constructor_psraw(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1389. - let expr0_0 = SseOpcode::Pinsrb; - let expr1_0 = OperandSize::Size32; - let expr2_0 = - constructor_xmm_rm_r_imm(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0, &expr1_0)?; - return Some(expr2_0); + // Rule at src/isa/x64/inst.isle line 1991. + let expr0_0 = SseOpcode::Psraw; + let expr1_0 = constructor_xmm_rmi_xmm(ctx, &expr0_0, pattern0_0, pattern1_0)?; + return Some(expr1_0); } -// Generated as internal constructor for term pinsrw. -pub fn constructor_pinsrw( - ctx: &mut C, - arg0: Reg, - arg1: &RegMem, - arg2: u8, -) -> Option { +// Generated as internal constructor for term psrad. +pub fn constructor_psrad(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1394. - let expr0_0 = SseOpcode::Pinsrw; - let expr1_0 = OperandSize::Size32; - let expr2_0 = - constructor_xmm_rm_r_imm(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0, &expr1_0)?; - return Some(expr2_0); + // Rule at src/isa/x64/inst.isle line 1996. + let expr0_0 = SseOpcode::Psrad; + let expr1_0 = constructor_xmm_rmi_xmm(ctx, &expr0_0, pattern0_0, pattern1_0)?; + return Some(expr1_0); } -// Generated as internal constructor for term pinsrd. -pub fn constructor_pinsrd( - ctx: &mut C, - arg0: Reg, - arg1: &RegMem, - arg2: u8, - arg3: &OperandSize, -) -> Option { +// Generated as internal constructor for term pextrd. +pub fn constructor_pextrd(ctx: &mut C, arg0: Type, arg1: Xmm, arg2: u8) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1399. - let expr0_0 = SseOpcode::Pinsrd; - let expr1_0 = constructor_xmm_rm_r_imm( - ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0, pattern3_0, - )?; + // Rule at src/isa/x64/inst.isle line 2001. + let expr0_0 = C::temp_writable_gpr(ctx); + let expr1_0 = C::writable_gpr_to_gpr(ctx, expr0_0); + let expr2_0 = SseOpcode::Pextrd; + let expr3_0 = C::gpr_to_reg(ctx, expr1_0); + let expr4_0 = C::xmm_to_reg(ctx, pattern1_0); + let expr5_0 = RegMem::Reg { reg: expr4_0 }; + let expr6_0 = C::writable_gpr_to_reg(ctx, expr0_0); + let expr7_0 = C::lane_type(ctx, pattern0_0); + let expr8_0 = C::operand_size_of_type_32_64(ctx, expr7_0); + let expr9_0 = MInst::XmmRmRImm { + op: expr2_0, + src1: expr3_0, + src2: expr5_0, + dst: expr6_0, + imm: pattern2_0, + size: expr8_0, + }; + let expr10_0 = C::emit(ctx, &expr9_0); return Some(expr1_0); } -// Generated as internal constructor for term insertps. -pub fn constructor_insertps( +// Generated as internal constructor for term cmppd. +pub fn constructor_cmppd( ctx: &mut C, - arg0: Reg, - arg1: &RegMem, - arg2: u8, -) -> Option { + arg0: Xmm, + arg1: &XmmMem, + arg2: &FcmpImm, +) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1404. - let expr0_0 = SseOpcode::Insertps; - let expr1_0 = OperandSize::Size32; - let expr2_0 = - constructor_xmm_rm_r_imm(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0, &expr1_0)?; - return Some(expr2_0); + // Rule at src/isa/x64/inst.isle line 2018. + let expr0_0 = SseOpcode::Cmppd; + let expr1_0 = C::xmm_to_reg(ctx, pattern0_0); + let expr2_0 = C::xmm_mem_to_reg_mem(ctx, pattern1_0); + let expr3_0 = C::encode_fcmp_imm(ctx, pattern2_0); + let expr4_0 = OperandSize::Size32; + let expr5_0 = constructor_xmm_rm_r_imm(ctx, &expr0_0, expr1_0, &expr2_0, expr3_0, &expr4_0)?; + return Some(expr5_0); } -// Generated as internal constructor for term pextrd. -pub fn constructor_pextrd(ctx: &mut C, arg0: Type, arg1: Reg, arg2: u8) -> Option { +// Generated as internal constructor for term gpr_to_xmm. +pub fn constructor_gpr_to_xmm( + ctx: &mut C, + arg0: &SseOpcode, + arg1: &GprMem, + arg2: &OperandSize, +) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1409. - let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); - let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); - let expr2_0 = SseOpcode::Pextrd; - let expr3_0 = RegMem::Reg { reg: pattern1_0 }; - let expr4_0 = C::lane_type(ctx, pattern0_0); - let expr5_0 = C::operand_size_of_type_32_64(ctx, expr4_0); - let expr6_0 = MInst::XmmRmRImm { - op: expr2_0, - src1: expr1_0, - src2: expr3_0, + // Rule at src/isa/x64/inst.isle line 2027. + let expr0_0 = C::temp_writable_xmm(ctx); + let expr1_0 = C::gpr_mem_to_reg_mem(ctx, pattern1_0); + let expr2_0 = MInst::GprToXmm { + op: pattern0_0.clone(), + src: expr1_0, dst: expr0_0, - imm: pattern2_0, - size: expr5_0, + src_size: pattern2_0.clone(), }; - let expr7_0 = C::emit(ctx, &expr6_0); - return Some(expr1_0); + let expr3_0 = C::emit(ctx, &expr2_0); + let expr4_0 = C::writable_xmm_to_xmm(ctx, expr0_0); + return Some(expr4_0); } // Generated as internal constructor for term not. -pub fn constructor_not(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { +pub fn constructor_not(ctx: &mut C, arg0: Type, arg1: Gpr) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1422. - let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); + // Rule at src/isa/x64/inst.isle line 2034. + let expr0_0 = C::temp_writable_gpr(ctx); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = MInst::Not { size: expr1_0, @@ -2473,16 +2908,16 @@ pub fn constructor_not(ctx: &mut C, arg0: Type, arg1: Reg) -> Option dst: expr0_0, }; let expr3_0 = C::emit(ctx, &expr2_0); - let expr4_0 = C::writable_reg_to_reg(ctx, expr0_0); + let expr4_0 = C::writable_gpr_to_gpr(ctx, expr0_0); return Some(expr4_0); } // Generated as internal constructor for term neg. -pub fn constructor_neg(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { +pub fn constructor_neg(ctx: &mut C, arg0: Type, arg1: Gpr) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1430. - let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); + // Rule at src/isa/x64/inst.isle line 2042. + let expr0_0 = C::temp_writable_gpr(ctx); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = MInst::Neg { size: expr1_0, @@ -2490,29 +2925,28 @@ pub fn constructor_neg(ctx: &mut C, arg0: Type, arg1: Reg) -> Option dst: expr0_0, }; let expr3_0 = C::emit(ctx, &expr2_0); - let expr4_0 = C::writable_reg_to_reg(ctx, expr0_0); + let expr4_0 = C::writable_gpr_to_gpr(ctx, expr0_0); return Some(expr4_0); } // Generated as internal constructor for term lea. -pub fn constructor_lea(ctx: &mut C, arg0: &SyntheticAmode) -> Option { +pub fn constructor_lea(ctx: &mut C, arg0: &SyntheticAmode) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1437. - let expr0_0: Type = I64; - let expr1_0 = C::temp_writable_reg(ctx, expr0_0); - let expr2_0 = MInst::LoadEffectiveAddress { + // Rule at src/isa/x64/inst.isle line 2049. + let expr0_0 = C::temp_writable_gpr(ctx); + let expr1_0 = MInst::LoadEffectiveAddress { addr: pattern0_0.clone(), - dst: expr1_0, + dst: expr0_0, }; - let expr3_0 = C::emit(ctx, &expr2_0); - let expr4_0 = C::writable_reg_to_reg(ctx, expr1_0); - return Some(expr4_0); + let expr2_0 = C::emit(ctx, &expr1_0); + let expr3_0 = C::writable_gpr_to_gpr(ctx, expr0_0); + return Some(expr3_0); } // Generated as internal constructor for term ud2. pub fn constructor_ud2(ctx: &mut C, arg0: &TrapCode) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1444. + // Rule at src/isa/x64/inst.isle line 2056. let expr0_0 = MInst::Ud2 { trap_code: pattern0_0.clone(), }; @@ -2557,13 +2991,13 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { match &pattern2_0 { &Opcode::Trap => { - // Rule at src/isa/x64/lower.isle line 1438. + // Rule at src/isa/x64/lower.isle line 1463. let expr0_0 = constructor_ud2(ctx, &pattern2_1)?; let expr1_0 = constructor_value_regs_none(ctx, &expr0_0)?; return Some(expr1_0); } &Opcode::ResumableTrap => { - // Rule at src/isa/x64/lower.isle line 1443. + // Rule at src/isa/x64/lower.isle line 1468. let expr0_0 = constructor_ud2(ctx, &pattern2_1)?; let expr1_0 = constructor_value_regs_none(ctx, &expr0_0)?; return Some(expr1_0); @@ -2580,12 +3014,12 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 384. + // Rule at src/isa/x64/lower.isle line 386. let expr0_0 = C::put_in_regs(ctx, pattern7_0); let expr1_0: usize = 0; let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); @@ -2649,7 +3083,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 458. + // Rule at src/isa/x64/lower.isle line 462. let expr0_0 = C::put_in_regs(ctx, pattern7_0); let expr1_0: usize = 0; let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); @@ -2665,7 +3099,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 524. + // Rule at src/isa/x64/lower.isle line 528. let expr0_0 = C::put_in_regs(ctx, pattern7_0); let expr1_0: usize = 0; let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); @@ -2686,7 +3120,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::Bnot = &pattern5_0 { - // Rule at src/isa/x64/lower.isle line 1304. + // Rule at src/isa/x64/lower.isle line 1321. let expr0_0 = constructor_i128_not(ctx, pattern5_1)?; return Some(expr0_0); } @@ -2769,7 +3203,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 992. + // Rule at src/isa/x64/lower.isle line 1007. let expr0_0 = C::put_in_regs(ctx, pattern7_0); let expr1_0: usize = 0; let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); @@ -2805,7 +3239,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 374. + // Rule at src/isa/x64/lower.isle line 376. let expr0_0 = C::put_in_regs(ctx, pattern7_0); let expr1_0: usize = 0; let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); @@ -2828,7 +3262,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 455. + // Rule at src/isa/x64/lower.isle line 459. let expr0_0 = C::put_in_regs(ctx, pattern7_0); let expr1_0 = C::put_in_regs(ctx, pattern7_1); let expr2_0 = constructor_or_i128(ctx, expr0_0, expr1_0)?; @@ -2837,7 +3271,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 514. + // Rule at src/isa/x64/lower.isle line 518. let expr0_0 = C::put_in_regs(ctx, pattern7_0); let expr1_0: usize = 0; let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); @@ -2860,7 +3294,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 860. + // Rule at src/isa/x64/lower.isle line 875. let expr0_0 = C::put_in_regs(ctx, pattern7_0); let expr1_0 = constructor_lo_reg(ctx, pattern7_1)?; let expr2_0 = constructor_shl_i128(ctx, expr0_0, expr1_0)?; @@ -2877,7 +3311,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 900. + // Rule at src/isa/x64/lower.isle line 915. let expr0_0 = C::put_in_regs(ctx, pattern7_0); let expr1_0 = constructor_lo_reg(ctx, pattern7_1)?; let expr2_0 = constructor_shr_i128(ctx, expr0_0, expr1_0)?; @@ -2894,7 +3328,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 568. + // Rule at src/isa/x64/lower.isle line 572. let expr0_0 = constructor_lo_reg(ctx, pattern7_1)?; let expr1_0 = C::put_in_regs(ctx, pattern7_0); let expr2_0 = constructor_shl_i128(ctx, expr1_0, expr0_0)?; @@ -2903,7 +3337,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 664. + // Rule at src/isa/x64/lower.isle line 671. let expr0_0 = constructor_lo_reg(ctx, pattern7_1)?; let expr1_0 = C::put_in_regs(ctx, pattern7_0); let expr2_0 = constructor_shr_i128(ctx, expr1_0, expr0_0)?; @@ -2912,7 +3346,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 761. + // Rule at src/isa/x64/lower.isle line 773. let expr0_0 = constructor_lo_reg(ctx, pattern7_1)?; let expr1_0 = C::put_in_regs(ctx, pattern7_0); let expr2_0 = constructor_sar_i128(ctx, expr1_0, expr0_0)?; @@ -2926,7 +3360,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::Bnot = &pattern5_0 { - // Rule at src/isa/x64/lower.isle line 1301. + // Rule at src/isa/x64/lower.isle line 1318. let expr0_0 = constructor_i128_not(ctx, pattern5_1)?; return Some(expr0_0); } @@ -2971,50 +3405,50 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1405. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); + // Rule at src/isa/x64/lower.isle line 1430. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_pminsb(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::Umin => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1427. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); + // Rule at src/isa/x64/lower.isle line 1452. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_pminub(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::Imax => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1394. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); + // Rule at src/isa/x64/lower.isle line 1419. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_pmaxsb(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::Umax => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1416. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); + // Rule at src/isa/x64/lower.isle line 1441. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_pmaxub(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::Ishl => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 580. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); + // Rule at src/isa/x64/lower.isle line 584. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = C::put_in_reg_mem_imm(ctx, pattern7_1); - let expr2_0 = constructor_reg_mem_imm_to_xmm(ctx, &expr1_0)?; + let expr2_0 = constructor_mov_rmi_to_xmm(ctx, &expr1_0)?; let expr3_0 = constructor_psllw(ctx, expr0_0, &expr2_0)?; let expr4_0 = constructor_ishl_i8x16_mask(ctx, &expr1_0)?; let expr5_0: Type = I8X16; @@ -3022,17 +3456,18 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 674. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); + // Rule at src/isa/x64/lower.isle line 681. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = C::put_in_reg_mem_imm(ctx, pattern7_1); - let expr2_0 = constructor_reg_mem_imm_to_xmm(ctx, &expr1_0)?; + let expr2_0 = constructor_mov_rmi_to_xmm(ctx, &expr1_0)?; let expr3_0 = constructor_psrlw(ctx, expr0_0, &expr2_0)?; let expr4_0 = constructor_ushr_i8x16_mask(ctx, &expr1_0)?; let expr5_0: Type = I8X16; @@ -3040,28 +3475,29 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); let pattern8_0 = C::value_type(ctx, pattern7_1); - // Rule at src/isa/x64/lower.isle line 782. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); - let expr1_0 = RegMem::Reg { reg: expr0_0 }; + // Rule at src/isa/x64/lower.isle line 794. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; + let expr1_0 = C::xmm_to_xmm_mem(ctx, expr0_0); let expr2_0 = constructor_punpcklbw(ctx, expr0_0, &expr1_0)?; - let expr3_0 = RegMem::Reg { reg: expr0_0 }; + let expr3_0 = C::xmm_to_xmm_mem(ctx, expr0_0); let expr4_0 = constructor_punpckhbw(ctx, expr0_0, &expr3_0)?; let expr5_0 = C::put_in_reg_mem_imm(ctx, pattern7_1); let expr6_0 = constructor_sshr_i8x16_bigger_shift(ctx, pattern8_0, &expr5_0)?; let expr7_0 = constructor_psraw(ctx, expr2_0, &expr6_0)?; let expr8_0 = constructor_psraw(ctx, expr4_0, &expr6_0)?; - let expr9_0 = RegMem::Reg { reg: expr8_0 }; + let expr9_0 = C::xmm_to_xmm_mem(ctx, expr8_0); let expr10_0 = constructor_packsswb(ctx, expr7_0, &expr9_0)?; - let expr11_0 = C::value_reg(ctx, expr10_0); + let expr11_0 = constructor_value_xmm(ctx, expr10_0)?; return Some(expr11_0); } _ => {} @@ -3073,20 +3509,21 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { match &pattern5_0 { &Opcode::Ineg => { - // Rule at src/isa/x64/lower.isle line 917. + // Rule at src/isa/x64/lower.isle line 932. let expr0_0: Type = I8X16; let expr1_0: u64 = 0; let expr2_0 = constructor_imm(ctx, expr0_0, expr1_0)?; - let expr3_0 = C::put_in_reg_mem(ctx, pattern5_1); - let expr4_0 = constructor_psubb(ctx, expr2_0, &expr3_0)?; - let expr5_0 = C::value_reg(ctx, expr4_0); - return Some(expr5_0); + let expr3_0 = C::xmm_new(ctx, expr2_0); + let expr4_0 = constructor_put_in_xmm_mem(ctx, pattern5_1)?; + let expr5_0 = constructor_psubb(ctx, expr3_0, &expr4_0)?; + let expr6_0 = constructor_value_xmm(ctx, expr5_0)?; + return Some(expr6_0); } &Opcode::Iabs => { - // Rule at src/isa/x64/lower.isle line 1247. - let expr0_0 = C::put_in_reg_mem(ctx, pattern5_1); + // Rule at src/isa/x64/lower.isle line 1262. + let expr0_0 = constructor_put_in_xmm_mem(ctx, pattern5_1)?; let expr1_0 = constructor_pabsb(ctx, &expr0_0)?; - let expr2_0 = C::value_reg(ctx, expr1_0); + let expr2_0 = constructor_value_xmm(ctx, expr1_0)?; return Some(expr2_0); } _ => {} @@ -3106,74 +3543,74 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1408. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); + // Rule at src/isa/x64/lower.isle line 1433. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_pminsw(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::Umin => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1430. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); + // Rule at src/isa/x64/lower.isle line 1455. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_pminuw(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::Imax => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1397. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); + // Rule at src/isa/x64/lower.isle line 1422. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_pmaxsw(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::Umax => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1419. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); + // Rule at src/isa/x64/lower.isle line 1444. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_pmaxuw(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::Ishl => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 621. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); + // Rule at src/isa/x64/lower.isle line 626. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = C::put_in_reg_mem_imm(ctx, pattern7_1); - let expr2_0 = constructor_reg_mem_imm_to_xmm(ctx, &expr1_0)?; + let expr2_0 = constructor_mov_rmi_to_xmm(ctx, &expr1_0)?; let expr3_0 = constructor_psllw(ctx, expr0_0, &expr2_0)?; - let expr4_0 = C::value_reg(ctx, expr3_0); + let expr4_0 = constructor_value_xmm(ctx, expr3_0)?; return Some(expr4_0); } &Opcode::Ushr => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 715. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); + // Rule at src/isa/x64/lower.isle line 725. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = C::put_in_reg_mem_imm(ctx, pattern7_1); - let expr2_0 = constructor_reg_mem_imm_to_xmm(ctx, &expr1_0)?; + let expr2_0 = constructor_mov_rmi_to_xmm(ctx, &expr1_0)?; let expr3_0 = constructor_psrlw(ctx, expr0_0, &expr2_0)?; - let expr4_0 = C::value_reg(ctx, expr3_0); + let expr4_0 = constructor_value_xmm(ctx, expr3_0)?; return Some(expr4_0); } &Opcode::Sshr => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 804. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); + // Rule at src/isa/x64/lower.isle line 817. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = C::put_in_reg_mem_imm(ctx, pattern7_1); - let expr2_0 = constructor_reg_mem_imm_to_xmm(ctx, &expr1_0)?; + let expr2_0 = constructor_mov_rmi_to_xmm(ctx, &expr1_0)?; let expr3_0 = constructor_psraw(ctx, expr0_0, &expr2_0)?; - let expr4_0 = C::value_reg(ctx, expr3_0); + let expr4_0 = constructor_value_xmm(ctx, expr3_0)?; return Some(expr4_0); } _ => {} @@ -3185,20 +3622,21 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { match &pattern5_0 { &Opcode::Ineg => { - // Rule at src/isa/x64/lower.isle line 921. + // Rule at src/isa/x64/lower.isle line 936. let expr0_0: Type = I16X8; let expr1_0: u64 = 0; let expr2_0 = constructor_imm(ctx, expr0_0, expr1_0)?; - let expr3_0 = C::put_in_reg_mem(ctx, pattern5_1); - let expr4_0 = constructor_psubw(ctx, expr2_0, &expr3_0)?; - let expr5_0 = C::value_reg(ctx, expr4_0); - return Some(expr5_0); + let expr3_0 = C::xmm_new(ctx, expr2_0); + let expr4_0 = constructor_put_in_xmm_mem(ctx, pattern5_1)?; + let expr5_0 = constructor_psubw(ctx, expr3_0, &expr4_0)?; + let expr6_0 = constructor_value_xmm(ctx, expr5_0)?; + return Some(expr6_0); } &Opcode::Iabs => { - // Rule at src/isa/x64/lower.isle line 1250. - let expr0_0 = C::put_in_reg_mem(ctx, pattern5_1); + // Rule at src/isa/x64/lower.isle line 1265. + let expr0_0 = constructor_put_in_xmm_mem(ctx, pattern5_1)?; let expr1_0 = constructor_pabsw(ctx, &expr0_0)?; - let expr2_0 = C::value_reg(ctx, expr1_0); + let expr2_0 = constructor_value_xmm(ctx, expr1_0)?; return Some(expr2_0); } _ => {} @@ -3218,74 +3656,74 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1411. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); + // Rule at src/isa/x64/lower.isle line 1436. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_pminsd(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::Umin => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1433. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); + // Rule at src/isa/x64/lower.isle line 1458. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_pminud(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::Imax => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1400. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); + // Rule at src/isa/x64/lower.isle line 1425. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_pmaxsd(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::Umax => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1422. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); + // Rule at src/isa/x64/lower.isle line 1447. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_pmaxud(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::Ishl => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 624. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); + // Rule at src/isa/x64/lower.isle line 630. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = C::put_in_reg_mem_imm(ctx, pattern7_1); - let expr2_0 = constructor_reg_mem_imm_to_xmm(ctx, &expr1_0)?; + let expr2_0 = constructor_mov_rmi_to_xmm(ctx, &expr1_0)?; let expr3_0 = constructor_pslld(ctx, expr0_0, &expr2_0)?; - let expr4_0 = C::value_reg(ctx, expr3_0); + let expr4_0 = constructor_value_xmm(ctx, expr3_0)?; return Some(expr4_0); } &Opcode::Ushr => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 718. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); + // Rule at src/isa/x64/lower.isle line 729. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = C::put_in_reg_mem_imm(ctx, pattern7_1); - let expr2_0 = constructor_reg_mem_imm_to_xmm(ctx, &expr1_0)?; + let expr2_0 = constructor_mov_rmi_to_xmm(ctx, &expr1_0)?; let expr3_0 = constructor_psrld(ctx, expr0_0, &expr2_0)?; - let expr4_0 = C::value_reg(ctx, expr3_0); + let expr4_0 = constructor_value_xmm(ctx, expr3_0)?; return Some(expr4_0); } &Opcode::Sshr => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 807. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); + // Rule at src/isa/x64/lower.isle line 821. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = C::put_in_reg_mem_imm(ctx, pattern7_1); - let expr2_0 = constructor_reg_mem_imm_to_xmm(ctx, &expr1_0)?; + let expr2_0 = constructor_mov_rmi_to_xmm(ctx, &expr1_0)?; let expr3_0 = constructor_psrad(ctx, expr0_0, &expr2_0)?; - let expr4_0 = C::value_reg(ctx, expr3_0); + let expr4_0 = constructor_value_xmm(ctx, expr3_0)?; return Some(expr4_0); } _ => {} @@ -3297,20 +3735,21 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { match &pattern5_0 { &Opcode::Ineg => { - // Rule at src/isa/x64/lower.isle line 925. + // Rule at src/isa/x64/lower.isle line 940. let expr0_0: Type = I32X4; let expr1_0: u64 = 0; let expr2_0 = constructor_imm(ctx, expr0_0, expr1_0)?; - let expr3_0 = C::put_in_reg_mem(ctx, pattern5_1); - let expr4_0 = constructor_psubd(ctx, expr2_0, &expr3_0)?; - let expr5_0 = C::value_reg(ctx, expr4_0); - return Some(expr5_0); + let expr3_0 = C::xmm_new(ctx, expr2_0); + let expr4_0 = constructor_put_in_xmm_mem(ctx, pattern5_1)?; + let expr5_0 = constructor_psubd(ctx, expr3_0, &expr4_0)?; + let expr6_0 = constructor_value_xmm(ctx, expr5_0)?; + return Some(expr6_0); } &Opcode::Iabs => { - // Rule at src/isa/x64/lower.isle line 1253. - let expr0_0 = C::put_in_reg_mem(ctx, pattern5_1); + // Rule at src/isa/x64/lower.isle line 1268. + let expr0_0 = constructor_put_in_xmm_mem(ctx, pattern5_1)?; let expr1_0 = constructor_pabsd(ctx, &expr0_0)?; - let expr2_0 = C::value_reg(ctx, expr1_0); + let expr2_0 = constructor_value_xmm(ctx, expr1_0)?; return Some(expr2_0); } _ => {} @@ -3330,30 +3769,30 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 627. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); + // Rule at src/isa/x64/lower.isle line 634. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = C::put_in_reg_mem_imm(ctx, pattern7_1); - let expr2_0 = constructor_reg_mem_imm_to_xmm(ctx, &expr1_0)?; + let expr2_0 = constructor_mov_rmi_to_xmm(ctx, &expr1_0)?; let expr3_0 = constructor_psllq(ctx, expr0_0, &expr2_0)?; - let expr4_0 = C::value_reg(ctx, expr3_0); + let expr4_0 = constructor_value_xmm(ctx, expr3_0)?; return Some(expr4_0); } &Opcode::Ushr => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 721. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); + // Rule at src/isa/x64/lower.isle line 733. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = C::put_in_reg_mem_imm(ctx, pattern7_1); - let expr2_0 = constructor_reg_mem_imm_to_xmm(ctx, &expr1_0)?; + let expr2_0 = constructor_mov_rmi_to_xmm(ctx, &expr1_0)?; let expr3_0 = constructor_psrlq(ctx, expr0_0, &expr2_0)?; - let expr4_0 = C::value_reg(ctx, expr3_0); + let expr4_0 = constructor_value_xmm(ctx, expr3_0)?; return Some(expr4_0); } &Opcode::Sshr => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 819. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); + // Rule at src/isa/x64/lower.isle line 833. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0: Type = I64; let expr2_0: u8 = 0; let expr3_0 = constructor_pextrd(ctx, expr1_0, expr0_0, expr2_0)?; @@ -3363,15 +3802,19 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option {} } @@ -3382,27 +3825,29 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { match &pattern5_0 { &Opcode::Ineg => { - // Rule at src/isa/x64/lower.isle line 929. + // Rule at src/isa/x64/lower.isle line 944. let expr0_0: Type = I64X2; let expr1_0: u64 = 0; let expr2_0 = constructor_imm(ctx, expr0_0, expr1_0)?; - let expr3_0 = C::put_in_reg_mem(ctx, pattern5_1); - let expr4_0 = constructor_psubq(ctx, expr2_0, &expr3_0)?; - let expr5_0 = C::value_reg(ctx, expr4_0); - return Some(expr5_0); + let expr3_0 = C::xmm_new(ctx, expr2_0); + let expr4_0 = constructor_put_in_xmm_mem(ctx, pattern5_1)?; + let expr5_0 = constructor_psubq(ctx, expr3_0, &expr4_0)?; + let expr6_0 = constructor_value_xmm(ctx, expr5_0)?; + return Some(expr6_0); } &Opcode::Iabs => { - // Rule at src/isa/x64/lower.isle line 1267. - let expr0_0 = C::put_in_reg(ctx, pattern5_1); + // Rule at src/isa/x64/lower.isle line 1282. + let expr0_0 = constructor_put_in_xmm(ctx, pattern5_1)?; let expr1_0: Type = I64X2; let expr2_0: u64 = 0; let expr3_0 = constructor_imm(ctx, expr1_0, expr2_0)?; - let expr4_0 = RegMem::Reg { reg: expr0_0 }; - let expr5_0 = constructor_psubq(ctx, expr3_0, &expr4_0)?; - let expr6_0 = RegMem::Reg { reg: expr0_0 }; - let expr7_0 = constructor_blendvpd(ctx, expr5_0, &expr6_0, expr5_0)?; - let expr8_0 = C::value_reg(ctx, expr7_0); - return Some(expr8_0); + let expr4_0 = C::xmm_new(ctx, expr3_0); + let expr5_0 = C::xmm_to_xmm_mem(ctx, expr0_0); + let expr6_0 = constructor_psubq(ctx, expr4_0, &expr5_0)?; + let expr7_0 = C::xmm_to_xmm_mem(ctx, expr0_0); + let expr8_0 = constructor_blendvpd(ctx, expr6_0, &expr7_0, expr6_0)?; + let expr9_0 = constructor_value_xmm(ctx, expr8_0)?; + return Some(expr9_0); } _ => {} } @@ -3418,17 +3863,18 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::BandNot = &pattern4_0 { let (pattern6_0, pattern6_1) = C::unpack_value_array_2(ctx, &pattern4_1); - // Rule at src/isa/x64/lower.isle line 1240. - let expr0_0 = C::put_in_reg(ctx, pattern6_1); - let expr1_0 = C::put_in_reg_mem(ctx, pattern6_0); + // Rule at src/isa/x64/lower.isle line 1255. + let expr0_0 = constructor_put_in_xmm(ctx, pattern6_1)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern6_0)?; let expr2_0 = constructor_sse_and_not(ctx, pattern2_0, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } } @@ -3497,11 +3944,11 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { let (pattern9_0, pattern9_1) = C::unpack_value_array_2(ctx, &pattern7_1); - // Rule at src/isa/x64/lower.isle line 935. - let expr0_0 = C::put_in_reg(ctx, pattern9_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); + // Rule at src/isa/x64/lower.isle line 950. + let expr0_0 = constructor_put_in_xmm(ctx, pattern9_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern9_1)?; let expr2_0 = constructor_pavgb(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::UaddSat => { let (pattern9_0, pattern9_1) = C::unpack_value_array_2(ctx, &pattern7_1); // Rule at src/isa/x64/lower.isle line 144. - let expr0_0 = C::put_in_reg(ctx, pattern9_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); + let expr0_0 = constructor_put_in_xmm(ctx, pattern9_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern9_1)?; let expr2_0 = constructor_paddusb(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::SaddSat => { let (pattern9_0, pattern9_1) = C::unpack_value_array_2(ctx, &pattern7_1); // Rule at src/isa/x64/lower.isle line 132. - let expr0_0 = C::put_in_reg(ctx, pattern9_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); + let expr0_0 = constructor_put_in_xmm(ctx, pattern9_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern9_1)?; let expr2_0 = constructor_paddsb(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::UsubSat => { let (pattern9_0, pattern9_1) = C::unpack_value_array_2(ctx, &pattern7_1); // Rule at src/isa/x64/lower.isle line 313. - let expr0_0 = C::put_in_reg(ctx, pattern9_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); + let expr0_0 = constructor_put_in_xmm(ctx, pattern9_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern9_1)?; let expr2_0 = constructor_psubusb(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::SsubSat => { let (pattern9_0, pattern9_1) = C::unpack_value_array_2(ctx, &pattern7_1); // Rule at src/isa/x64/lower.isle line 301. - let expr0_0 = C::put_in_reg(ctx, pattern9_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); + let expr0_0 = constructor_put_in_xmm(ctx, pattern9_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern9_1)?; let expr2_0 = constructor_psubsb(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::Iadd => { let (pattern9_0, pattern9_1) = C::unpack_value_array_2(ctx, &pattern7_1); // Rule at src/isa/x64/lower.isle line 96. - let expr0_0 = C::put_in_reg(ctx, pattern9_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); + let expr0_0 = constructor_put_in_xmm(ctx, pattern9_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern9_1)?; let expr2_0 = constructor_paddb(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::Isub => { let (pattern9_0, pattern9_1) = C::unpack_value_array_2(ctx, &pattern7_1); // Rule at src/isa/x64/lower.isle line 265. - let expr0_0 = C::put_in_reg(ctx, pattern9_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); + let expr0_0 = constructor_put_in_xmm(ctx, pattern9_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern9_1)?; let expr2_0 = constructor_psubb(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } _ => {} @@ -3625,71 +4072,71 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern9_0, pattern9_1) = C::unpack_value_array_2(ctx, &pattern7_1); - // Rule at src/isa/x64/lower.isle line 939. - let expr0_0 = C::put_in_reg(ctx, pattern9_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); + // Rule at src/isa/x64/lower.isle line 954. + let expr0_0 = constructor_put_in_xmm(ctx, pattern9_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern9_1)?; let expr2_0 = constructor_pavgw(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::UaddSat => { let (pattern9_0, pattern9_1) = C::unpack_value_array_2(ctx, &pattern7_1); // Rule at src/isa/x64/lower.isle line 149. - let expr0_0 = C::put_in_reg(ctx, pattern9_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); + let expr0_0 = constructor_put_in_xmm(ctx, pattern9_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern9_1)?; let expr2_0 = constructor_paddusw(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::SaddSat => { let (pattern9_0, pattern9_1) = C::unpack_value_array_2(ctx, &pattern7_1); // Rule at src/isa/x64/lower.isle line 137. - let expr0_0 = C::put_in_reg(ctx, pattern9_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); + let expr0_0 = constructor_put_in_xmm(ctx, pattern9_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern9_1)?; let expr2_0 = constructor_paddsw(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::UsubSat => { let (pattern9_0, pattern9_1) = C::unpack_value_array_2(ctx, &pattern7_1); // Rule at src/isa/x64/lower.isle line 318. - let expr0_0 = C::put_in_reg(ctx, pattern9_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); + let expr0_0 = constructor_put_in_xmm(ctx, pattern9_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern9_1)?; let expr2_0 = constructor_psubusw(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::SsubSat => { let (pattern9_0, pattern9_1) = C::unpack_value_array_2(ctx, &pattern7_1); // Rule at src/isa/x64/lower.isle line 306. - let expr0_0 = C::put_in_reg(ctx, pattern9_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); + let expr0_0 = constructor_put_in_xmm(ctx, pattern9_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern9_1)?; let expr2_0 = constructor_psubsw(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::Iadd => { let (pattern9_0, pattern9_1) = C::unpack_value_array_2(ctx, &pattern7_1); // Rule at src/isa/x64/lower.isle line 101. - let expr0_0 = C::put_in_reg(ctx, pattern9_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); + let expr0_0 = constructor_put_in_xmm(ctx, pattern9_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern9_1)?; let expr2_0 = constructor_paddw(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::Isub => { let (pattern9_0, pattern9_1) = C::unpack_value_array_2(ctx, &pattern7_1); // Rule at src/isa/x64/lower.isle line 270. - let expr0_0 = C::put_in_reg(ctx, pattern9_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); + let expr0_0 = constructor_put_in_xmm(ctx, pattern9_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern9_1)?; let expr2_0 = constructor_psubw(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::Imul => { @@ -3738,16 +4185,14 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option {} @@ -4000,20 +4423,20 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern9_0, pattern9_1) = C::unpack_value_array_2(ctx, &pattern7_1); // Rule at src/isa/x64/lower.isle line 275. - let expr0_0 = C::put_in_reg(ctx, pattern9_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); + let expr0_0 = constructor_put_in_xmm(ctx, pattern9_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern9_1)?; let expr2_0 = constructor_psubd(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::Imul => { @@ -4062,22 +4485,16 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option {} @@ -4312,20 +4711,20 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern9_0, pattern9_1) = C::unpack_value_array_2(ctx, &pattern7_1); // Rule at src/isa/x64/lower.isle line 280. - let expr0_0 = C::put_in_reg(ctx, pattern9_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); + let expr0_0 = constructor_put_in_xmm(ctx, pattern9_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern9_1)?; let expr2_0 = constructor_psubq(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::Imul => { @@ -4374,20 +4773,18 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option {} } @@ -4632,30 +5026,30 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 440. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); + // Rule at src/isa/x64/lower.isle line 442. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_sse_or(ctx, pattern2_0, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::Bxor => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 509. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); + // Rule at src/isa/x64/lower.isle line 513. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_sse_xor(ctx, pattern2_0, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_xmm(ctx, expr2_0)?; return Some(expr3_0); } _ => {} @@ -4669,30 +5063,30 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1, pattern7_2) = C::unpack_value_array_3(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1314. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); - let expr1_0 = C::put_in_reg(ctx, pattern7_1); - let expr2_0 = RegMem::Reg { reg: expr0_0 }; + // Rule at src/isa/x64/lower.isle line 1331. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; + let expr1_0 = constructor_put_in_xmm(ctx, pattern7_1)?; + let expr2_0 = C::xmm_to_xmm_mem(ctx, expr0_0); let expr3_0 = constructor_sse_and(ctx, pattern2_0, expr1_0, &expr2_0)?; - let expr4_0 = C::put_in_reg_mem(ctx, pattern7_2); + let expr4_0 = constructor_put_in_xmm_mem(ctx, pattern7_2)?; let expr5_0 = constructor_sse_and_not(ctx, pattern2_0, expr0_0, &expr4_0)?; - let expr6_0 = RegMem::Reg { reg: expr3_0 }; + let expr6_0 = C::xmm_to_xmm_mem(ctx, expr3_0); let expr7_0 = constructor_sse_or(ctx, pattern2_0, expr5_0, &expr6_0)?; - let expr8_0 = C::value_reg(ctx, expr7_0); + let expr8_0 = constructor_value_xmm(ctx, expr7_0)?; return Some(expr8_0); } &Opcode::Vselect => { let (pattern7_0, pattern7_1, pattern7_2) = C::unpack_value_array_3(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1328. - let expr0_0 = C::put_in_reg_mem(ctx, pattern7_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); - let expr2_0 = C::put_in_reg(ctx, pattern7_2); + // Rule at src/isa/x64/lower.isle line 1345. + let expr0_0 = constructor_put_in_xmm_mem(ctx, pattern7_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; + let expr2_0 = constructor_put_in_xmm(ctx, pattern7_2)?; let expr3_0 = constructor_sse_blend( ctx, pattern2_0, &expr0_0, &expr1_0, expr2_0, )?; - let expr4_0 = C::value_reg(ctx, expr3_0); + let expr4_0 = constructor_value_xmm(ctx, expr3_0)?; return Some(expr4_0); } _ => {} @@ -4703,12 +5097,12 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::Bnot = &pattern5_0 { - // Rule at src/isa/x64/lower.isle line 1309. - let expr0_0 = C::put_in_reg(ctx, pattern5_1); + // Rule at src/isa/x64/lower.isle line 1326. + let expr0_0 = constructor_put_in_xmm(ctx, pattern5_1)?; let expr1_0 = constructor_vector_all_ones(ctx, pattern2_0)?; - let expr2_0 = RegMem::Reg { reg: expr1_0 }; + let expr2_0 = C::xmm_to_xmm_mem(ctx, expr1_0); let expr3_0 = constructor_sse_xor(ctx, pattern2_0, expr0_0, &expr2_0)?; - let expr4_0 = C::value_reg(ctx, expr3_0); + let expr4_0 = constructor_value_xmm(ctx, expr3_0)?; return Some(expr4_0); } } @@ -4830,7 +5224,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 539. + // Rule at src/isa/x64/lower.isle line 543. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_masked_in_imm8_reg(ctx, pattern7_1, pattern3_0); let expr2_0 = constructor_shl(ctx, pattern3_0, expr0_0, &expr1_0)?; @@ -5071,7 +5465,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 635. + // Rule at src/isa/x64/lower.isle line 642. let expr0_0 = ExtendKind::Zero; let expr1_0 = constructor_extend_to_reg(ctx, pattern7_0, pattern3_0, &expr0_0)?; @@ -5083,7 +5477,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 729. + // Rule at src/isa/x64/lower.isle line 741. let expr0_0 = ExtendKind::Sign; let expr1_0 = constructor_extend_to_reg(ctx, pattern7_0, pattern3_0, &expr0_0)?; @@ -5101,17 +5495,17 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { match &pattern5_0 { &Opcode::Ineg => { - // Rule at src/isa/x64/lower.isle line 912. - let expr0_0 = C::put_in_reg(ctx, pattern5_1); + // Rule at src/isa/x64/lower.isle line 927. + let expr0_0 = constructor_put_in_gpr(ctx, pattern5_1)?; let expr1_0 = constructor_neg(ctx, pattern3_0, expr0_0)?; - let expr2_0 = C::value_reg(ctx, expr1_0); + let expr2_0 = constructor_value_gpr(ctx, expr1_0)?; return Some(expr2_0); } &Opcode::Bnot => { - // Rule at src/isa/x64/lower.isle line 1288. - let expr0_0 = C::put_in_reg(ctx, pattern5_1); + // Rule at src/isa/x64/lower.isle line 1305. + let expr0_0 = constructor_put_in_gpr(ctx, pattern5_1)?; let expr1_0 = constructor_not(ctx, pattern3_0, expr0_0)?; - let expr2_0 = C::value_reg(ctx, expr1_0); + let expr2_0 = constructor_value_gpr(ctx, expr1_0)?; return Some(expr2_0); } _ => {} @@ -5155,7 +5549,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option( ctx: &mut C, arg0: Type, - arg1: Reg, - arg2: &RegMem, -) -> Option { + arg1: Xmm, + arg2: &XmmMem, +) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32X4 { let pattern2_0 = arg1; @@ -5326,28 +5720,28 @@ pub fn constructor_sse_and( pub fn constructor_sse_or( ctx: &mut C, arg0: Type, - arg1: Reg, - arg2: &RegMem, -) -> Option { + arg1: Xmm, + arg2: &XmmMem, +) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32X4 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/lower.isle line 436. + // Rule at src/isa/x64/lower.isle line 438. let expr0_0 = constructor_orps(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if pattern0_0 == F64X2 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/lower.isle line 437. + // Rule at src/isa/x64/lower.isle line 439. let expr0_0 = constructor_orpd(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/lower.isle line 438. + // Rule at src/isa/x64/lower.isle line 440. let expr0_0 = constructor_por(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } @@ -5362,7 +5756,7 @@ pub fn constructor_or_i128( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/lower.isle line 447. + // Rule at src/isa/x64/lower.isle line 451. let expr0_0: usize = 0; let expr1_0 = C::value_regs_get(ctx, pattern0_0, expr0_0); let expr2_0: usize = 1; @@ -5389,7 +5783,7 @@ pub fn constructor_shl_i128( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/lower.isle line 545. + // Rule at src/isa/x64/lower.isle line 549. let expr0_0: usize = 0; let expr1_0 = C::value_regs_get(ctx, pattern0_0, expr0_0); let expr2_0: usize = 1; @@ -5448,12 +5842,12 @@ pub fn constructor_ishl_i8x16_mask( let pattern0_0 = arg0; match pattern0_0 { &RegMemImm::Imm { simm32: pattern1_0 } => { - // Rule at src/isa/x64/lower.isle line 600. + // Rule at src/isa/x64/lower.isle line 604. let expr0_0 = C::ishl_i8x16_mask_for_const(ctx, pattern1_0); return Some(expr0_0); } &RegMemImm::Reg { reg: pattern1_0 } => { - // Rule at src/isa/x64/lower.isle line 609. + // Rule at src/isa/x64/lower.isle line 613. let expr0_0 = C::ishl_i8x16_mask_table(ctx); let expr1_0 = constructor_lea(ctx, &expr0_0)?; let expr2_0: Type = I64; @@ -5461,15 +5855,16 @@ pub fn constructor_ishl_i8x16_mask( let expr4_0 = Imm8Reg::Imm8 { imm: expr3_0 }; let expr5_0 = constructor_shl(ctx, expr2_0, pattern1_0, &expr4_0)?; let expr6_0: u32 = 0; - let expr7_0: u8 = 0; - let expr8_0 = C::amode_imm_reg_reg_shift(ctx, expr6_0, expr1_0, expr5_0, expr7_0); - let expr9_0 = C::amode_to_synthetic_amode(ctx, &expr8_0); - return Some(expr9_0); + let expr7_0 = C::gpr_new(ctx, expr5_0); + let expr8_0: u8 = 0; + let expr9_0 = C::amode_imm_reg_reg_shift(ctx, expr6_0, expr1_0, expr7_0, expr8_0); + let expr10_0 = C::amode_to_synthetic_amode(ctx, &expr9_0); + return Some(expr10_0); } &RegMemImm::Mem { addr: ref pattern1_0, } => { - // Rule at src/isa/x64/lower.isle line 617. + // Rule at src/isa/x64/lower.isle line 621. let expr0_0: Type = I64; let expr1_0 = ExtKind::None; let expr2_0 = constructor_x64_load(ctx, expr0_0, &pattern1_0, &expr1_0)?; @@ -5490,7 +5885,7 @@ pub fn constructor_shr_i128( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/lower.isle line 642. + // Rule at src/isa/x64/lower.isle line 649. let expr0_0: usize = 0; let expr1_0 = C::value_regs_get(ctx, pattern0_0, expr0_0); let expr2_0: usize = 1; @@ -5552,12 +5947,12 @@ pub fn constructor_ushr_i8x16_mask( let pattern0_0 = arg0; match pattern0_0 { &RegMemImm::Imm { simm32: pattern1_0 } => { - // Rule at src/isa/x64/lower.isle line 694. + // Rule at src/isa/x64/lower.isle line 703. let expr0_0 = C::ushr_i8x16_mask_for_const(ctx, pattern1_0); return Some(expr0_0); } &RegMemImm::Reg { reg: pattern1_0 } => { - // Rule at src/isa/x64/lower.isle line 703. + // Rule at src/isa/x64/lower.isle line 712. let expr0_0 = C::ushr_i8x16_mask_table(ctx); let expr1_0 = constructor_lea(ctx, &expr0_0)?; let expr2_0: Type = I64; @@ -5565,15 +5960,16 @@ pub fn constructor_ushr_i8x16_mask( let expr4_0 = Imm8Reg::Imm8 { imm: expr3_0 }; let expr5_0 = constructor_shl(ctx, expr2_0, pattern1_0, &expr4_0)?; let expr6_0: u32 = 0; - let expr7_0: u8 = 0; - let expr8_0 = C::amode_imm_reg_reg_shift(ctx, expr6_0, expr1_0, expr5_0, expr7_0); - let expr9_0 = C::amode_to_synthetic_amode(ctx, &expr8_0); - return Some(expr9_0); + let expr7_0 = C::gpr_new(ctx, expr5_0); + let expr8_0: u8 = 0; + let expr9_0 = C::amode_imm_reg_reg_shift(ctx, expr6_0, expr1_0, expr7_0, expr8_0); + let expr10_0 = C::amode_to_synthetic_amode(ctx, &expr9_0); + return Some(expr10_0); } &RegMemImm::Mem { addr: ref pattern1_0, } => { - // Rule at src/isa/x64/lower.isle line 711. + // Rule at src/isa/x64/lower.isle line 720. let expr0_0: Type = I64; let expr1_0 = ExtKind::None; let expr2_0 = constructor_x64_load(ctx, expr0_0, &pattern1_0, &expr1_0)?; @@ -5594,7 +5990,7 @@ pub fn constructor_sar_i128( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/lower.isle line 736. + // Rule at src/isa/x64/lower.isle line 748. let expr0_0: usize = 0; let expr1_0 = C::value_regs_get(ctx, pattern0_0, expr0_0); let expr2_0: usize = 1; @@ -5654,35 +6050,36 @@ pub fn constructor_sshr_i8x16_bigger_shift( ctx: &mut C, arg0: Type, arg1: &RegMemImm, -) -> Option { +) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; match pattern1_0 { &RegMemImm::Imm { simm32: pattern2_0 } => { - // Rule at src/isa/x64/lower.isle line 795. + // Rule at src/isa/x64/lower.isle line 807. let expr0_0: u32 = 8; let expr1_0 = C::u32_add(ctx, pattern2_0, expr0_0); let expr2_0 = RegMemImm::Imm { simm32: expr1_0 }; - return Some(expr2_0); + let expr3_0 = C::xmm_mem_imm_new(ctx, &expr2_0); + return Some(expr3_0); } &RegMemImm::Reg { reg: pattern2_0 } => { - // Rule at src/isa/x64/lower.isle line 797. + // Rule at src/isa/x64/lower.isle line 809. let expr0_0: u32 = 8; let expr1_0 = RegMemImm::Imm { simm32: expr0_0 }; let expr2_0 = constructor_add(ctx, pattern0_0, pattern2_0, &expr1_0)?; let expr3_0 = RegMemImm::Reg { reg: expr2_0 }; - let expr4_0 = constructor_reg_mem_imm_to_xmm(ctx, &expr3_0)?; + let expr4_0 = constructor_mov_rmi_to_xmm(ctx, &expr3_0)?; return Some(expr4_0); } &RegMemImm::Mem { addr: ref pattern2_0, } => { - // Rule at src/isa/x64/lower.isle line 799. + // Rule at src/isa/x64/lower.isle line 811. let expr0_0: u64 = 8; let expr1_0 = constructor_imm(ctx, pattern0_0, expr0_0)?; let expr2_0 = constructor_add(ctx, pattern0_0, expr1_0, pattern1_0)?; let expr3_0 = RegMemImm::Reg { reg: expr2_0 }; - let expr4_0 = constructor_reg_mem_imm_to_xmm(ctx, &expr3_0)?; + let expr4_0 = constructor_mov_rmi_to_xmm(ctx, &expr3_0)?; return Some(expr4_0); } _ => {} @@ -5694,28 +6091,28 @@ pub fn constructor_sshr_i8x16_bigger_shift( pub fn constructor_sse_and_not( ctx: &mut C, arg0: Type, - arg1: Reg, - arg2: &RegMem, -) -> Option { + arg1: Xmm, + arg2: &XmmMem, +) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32X4 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/lower.isle line 1229. + // Rule at src/isa/x64/lower.isle line 1244. let expr0_0 = constructor_andnps(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if pattern0_0 == F64X2 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/lower.isle line 1230. + // Rule at src/isa/x64/lower.isle line 1245. let expr0_0 = constructor_andnpd(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/lower.isle line 1231. + // Rule at src/isa/x64/lower.isle line 1246. let expr0_0 = constructor_pandn(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } @@ -5725,71 +6122,78 @@ pub fn constructor_sse_and_not( // Generated as internal constructor for term i128_not. pub fn constructor_i128_not(ctx: &mut C, arg0: Value) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/lower.isle line 1294. + // Rule at src/isa/x64/lower.isle line 1311. let expr0_0 = C::put_in_regs(ctx, pattern0_0); let expr1_0: usize = 0; let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); - let expr3_0: usize = 1; - let expr4_0 = C::value_regs_get(ctx, expr0_0, expr3_0); - let expr5_0: Type = I64; - let expr6_0 = constructor_not(ctx, expr5_0, expr2_0)?; + let expr3_0 = C::gpr_new(ctx, expr2_0); + let expr4_0: usize = 1; + let expr5_0 = C::value_regs_get(ctx, expr0_0, expr4_0); + let expr6_0 = C::gpr_new(ctx, expr5_0); let expr7_0: Type = I64; - let expr8_0 = constructor_not(ctx, expr7_0, expr4_0)?; - let expr9_0 = C::value_regs(ctx, expr6_0, expr8_0); - return Some(expr9_0); + let expr8_0 = constructor_not(ctx, expr7_0, expr3_0)?; + let expr9_0: Type = I64; + let expr10_0 = constructor_not(ctx, expr9_0, expr6_0)?; + let expr11_0 = constructor_value_gprs(ctx, expr8_0, expr10_0)?; + return Some(expr11_0); } // Generated as internal constructor for term vec_insert_lane. pub fn constructor_vec_insert_lane( ctx: &mut C, arg0: Type, - arg1: Reg, + arg1: Xmm, arg2: &RegMem, arg3: u8, -) -> Option { +) -> Option { let pattern0_0 = arg0; if pattern0_0 == I8X16 { let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/lower.isle line 1348. - let expr0_0 = constructor_pinsrb(ctx, pattern2_0, pattern3_0, pattern4_0)?; - return Some(expr0_0); + // Rule at src/isa/x64/lower.isle line 1365. + let expr0_0 = C::gpr_mem_new(ctx, pattern3_0); + let expr1_0 = constructor_pinsrb(ctx, pattern2_0, &expr0_0, pattern4_0)?; + return Some(expr1_0); } if pattern0_0 == I16X8 { let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/lower.isle line 1351. - let expr0_0 = constructor_pinsrw(ctx, pattern2_0, pattern3_0, pattern4_0)?; - return Some(expr0_0); + // Rule at src/isa/x64/lower.isle line 1369. + let expr0_0 = C::gpr_mem_new(ctx, pattern3_0); + let expr1_0 = constructor_pinsrw(ctx, pattern2_0, &expr0_0, pattern4_0)?; + return Some(expr1_0); } if pattern0_0 == I32X4 { let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/lower.isle line 1354. - let expr0_0 = OperandSize::Size32; - let expr1_0 = constructor_pinsrd(ctx, pattern2_0, pattern3_0, pattern4_0, &expr0_0)?; - return Some(expr1_0); + // Rule at src/isa/x64/lower.isle line 1373. + let expr0_0 = C::gpr_mem_new(ctx, pattern3_0); + let expr1_0 = OperandSize::Size32; + let expr2_0 = constructor_pinsrd(ctx, pattern2_0, &expr0_0, pattern4_0, &expr1_0)?; + return Some(expr2_0); } if pattern0_0 == I64X2 { let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/lower.isle line 1357. - let expr0_0 = OperandSize::Size64; - let expr1_0 = constructor_pinsrd(ctx, pattern2_0, pattern3_0, pattern4_0, &expr0_0)?; - return Some(expr1_0); + // Rule at src/isa/x64/lower.isle line 1377. + let expr0_0 = C::gpr_mem_new(ctx, pattern3_0); + let expr1_0 = OperandSize::Size64; + let expr2_0 = constructor_pinsrd(ctx, pattern2_0, &expr0_0, pattern4_0, &expr1_0)?; + return Some(expr2_0); } if pattern0_0 == F32X4 { let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/lower.isle line 1360. - let expr0_0 = C::sse_insertps_lane_imm(ctx, pattern4_0); - let expr1_0 = constructor_insertps(ctx, pattern2_0, pattern3_0, expr0_0)?; - return Some(expr1_0); + // Rule at src/isa/x64/lower.isle line 1381. + let expr0_0 = C::xmm_mem_new(ctx, pattern3_0); + let expr1_0 = C::sse_insertps_lane_imm(ctx, pattern4_0); + let expr2_0 = constructor_insertps(ctx, pattern2_0, &expr0_0, expr1_0)?; + return Some(expr2_0); } if pattern0_0 == F64X2 { let pattern2_0 = arg1; @@ -5797,25 +6201,28 @@ pub fn constructor_vec_insert_lane( if let &RegMem::Reg { reg: pattern4_0 } = pattern3_0 { let pattern5_0 = arg3; if pattern5_0 == 0 { - // Rule at src/isa/x64/lower.isle line 1381. + // Rule at src/isa/x64/lower.isle line 1403. let expr0_0 = RegMem::Reg { reg: pattern4_0 }; - let expr1_0 = constructor_movsd(ctx, pattern2_0, &expr0_0)?; - return Some(expr1_0); + let expr1_0 = C::xmm_mem_new(ctx, &expr0_0); + let expr2_0 = constructor_movsd(ctx, pattern2_0, &expr1_0)?; + return Some(expr2_0); } } let pattern4_0 = arg3; if pattern4_0 == 0 { - // Rule at src/isa/x64/lower.isle line 1382. + // Rule at src/isa/x64/lower.isle line 1405. let expr0_0 = SseOpcode::Movsd; - let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern3_0)?; - let expr2_0 = RegMem::Reg { reg: expr1_0 }; - let expr3_0 = constructor_movsd(ctx, pattern2_0, &expr2_0)?; - return Some(expr3_0); + let expr1_0 = C::xmm_mem_new(ctx, pattern3_0); + let expr2_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, &expr1_0)?; + let expr3_0 = C::xmm_to_xmm_mem(ctx, expr2_0); + let expr4_0 = constructor_movsd(ctx, pattern2_0, &expr3_0)?; + return Some(expr4_0); } if pattern4_0 == 1 { - // Rule at src/isa/x64/lower.isle line 1390. - let expr0_0 = constructor_movlhps(ctx, pattern2_0, pattern3_0)?; - return Some(expr0_0); + // Rule at src/isa/x64/lower.isle line 1414. + let expr0_0 = C::xmm_mem_new(ctx, pattern3_0); + let expr1_0 = constructor_movlhps(ctx, pattern2_0, &expr0_0)?; + return Some(expr1_0); } } return None; diff --git a/cranelift/codegen/src/machinst/isle.rs b/cranelift/codegen/src/machinst/isle.rs index 1869ba6762a8..b54ba51a5366 100644 --- a/cranelift/codegen/src/machinst/isle.rs +++ b/cranelift/codegen/src/machinst/isle.rs @@ -1,14 +1,25 @@ use crate::ir::{Inst, Value}; use crate::machinst::{get_output_reg, InsnOutput, LowerCtx, MachInst, RegRenamer}; +use alloc::boxed::Box; +use alloc::vec::Vec; use regalloc::{Reg, Writable}; use smallvec::SmallVec; +pub use super::MachLabel; +pub use crate::ir::ExternalName; +pub use crate::isa::unwind::UnwindInst; + pub type Unit = (); pub type ValueSlice<'a> = &'a [Value]; pub type ValueArray2 = [Value; 2]; pub type ValueArray3 = [Value; 3]; pub type WritableReg = Writable; +pub type OptionWritableReg = Option; +pub type VecReg = Vec; +pub type VecWritableReg = Vec; pub type ValueRegs = crate::machinst::ValueRegs; +pub type VecMachLabel = Vec; +pub type BoxExternalName = Box; /// Helper macro to define methods in `prelude.isle` within `impl Context for /// ...` for each backend. These methods are shared amongst all backends. diff --git a/cranelift/codegen/src/prelude.isle b/cranelift/codegen/src/prelude.isle index 5b7d161d7ea9..78f0395f42aa 100644 --- a/cranelift/codegen/src/prelude.isle +++ b/cranelift/codegen/src/prelude.isle @@ -48,6 +48,9 @@ (type Reg (primitive Reg)) (type WritableReg (primitive WritableReg)) +(type OptionWritableReg (primitive OptionWritableReg)) +(type VecReg extern (enum)) +(type VecWritableReg extern (enum)) ;; Construct a `ValueRegs` of one register. (decl value_reg (Reg) ValueRegs) @@ -106,6 +109,15 @@ (let ((regs ValueRegs (put_in_regs val))) (value_regs_get regs 0))) +;;;; Common Mach Types ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(type MachLabel (primitive MachLabel)) +(type VecMachLabel extern (enum)) +(type ValueLabel (primitive ValueLabel)) +(type UnwindInst (primitive UnwindInst)) +(type ExternalName (primitive ExternalName)) +(type BoxExternalName (primitive BoxExternalName)) + ;;;; Primitive Type Conversions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (decl u8_as_u64 (u8) u64) @@ -368,4 +380,3 @@ (decl avoid_div_traps () Type) (extern extractor avoid_div_traps avoid_div_traps) - From 2c77cf866a63fb1077a5f3734be55265eb6646ea Mon Sep 17 00:00:00 2001 From: Nick Fitzgerald Date: Thu, 3 Feb 2022 11:31:53 -0800 Subject: [PATCH 3/3] ISLE: Rename `{gpr,xmm}_mem_new` constructors to `reg_mem_to_{gpr,xmm}_mem` --- cranelift/codegen/src/isa/x64/inst.isle | 26 +++++------ cranelift/codegen/src/isa/x64/lower.isle | 24 +++++----- cranelift/codegen/src/isa/x64/lower/isle.rs | 4 +- .../x64/lower/isle/generated_code.manifest | 4 +- .../src/isa/x64/lower/isle/generated_code.rs | 46 +++++++++---------- 5 files changed, 52 insertions(+), 52 deletions(-) diff --git a/cranelift/codegen/src/isa/x64/inst.isle b/cranelift/codegen/src/isa/x64/inst.isle index 6f3081a4bbf1..7c23e75a0316 100644 --- a/cranelift/codegen/src/isa/x64/inst.isle +++ b/cranelift/codegen/src/isa/x64/inst.isle @@ -876,8 +876,8 @@ ;; Construct a new `XmmMem` from the given `RegMem`. ;; ;; Asserts that the `RegMem`'s register, if any, is an XMM register. -(decl xmm_mem_new (RegMem) XmmMem) -(extern constructor xmm_mem_new xmm_mem_new) +(decl reg_mem_to_xmm_mem (RegMem) XmmMem) +(extern constructor reg_mem_to_xmm_mem reg_mem_to_xmm_mem) ;; Construct a new `GprMemImm` from the given `RegMemImm`. ;; @@ -918,8 +918,8 @@ ;; Construct a new `GprMem` from a `RegMem`. ;; ;; Asserts that the `RegMem`'s register, if any, is a GPR. -(decl gpr_mem_new (RegMem) GprMem) -(extern constructor gpr_mem_new gpr_mem_new) +(decl reg_mem_to_gpr_mem (RegMem) GprMem) +(extern constructor reg_mem_to_gpr_mem reg_mem_to_gpr_mem) ;; Construct a `GprMem` from a `Reg`. ;; @@ -939,7 +939,7 @@ ;; Asserts that the value goes into a GPR. (decl put_in_gpr_mem (Value) GprMem) (rule (put_in_gpr_mem val) - (gpr_mem_new (put_in_reg_mem val))) + (reg_mem_to_gpr_mem (put_in_reg_mem val))) ;; Put a value into a `GprMemImm`. ;; @@ -960,7 +960,7 @@ ;; Asserts that the value goes into a XMM. (decl put_in_xmm_mem (Value) XmmMem) (rule (put_in_xmm_mem val) - (xmm_mem_new (put_in_reg_mem val))) + (reg_mem_to_xmm_mem (put_in_reg_mem val))) ;; Put a value into a `XmmMemImm`. ;; @@ -1177,23 +1177,23 @@ (rule (x64_load $F32 addr _ext_kind) (xmm_to_reg (xmm_unary_rm_r (SseOpcode.Movss) - (xmm_mem_new (synthetic_amode_to_reg_mem addr))))) + (reg_mem_to_xmm_mem (synthetic_amode_to_reg_mem addr))))) (rule (x64_load $F64 addr _ext_kind) (xmm_to_reg (xmm_unary_rm_r (SseOpcode.Movsd) - (xmm_mem_new (synthetic_amode_to_reg_mem addr))))) + (reg_mem_to_xmm_mem (synthetic_amode_to_reg_mem addr))))) (rule (x64_load $F32X4 addr _ext_kind) (xmm_to_reg (xmm_unary_rm_r (SseOpcode.Movups) - (xmm_mem_new (synthetic_amode_to_reg_mem addr))))) + (reg_mem_to_xmm_mem (synthetic_amode_to_reg_mem addr))))) (rule (x64_load $F64X2 addr _ext_kind) (xmm_to_reg (xmm_unary_rm_r (SseOpcode.Movupd) - (xmm_mem_new (synthetic_amode_to_reg_mem addr))))) + (reg_mem_to_xmm_mem (synthetic_amode_to_reg_mem addr))))) (rule (x64_load (multi_lane _bits _lanes) addr _ext_kind) (xmm_to_reg (xmm_unary_rm_r (SseOpcode.Movdqu) - (xmm_mem_new (synthetic_amode_to_reg_mem addr))))) + (reg_mem_to_xmm_mem (synthetic_amode_to_reg_mem addr))))) ;;;; Instruction Constructors ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; @@ -1317,13 +1317,13 @@ ;; `f32` immediates. (rule (imm $F32 bits) (xmm_to_reg (gpr_to_xmm (SseOpcode.Movd) - (gpr_mem_new (RegMem.Reg (imm $I32 bits))) + (reg_mem_to_gpr_mem (RegMem.Reg (imm $I32 bits))) (OperandSize.Size32)))) ;; `f64` immediates. (rule (imm $F64 bits) (xmm_to_reg (gpr_to_xmm (SseOpcode.Movq) - (gpr_mem_new (RegMem.Reg (imm $I64 bits))) + (reg_mem_to_gpr_mem (RegMem.Reg (imm $I64 bits))) (OperandSize.Size64)))) (decl nonzero_u64_fits_in_u32 (u64) u64) diff --git a/cranelift/codegen/src/isa/x64/lower.isle b/cranelift/codegen/src/isa/x64/lower.isle index 8de9110bbdfe..8abdc50dd339 100644 --- a/cranelift/codegen/src/isa/x64/lower.isle +++ b/cranelift/codegen/src/isa/x64/lower.isle @@ -591,7 +591,7 @@ (unmasked Xmm (psllw src_ amt_xmm)) (mask_addr SyntheticAmode (ishl_i8x16_mask amt_gpr)) (mask Reg (x64_load $I8X16 mask_addr (ExtKind.None)))) - (value_xmm (sse_and $I8X16 unmasked (xmm_mem_new (RegMem.Reg mask)))))) + (value_xmm (sse_and $I8X16 unmasked (reg_mem_to_xmm_mem (RegMem.Reg mask)))))) ;; Get the address of the mask to use when fixing up the lanes that weren't ;; correctly generated by the 16x8 shift. @@ -690,7 +690,7 @@ (mask Reg (x64_load $I8X16 mask_addr (ExtKind.None)))) (value_xmm (sse_and $I8X16 unmasked - (xmm_mem_new (RegMem.Reg mask)))))) + (reg_mem_to_xmm_mem (RegMem.Reg mask)))))) ;; Get the address of the mask to use when fixing up the lanes that weren't ;; correctly generated by the 16x8 shift. @@ -837,8 +837,8 @@ (amt_ Imm8Reg (put_masked_in_imm8_reg amt $I64)) (shifted_lo Reg (sar $I64 (gpr_to_reg lo) amt_)) (shifted_hi Reg (sar $I64 (gpr_to_reg hi) amt_))) - (value_xmm (make_i64x2_from_lanes (gpr_mem_new (RegMem.Reg shifted_lo)) - (gpr_mem_new (RegMem.Reg shifted_hi)))))) + (value_xmm (make_i64x2_from_lanes (reg_mem_to_gpr_mem (RegMem.Reg shifted_lo)) + (reg_mem_to_gpr_mem (RegMem.Reg shifted_hi)))))) ;;;; Rules for `rotl` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -1363,23 +1363,23 @@ ;; i8x16.replace_lane (rule (vec_insert_lane $I8X16 vec val idx) - (pinsrb vec (gpr_mem_new val) idx)) + (pinsrb vec (reg_mem_to_gpr_mem val) idx)) ;; i16x8.replace_lane (rule (vec_insert_lane $I16X8 vec val idx) - (pinsrw vec (gpr_mem_new val) idx)) + (pinsrw vec (reg_mem_to_gpr_mem val) idx)) ;; i32x4.replace_lane (rule (vec_insert_lane $I32X4 vec val idx) - (pinsrd vec (gpr_mem_new val) idx (OperandSize.Size32))) + (pinsrd vec (reg_mem_to_gpr_mem val) idx (OperandSize.Size32))) ;; i64x2.replace_lane (rule (vec_insert_lane $I64X2 vec val idx) - (pinsrd vec (gpr_mem_new val) idx (OperandSize.Size64))) + (pinsrd vec (reg_mem_to_gpr_mem val) idx (OperandSize.Size64))) ;; f32x4.replace_lane (rule (vec_insert_lane $F32X4 vec val idx) - (insertps vec (xmm_mem_new val) (sse_insertps_lane_imm idx))) + (insertps vec (reg_mem_to_xmm_mem val) (sse_insertps_lane_imm idx))) ;; External rust code used to calculate the immediate value to `insertps`. (decl sse_insertps_lane_imm (u8) u8) @@ -1401,10 +1401,10 @@ ;; internally as `xmm_rm_r` will merge the temp register into our `vec` ;; register. (rule (vec_insert_lane $F64X2 vec (RegMem.Reg val) 0) - (movsd vec (xmm_mem_new (RegMem.Reg val)))) + (movsd vec (reg_mem_to_xmm_mem (RegMem.Reg val)))) (rule (vec_insert_lane $F64X2 vec mem 0) (movsd vec (xmm_to_xmm_mem (xmm_unary_rm_r (SseOpcode.Movsd) - (xmm_mem_new mem))))) + (reg_mem_to_xmm_mem mem))))) ;; f64x2.replace_lane 1 ;; @@ -1412,7 +1412,7 @@ ;; into the second lane where unlike above cases we're not using the lane ;; immediate as an immediate to the instruction itself. (rule (vec_insert_lane $F64X2 vec val 1) - (movlhps vec (xmm_mem_new val))) + (movlhps vec (reg_mem_to_xmm_mem val))) ;;;; Rules for `imax` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; diff --git a/cranelift/codegen/src/isa/x64/lower/isle.rs b/cranelift/codegen/src/isa/x64/lower/isle.rs index 87a5ea9f6ece..f5aa60945738 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle.rs +++ b/cranelift/codegen/src/isa/x64/lower/isle.rs @@ -360,7 +360,7 @@ where } #[inline] - fn xmm_mem_new(&mut self, rm: &RegMem) -> XmmMem { + fn reg_mem_to_xmm_mem(&mut self, rm: &RegMem) -> XmmMem { XmmMem::new(rm.clone()).unwrap() } @@ -400,7 +400,7 @@ where } #[inline] - fn gpr_mem_new(&mut self, rm: &RegMem) -> GprMem { + fn reg_mem_to_gpr_mem(&mut self, rm: &RegMem) -> GprMem { GprMem::new(rm.clone()).unwrap() } diff --git a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest index 2c8fb4b29805..b8ba09a9eb46 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest +++ b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest @@ -1,4 +1,4 @@ src/clif.isle 9ea75a6f790b5c03 src/prelude.isle 6aaf8ce0f5a5c2ec -src/isa/x64/inst.isle 2f76eb1f9ecf0c5e -src/isa/x64/lower.isle 144c33c4e64a17a7 +src/isa/x64/inst.isle 7513533d16948249 +src/isa/x64/lower.isle ccda13e9fe83c89a diff --git a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs index 31aaea3e4844..5529576b3298 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs +++ b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs @@ -84,7 +84,7 @@ pub trait Context { fn xmm_to_xmm_mem_imm(&mut self, arg0: Xmm) -> XmmMemImm; fn temp_writable_gpr(&mut self) -> WritableGpr; fn temp_writable_xmm(&mut self) -> WritableXmm; - fn xmm_mem_new(&mut self, arg0: &RegMem) -> XmmMem; + fn reg_mem_to_xmm_mem(&mut self, arg0: &RegMem) -> XmmMem; fn gpr_mem_imm_new(&mut self, arg0: &RegMemImm) -> GprMemImm; fn xmm_mem_imm_new(&mut self, arg0: &RegMemImm) -> XmmMemImm; fn xmm_to_xmm_mem(&mut self, arg0: Xmm) -> XmmMem; @@ -92,7 +92,7 @@ pub trait Context { fn gpr_mem_to_reg_mem(&mut self, arg0: &GprMem) -> RegMem; fn xmm_new(&mut self, arg0: Reg) -> Xmm; fn gpr_new(&mut self, arg0: Reg) -> Gpr; - fn gpr_mem_new(&mut self, arg0: &RegMem) -> GprMem; + fn reg_mem_to_gpr_mem(&mut self, arg0: &RegMem) -> GprMem; fn reg_to_gpr_mem(&mut self, arg0: Reg) -> GprMem; fn xmm0(&mut self) -> WritableXmm; fn avx512vl_enabled(&mut self, arg0: Type) -> Option<()>; @@ -639,7 +639,7 @@ pub fn constructor_put_in_gpr_mem(ctx: &mut C, arg0: Value) -> Optio let pattern0_0 = arg0; // Rule at src/isa/x64/inst.isle line 941. let expr0_0 = C::put_in_reg_mem(ctx, pattern0_0); - let expr1_0 = C::gpr_mem_new(ctx, &expr0_0); + let expr1_0 = C::reg_mem_to_gpr_mem(ctx, &expr0_0); return Some(expr1_0); } @@ -666,7 +666,7 @@ pub fn constructor_put_in_xmm_mem(ctx: &mut C, arg0: Value) -> Optio let pattern0_0 = arg0; // Rule at src/isa/x64/inst.isle line 962. let expr0_0 = C::put_in_reg_mem(ctx, pattern0_0); - let expr1_0 = C::xmm_mem_new(ctx, &expr0_0); + let expr1_0 = C::reg_mem_to_xmm_mem(ctx, &expr0_0); return Some(expr1_0); } @@ -970,7 +970,7 @@ pub fn constructor_x64_load( // Rule at src/isa/x64/inst.isle line 1178. let expr0_0 = SseOpcode::Movss; let expr1_0 = C::synthetic_amode_to_reg_mem(ctx, pattern2_0); - let expr2_0 = C::xmm_mem_new(ctx, &expr1_0); + let expr2_0 = C::reg_mem_to_xmm_mem(ctx, &expr1_0); let expr3_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, &expr2_0)?; let expr4_0 = C::xmm_to_reg(ctx, expr3_0); return Some(expr4_0); @@ -981,7 +981,7 @@ pub fn constructor_x64_load( // Rule at src/isa/x64/inst.isle line 1182. let expr0_0 = SseOpcode::Movsd; let expr1_0 = C::synthetic_amode_to_reg_mem(ctx, pattern2_0); - let expr2_0 = C::xmm_mem_new(ctx, &expr1_0); + let expr2_0 = C::reg_mem_to_xmm_mem(ctx, &expr1_0); let expr3_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, &expr2_0)?; let expr4_0 = C::xmm_to_reg(ctx, expr3_0); return Some(expr4_0); @@ -992,7 +992,7 @@ pub fn constructor_x64_load( // Rule at src/isa/x64/inst.isle line 1186. let expr0_0 = SseOpcode::Movups; let expr1_0 = C::synthetic_amode_to_reg_mem(ctx, pattern2_0); - let expr2_0 = C::xmm_mem_new(ctx, &expr1_0); + let expr2_0 = C::reg_mem_to_xmm_mem(ctx, &expr1_0); let expr3_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, &expr2_0)?; let expr4_0 = C::xmm_to_reg(ctx, expr3_0); return Some(expr4_0); @@ -1003,7 +1003,7 @@ pub fn constructor_x64_load( // Rule at src/isa/x64/inst.isle line 1190. let expr0_0 = SseOpcode::Movupd; let expr1_0 = C::synthetic_amode_to_reg_mem(ctx, pattern2_0); - let expr2_0 = C::xmm_mem_new(ctx, &expr1_0); + let expr2_0 = C::reg_mem_to_xmm_mem(ctx, &expr1_0); let expr3_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, &expr2_0)?; let expr4_0 = C::xmm_to_reg(ctx, expr3_0); return Some(expr4_0); @@ -1014,7 +1014,7 @@ pub fn constructor_x64_load( // Rule at src/isa/x64/inst.isle line 1194. let expr0_0 = SseOpcode::Movdqu; let expr1_0 = C::synthetic_amode_to_reg_mem(ctx, pattern2_0); - let expr2_0 = C::xmm_mem_new(ctx, &expr1_0); + let expr2_0 = C::reg_mem_to_xmm_mem(ctx, &expr1_0); let expr3_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, &expr2_0)?; let expr4_0 = C::xmm_to_reg(ctx, expr3_0); return Some(expr4_0); @@ -1317,7 +1317,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option let expr1_0: Type = I32; let expr2_0 = constructor_imm(ctx, expr1_0, pattern2_0)?; let expr3_0 = RegMem::Reg { reg: expr2_0 }; - let expr4_0 = C::gpr_mem_new(ctx, &expr3_0); + let expr4_0 = C::reg_mem_to_gpr_mem(ctx, &expr3_0); let expr5_0 = OperandSize::Size32; let expr6_0 = constructor_gpr_to_xmm(ctx, &expr0_0, &expr4_0, &expr5_0)?; let expr7_0 = C::xmm_to_reg(ctx, expr6_0); @@ -1346,7 +1346,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option let expr1_0: Type = I64; let expr2_0 = constructor_imm(ctx, expr1_0, pattern2_0)?; let expr3_0 = RegMem::Reg { reg: expr2_0 }; - let expr4_0 = C::gpr_mem_new(ctx, &expr3_0); + let expr4_0 = C::reg_mem_to_gpr_mem(ctx, &expr3_0); let expr5_0 = OperandSize::Size64; let expr6_0 = constructor_gpr_to_xmm(ctx, &expr0_0, &expr4_0, &expr5_0)?; let expr7_0 = C::xmm_to_reg(ctx, expr6_0); @@ -3456,7 +3456,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option( let pattern3_0 = arg2; let pattern4_0 = arg3; // Rule at src/isa/x64/lower.isle line 1365. - let expr0_0 = C::gpr_mem_new(ctx, pattern3_0); + let expr0_0 = C::reg_mem_to_gpr_mem(ctx, pattern3_0); let expr1_0 = constructor_pinsrb(ctx, pattern2_0, &expr0_0, pattern4_0)?; return Some(expr1_0); } @@ -6161,7 +6161,7 @@ pub fn constructor_vec_insert_lane( let pattern3_0 = arg2; let pattern4_0 = arg3; // Rule at src/isa/x64/lower.isle line 1369. - let expr0_0 = C::gpr_mem_new(ctx, pattern3_0); + let expr0_0 = C::reg_mem_to_gpr_mem(ctx, pattern3_0); let expr1_0 = constructor_pinsrw(ctx, pattern2_0, &expr0_0, pattern4_0)?; return Some(expr1_0); } @@ -6170,7 +6170,7 @@ pub fn constructor_vec_insert_lane( let pattern3_0 = arg2; let pattern4_0 = arg3; // Rule at src/isa/x64/lower.isle line 1373. - let expr0_0 = C::gpr_mem_new(ctx, pattern3_0); + let expr0_0 = C::reg_mem_to_gpr_mem(ctx, pattern3_0); let expr1_0 = OperandSize::Size32; let expr2_0 = constructor_pinsrd(ctx, pattern2_0, &expr0_0, pattern4_0, &expr1_0)?; return Some(expr2_0); @@ -6180,7 +6180,7 @@ pub fn constructor_vec_insert_lane( let pattern3_0 = arg2; let pattern4_0 = arg3; // Rule at src/isa/x64/lower.isle line 1377. - let expr0_0 = C::gpr_mem_new(ctx, pattern3_0); + let expr0_0 = C::reg_mem_to_gpr_mem(ctx, pattern3_0); let expr1_0 = OperandSize::Size64; let expr2_0 = constructor_pinsrd(ctx, pattern2_0, &expr0_0, pattern4_0, &expr1_0)?; return Some(expr2_0); @@ -6190,7 +6190,7 @@ pub fn constructor_vec_insert_lane( let pattern3_0 = arg2; let pattern4_0 = arg3; // Rule at src/isa/x64/lower.isle line 1381. - let expr0_0 = C::xmm_mem_new(ctx, pattern3_0); + let expr0_0 = C::reg_mem_to_xmm_mem(ctx, pattern3_0); let expr1_0 = C::sse_insertps_lane_imm(ctx, pattern4_0); let expr2_0 = constructor_insertps(ctx, pattern2_0, &expr0_0, expr1_0)?; return Some(expr2_0); @@ -6203,7 +6203,7 @@ pub fn constructor_vec_insert_lane( if pattern5_0 == 0 { // Rule at src/isa/x64/lower.isle line 1403. let expr0_0 = RegMem::Reg { reg: pattern4_0 }; - let expr1_0 = C::xmm_mem_new(ctx, &expr0_0); + let expr1_0 = C::reg_mem_to_xmm_mem(ctx, &expr0_0); let expr2_0 = constructor_movsd(ctx, pattern2_0, &expr1_0)?; return Some(expr2_0); } @@ -6212,7 +6212,7 @@ pub fn constructor_vec_insert_lane( if pattern4_0 == 0 { // Rule at src/isa/x64/lower.isle line 1405. let expr0_0 = SseOpcode::Movsd; - let expr1_0 = C::xmm_mem_new(ctx, pattern3_0); + let expr1_0 = C::reg_mem_to_xmm_mem(ctx, pattern3_0); let expr2_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, &expr1_0)?; let expr3_0 = C::xmm_to_xmm_mem(ctx, expr2_0); let expr4_0 = constructor_movsd(ctx, pattern2_0, &expr3_0)?; @@ -6220,7 +6220,7 @@ pub fn constructor_vec_insert_lane( } if pattern4_0 == 1 { // Rule at src/isa/x64/lower.isle line 1414. - let expr0_0 = C::xmm_mem_new(ctx, pattern3_0); + let expr0_0 = C::reg_mem_to_xmm_mem(ctx, pattern3_0); let expr1_0 = constructor_movlhps(ctx, pattern2_0, &expr0_0)?; return Some(expr1_0); }