Auto merge of #120080 - cuviper:128-align-packed, r=<try>

[WIP] pack u128 in the compiler to mitigate new alignment This is based on #116672, adding a new `#[repr(packed(8))]` wrapper on `u128` to avoid changing any of the compiler's size assertions. This is needed in two places: * `SwitchTargets`, otherwise its `SmallVec<[u128; 1]>` gets padded up to 32 bytes. * `LitKind::Int`, so that entire `enum` can stay 24 bytes. * This change definitely has far-reaching effects though, since it's public. r? ghost
rust-lang · Jan 18, 2024 · f72022d · f72022d
2 parents 2457c02 + 477f5f5
commit f72022d
Show file tree

Hide file tree

Showing 54 changed files with 175 additions and 88 deletions.
diff --git a/compiler/rustc_ast/src/ast.rs b/compiler/rustc_ast/src/ast.rs
@@ -3300,9 +3300,13 @@ mod size_asserts {
     static_assert_size!(Impl, 136);
     static_assert_size!(Item, 136);
     static_assert_size!(ItemKind, 64);
-    static_assert_size!(LitKind, 24);
+    // This can be removed after i128:128 is in the bootstrap compiler's target.
+    #[cfg(not(bootstrap))]
+    static_assert_size!(LitKind, 32);
     static_assert_size!(Local, 72);
-    static_assert_size!(MetaItemLit, 40);
+    // This can be removed after i128:128 is in the bootstrap compiler's target.
+    #[cfg(not(bootstrap))]
+    static_assert_size!(MetaItemLit, 48);
     static_assert_size!(Param, 40);
     static_assert_size!(Pat, 72);
     static_assert_size!(Path, 24);

diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs
@@ -145,6 +145,14 @@ pub unsafe fn create_module<'ll>(
                 .replace("-Fi64", "");
         }
     }
+    if llvm_version < (18, 0, 0) {
+        if sess.target.arch == "x86" || sess.target.arch == "x86_64" {
+            // LLVM 18 adjusts i128 to be 128-bit aligned on x86 variants.
+            // Earlier LLVMs leave this as default alignment, so remove it.
+            // See https://reviews.llvm.org/D86310
+            target_data_layout = target_data_layout.replace("-i128:128", "");
+        }
+    }
 
     // Ensure the data-layout values hardcoded remain the defaults.
     if sess.target.is_builtin {

diff --git a/compiler/rustc_data_structures/src/lib.rs b/compiler/rustc_data_structures/src/lib.rs
@@ -93,6 +93,7 @@ pub mod aligned;
 pub mod frozen;
 mod hashes;
 pub mod owned_slice;
+pub mod packed;
 pub mod sso;
 pub mod steal;
 pub mod tagged_ptr;

diff --git a/compiler/rustc_data_structures/src/packed.rs b/compiler/rustc_data_structures/src/packed.rs
@@ -0,0 +1,71 @@
+use crate::stable_hasher::{HashStable, StableHasher};
+use rustc_serialize::{Decodable, Decoder, Encodable, Encoder};
+use std::cmp::Ordering;
+use std::fmt;
+
+#[repr(packed(8))]
+#[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord)]
+pub struct Pu128(pub u128);
+
+impl Pu128 {
+    #[inline]
+    pub fn get(self) -> u128 {
+        self.0
+    }
+}
+
+impl From<u128> for Pu128 {
+    #[inline]
+    fn from(value: u128) -> Self {
+        Self(value)
+    }
+}
+
+impl PartialEq<u128> for Pu128 {
+    #[inline]
+    fn eq(&self, other: &u128) -> bool {
+        ({ self.0 }) == *other
+    }
+}
+
+impl PartialOrd<u128> for Pu128 {
+    #[inline]
+    fn partial_cmp(&self, other: &u128) -> Option<Ordering> {
+        { self.0 }.partial_cmp(other)
+    }
+}
+
+impl fmt::Display for Pu128 {
+    #[inline]
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        { self.0 }.fmt(f)
+    }
+}
+
+impl fmt::UpperHex for Pu128 {
+    #[inline]
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        { self.0 }.fmt(f)
+    }
+}
+
+impl<CTX> HashStable<CTX> for Pu128 {
+    #[inline]
+    fn hash_stable(&self, ctx: &mut CTX, hasher: &mut StableHasher) {
+        { self.0 }.hash_stable(ctx, hasher)
+    }
+}
+
+impl<S: Encoder> Encodable<S> for Pu128 {
+    #[inline]
+    fn encode(&self, s: &mut S) {
+        { self.0 }.encode(s);
+    }
+}
+
+impl<D: Decoder> Decodable<D> for Pu128 {
+    #[inline]
+    fn decode(d: &mut D) -> Self {
+        Self(u128::decode(d))
+    }
+}
diff --git a/compiler/rustc_middle/src/mir/syntax.rs b/compiler/rustc_middle/src/mir/syntax.rs
@@ -13,6 +13,7 @@ use crate::ty::{self, List, Ty};
 use crate::ty::{Region, UserTypeAnnotationIndex};
 
 use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece};
+use rustc_data_structures::packed::Pu128;
 use rustc_hir::def_id::DefId;
 use rustc_hir::{self, CoroutineKind};
 use rustc_index::IndexVec;
@@ -829,7 +830,7 @@ impl TerminatorKind<'_> {
 pub struct SwitchTargets {
     /// Possible values. The locations to branch to in each case
     /// are found in the corresponding indices from the `targets` vector.
-    pub(super) values: SmallVec<[u128; 1]>,
+    pub(super) values: SmallVec<[Pu128; 1]>,
 
     /// Possible branch sites. The last element of this vector is used
     /// for the otherwise branch, so targets.len() == values.len() + 1

diff --git a/compiler/rustc_middle/src/mir/terminator.rs b/compiler/rustc_middle/src/mir/terminator.rs
@@ -3,6 +3,7 @@ use rustc_hir::LangItem;
 use smallvec::SmallVec;
 
 use super::TerminatorKind;
+use rustc_data_structures::packed::Pu128;
 use rustc_macros::HashStable;
 use std::slice;
 
@@ -14,15 +15,16 @@ impl SwitchTargets {
     /// The iterator may be empty, in which case the `SwitchInt` instruction is equivalent to
     /// `goto otherwise;`.
     pub fn new(targets: impl Iterator<Item = (u128, BasicBlock)>, otherwise: BasicBlock) -> Self {
-        let (values, mut targets): (SmallVec<_>, SmallVec<_>) = targets.unzip();
+        let (values, mut targets): (SmallVec<_>, SmallVec<_>) =
+            targets.map(|(v, t)| (Pu128(v), t)).unzip();
         targets.push(otherwise);
         Self { values, targets }
     }
 
     /// Builds a switch targets definition that jumps to `then` if the tested value equals `value`,
     /// and to `else_` if not.
     pub fn static_if(value: u128, then: BasicBlock, else_: BasicBlock) -> Self {
-        Self { values: smallvec![value], targets: smallvec![then, else_] }
+        Self { values: smallvec![Pu128(value)], targets: smallvec![then, else_] }
     }
 
     /// Inverse of `SwitchTargets::static_if`.
@@ -31,7 +33,7 @@ impl SwitchTargets {
         if let &[value] = &self.values[..]
             && let &[then, else_] = &self.targets[..]
         {
-            Some((value, then, else_))
+            Some((value.get(), then, else_))
         } else {
             None
         }
@@ -75,15 +77,15 @@ impl SwitchTargets {
 }
 
 pub struct SwitchTargetsIter<'a> {
-    inner: iter::Zip<slice::Iter<'a, u128>, slice::Iter<'a, BasicBlock>>,
+    inner: iter::Zip<slice::Iter<'a, Pu128>, slice::Iter<'a, BasicBlock>>,
 }
 
 impl<'a> Iterator for SwitchTargetsIter<'a> {
     type Item = (u128, BasicBlock);
 
     #[inline]
     fn next(&mut self) -> Option<Self::Item> {
-        self.inner.next().map(|(val, bb)| (*val, *bb))
+        self.inner.next().map(|(val, bb)| (val.get(), *bb))
     }
 
     #[inline]

diff --git a/compiler/rustc_target/src/spec/targets/i386_apple_ios.rs b/compiler/rustc_target/src/spec/targets/i386_apple_ios.rs
@@ -11,7 +11,7 @@ pub fn target() -> Target {
         llvm_target: ios_sim_llvm_target(arch).into(),
         pointer_width: 32,
         data_layout: "e-m:o-p:32:32-p270:32:32-p271:32:32-p272:64:64-\
-            f64:32:64-f80:128-n8:16:32-S128"
+            i128:128-f64:32:64-f80:128-n8:16:32-S128"
             .into(),
         arch: arch.target_arch(),
         options: TargetOptions { max_atomic_width: Some(64), ..opts("ios", arch) },

diff --git a/compiler/rustc_target/src/spec/targets/i686_apple_darwin.rs b/compiler/rustc_target/src/spec/targets/i686_apple_darwin.rs
@@ -18,7 +18,7 @@ pub fn target() -> Target {
         llvm_target: macos_llvm_target(Arch::I686).into(),
         pointer_width: 32,
         data_layout: "e-m:o-p:32:32-p270:32:32-p271:32:32-p272:64:64-\
-            f64:32:64-f80:128-n8:16:32-S128"
+            i128:128-f64:32:64-f80:128-n8:16:32-S128"
             .into(),
         arch: arch.target_arch(),
         options: TargetOptions { mcount: "\u{1}mcount".into(), ..base },

diff --git a/compiler/rustc_target/src/spec/targets/i686_linux_android.rs b/compiler/rustc_target/src/spec/targets/i686_linux_android.rs
@@ -17,7 +17,7 @@ pub fn target() -> Target {
         llvm_target: "i686-linux-android".into(),
         pointer_width: 32,
         data_layout: "e-m:e-p:32:32-p270:32:32-p271:32:32-p272:64:64-\
-            f64:32:64-f80:32-n8:16:32-S128"
+            i128:128-f64:32:64-f80:32-n8:16:32-S128"
             .into(),
         arch: "x86".into(),
         options: TargetOptions { supported_sanitizers: SanitizerSet::ADDRESS, ..base },

diff --git a/compiler/rustc_target/src/spec/targets/i686_pc_windows_gnu.rs b/compiler/rustc_target/src/spec/targets/i686_pc_windows_gnu.rs
@@ -19,7 +19,7 @@ pub fn target() -> Target {
         llvm_target: "i686-pc-windows-gnu".into(),
         pointer_width: 32,
         data_layout: "e-m:x-p:32:32-p270:32:32-p271:32:32-p272:64:64-\
-            i64:64-f80:32-n8:16:32-a:0:32-S32"
+            i64:64-i128:128-f80:32-n8:16:32-a:0:32-S32"
             .into(),
         arch: "x86".into(),
         options: base,

diff --git a/compiler/rustc_target/src/spec/targets/i686_pc_windows_gnullvm.rs b/compiler/rustc_target/src/spec/targets/i686_pc_windows_gnullvm.rs
@@ -18,7 +18,7 @@ pub fn target() -> Target {
         llvm_target: "i686-pc-windows-gnu".into(),
         pointer_width: 32,
         data_layout: "e-m:x-p:32:32-p270:32:32-p271:32:32-p272:64:64-\
-            i64:64-f80:32-n8:16:32-a:0:32-S32"
+            i64:64-i128:128-f80:32-n8:16:32-a:0:32-S32"
             .into(),
         arch: "x86".into(),
         options: base,

diff --git a/compiler/rustc_target/src/spec/targets/i686_pc_windows_msvc.rs b/compiler/rustc_target/src/spec/targets/i686_pc_windows_msvc.rs
@@ -25,7 +25,7 @@ pub fn target() -> Target {
         llvm_target: "i686-pc-windows-msvc".into(),
         pointer_width: 32,
         data_layout: "e-m:x-p:32:32-p270:32:32-p271:32:32-p272:64:64-\
-            i64:64-f80:128-n8:16:32-a:0:32-S32"
+            i64:64-i128:128-f80:128-n8:16:32-a:0:32-S32"
             .into(),
         arch: "x86".into(),
         options: base,

diff --git a/compiler/rustc_target/src/spec/targets/i686_unknown_freebsd.rs b/compiler/rustc_target/src/spec/targets/i686_unknown_freebsd.rs
@@ -11,7 +11,7 @@ pub fn target() -> Target {
         llvm_target: "i686-unknown-freebsd".into(),
         pointer_width: 32,
         data_layout: "e-m:e-p:32:32-p270:32:32-p271:32:32-p272:64:64-\
-            f64:32:64-f80:32-n8:16:32-S128"
+            i128:128-f64:32:64-f80:32-n8:16:32-S128"
             .into(),
         arch: "x86".into(),
         options: base,

diff --git a/compiler/rustc_target/src/spec/targets/i686_unknown_haiku.rs b/compiler/rustc_target/src/spec/targets/i686_unknown_haiku.rs
@@ -11,7 +11,7 @@ pub fn target() -> Target {
         llvm_target: "i686-unknown-haiku".into(),
         pointer_width: 32,
         data_layout: "e-m:e-p:32:32-p270:32:32-p271:32:32-p272:64:64-\
-            f64:32:64-f80:32-n8:16:32-S128"
+            i128:128-f64:32:64-f80:32-n8:16:32-S128"
             .into(),
         arch: "x86".into(),
         options: base,

diff --git a/compiler/rustc_target/src/spec/targets/i686_unknown_hurd_gnu.rs b/compiler/rustc_target/src/spec/targets/i686_unknown_hurd_gnu.rs
@@ -11,7 +11,7 @@ pub fn target() -> Target {
         llvm_target: "i686-unknown-hurd-gnu".into(),
         pointer_width: 32,
         data_layout: "e-m:e-p:32:32-p270:32:32-p271:32:32-p272:64:64-\
-            f64:32:64-f80:32-n8:16:32-S128"
+            i128:128-f64:32:64-f80:32-n8:16:32-S128"
             .into(),
         arch: "x86".into(),
         options: base,

diff --git a/compiler/rustc_target/src/spec/targets/i686_unknown_linux_gnu.rs b/compiler/rustc_target/src/spec/targets/i686_unknown_linux_gnu.rs
@@ -12,7 +12,7 @@ pub fn target() -> Target {
         llvm_target: "i686-unknown-linux-gnu".into(),
         pointer_width: 32,
         data_layout: "e-m:e-p:32:32-p270:32:32-p271:32:32-p272:64:64-\
-            f64:32:64-f80:32-n8:16:32-S128"
+            i128:128-f64:32:64-f80:32-n8:16:32-S128"
             .into(),
         arch: "x86".into(),
         options: base,

diff --git a/compiler/rustc_target/src/spec/targets/i686_unknown_linux_musl.rs b/compiler/rustc_target/src/spec/targets/i686_unknown_linux_musl.rs
@@ -25,7 +25,7 @@ pub fn target() -> Target {
         llvm_target: "i686-unknown-linux-musl".into(),
         pointer_width: 32,
         data_layout: "e-m:e-p:32:32-p270:32:32-p271:32:32-p272:64:64-\
-            f64:32:64-f80:32-n8:16:32-S128"
+            i128:128-f64:32:64-f80:32-n8:16:32-S128"
             .into(),
         arch: "x86".into(),
         options: base,

diff --git a/compiler/rustc_target/src/spec/targets/i686_unknown_netbsd.rs b/compiler/rustc_target/src/spec/targets/i686_unknown_netbsd.rs
@@ -11,7 +11,7 @@ pub fn target() -> Target {
         llvm_target: "i686-unknown-netbsdelf".into(),
         pointer_width: 32,
         data_layout: "e-m:e-p:32:32-p270:32:32-p271:32:32-p272:64:64-\
-            f64:32:64-f80:32-n8:16:32-S128"
+            i128:128-f64:32:64-f80:32-n8:16:32-S128"
             .into(),
         arch: "x86".into(),
         options: TargetOptions { mcount: "__mcount".into(), ..base },

diff --git a/compiler/rustc_target/src/spec/targets/i686_unknown_openbsd.rs b/compiler/rustc_target/src/spec/targets/i686_unknown_openbsd.rs
@@ -11,7 +11,7 @@ pub fn target() -> Target {
         llvm_target: "i686-unknown-openbsd".into(),
         pointer_width: 32,
         data_layout: "e-m:e-p:32:32-p270:32:32-p271:32:32-p272:64:64-\
-            f64:32:64-f80:32-n8:16:32-S128"
+            i128:128-f64:32:64-f80:32-n8:16:32-S128"
             .into(),
         arch: "x86".into(),
         options: base,

diff --git a/compiler/rustc_target/src/spec/targets/i686_uwp_windows_gnu.rs b/compiler/rustc_target/src/spec/targets/i686_uwp_windows_gnu.rs
@@ -18,7 +18,7 @@ pub fn target() -> Target {
         llvm_target: "i686-pc-windows-gnu".into(),
         pointer_width: 32,
         data_layout: "e-m:x-p:32:32-p270:32:32-p271:32:32-p272:64:64-\
-            i64:64-f80:32-n8:16:32-a:0:32-S32"
+            i64:64-i128:128-f80:32-n8:16:32-a:0:32-S32"
             .into(),
         arch: "x86".into(),
         options: base,

diff --git a/compiler/rustc_target/src/spec/targets/i686_uwp_windows_msvc.rs b/compiler/rustc_target/src/spec/targets/i686_uwp_windows_msvc.rs
@@ -9,7 +9,7 @@ pub fn target() -> Target {
         llvm_target: "i686-pc-windows-msvc".into(),
         pointer_width: 32,
         data_layout: "e-m:x-p:32:32-p270:32:32-p271:32:32-p272:64:64-\
-            i64:64-f80:128-n8:16:32-a:0:32-S32"
+            i64:64-i128:128-f80:128-n8:16:32-a:0:32-S32"
             .into(),
         arch: "x86".into(),
         options: base,

diff --git a/compiler/rustc_target/src/spec/targets/i686_wrs_vxworks.rs b/compiler/rustc_target/src/spec/targets/i686_wrs_vxworks.rs
@@ -11,7 +11,7 @@ pub fn target() -> Target {
         llvm_target: "i686-unknown-linux-gnu".into(),
         pointer_width: 32,
         data_layout: "e-m:e-p:32:32-p270:32:32-p271:32:32-p272:64:64-\
-            f64:32:64-f80:32-n8:16:32-S128"
+            i128:128-f64:32:64-f80:32-n8:16:32-S128"
             .into(),
         arch: "x86".into(),
         options: base,

diff --git a/compiler/rustc_target/src/spec/targets/x86_64_apple_darwin.rs b/compiler/rustc_target/src/spec/targets/x86_64_apple_darwin.rs
@@ -17,8 +17,8 @@ pub fn target() -> Target {
         // correctly, we do too.
         llvm_target: macos_llvm_target(arch).into(),
         pointer_width: 64,
-        data_layout: "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
-            .into(),
+        data_layout:
+            "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128".into(),
         arch: arch.target_arch(),
         options: TargetOptions { mcount: "\u{1}mcount".into(), ..base },
     }

diff --git a/compiler/rustc_target/src/spec/targets/x86_64_apple_ios.rs b/compiler/rustc_target/src/spec/targets/x86_64_apple_ios.rs
@@ -9,8 +9,8 @@ pub fn target() -> Target {
     Target {
         llvm_target: ios_sim_llvm_target(arch).into(),
         pointer_width: 64,
-        data_layout: "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
-            .into(),
+        data_layout:
+            "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128".into(),
         arch: arch.target_arch(),
         options: TargetOptions { max_atomic_width: Some(128), ..base },
     }

diff --git a/compiler/rustc_target/src/spec/targets/x86_64_apple_ios_macabi.rs b/compiler/rustc_target/src/spec/targets/x86_64_apple_ios_macabi.rs
@@ -12,8 +12,8 @@ pub fn target() -> Target {
     Target {
         llvm_target: llvm_target.into(),
         pointer_width: 64,
-        data_layout: "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
-            .into(),
+        data_layout:
+            "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128".into(),
         arch: arch.target_arch(),
         options: TargetOptions { max_atomic_width: Some(128), ..base },
     }

diff --git a/compiler/rustc_target/src/spec/targets/x86_64_apple_tvos.rs b/compiler/rustc_target/src/spec/targets/x86_64_apple_tvos.rs
@@ -6,8 +6,8 @@ pub fn target() -> Target {
     Target {
         llvm_target: tvos_sim_llvm_target(arch).into(),
         pointer_width: 64,
-        data_layout: "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
-            .into(),
+        data_layout:
+            "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128".into(),
         arch: arch.target_arch(),
         options: TargetOptions { max_atomic_width: Some(128), ..opts("tvos", arch) },
     }

diff --git a/compiler/rustc_target/src/spec/targets/x86_64_apple_watchos_sim.rs b/compiler/rustc_target/src/spec/targets/x86_64_apple_watchos_sim.rs
@@ -6,8 +6,8 @@ pub fn target() -> Target {
     Target {
         llvm_target: watchos_sim_llvm_target(arch).into(),
         pointer_width: 64,
-        data_layout: "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
-            .into(),
+        data_layout:
+            "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128".into(),
         arch: arch.target_arch(),
         options: TargetOptions { max_atomic_width: Some(128), ..opts("watchos", arch) },
     }

diff --git a/compiler/rustc_target/src/spec/targets/x86_64_fortanix_unknown_sgx.rs b/compiler/rustc_target/src/spec/targets/x86_64_fortanix_unknown_sgx.rs
@@ -75,8 +75,8 @@ pub fn target() -> Target {
     Target {
         llvm_target: "x86_64-elf".into(),
         pointer_width: 64,
-        data_layout: "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
-            .into(),
+        data_layout:
+            "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128".into(),
         arch: "x86_64".into(),
         options: opts,
     }