Add bit_{set,clear,toggle} to Atomic{I,U}* and AtomicPtr

taiki-e · Feb 27, 2023 · 6d42701 · 6d42701
1 parent e306885
commit 6d42701
Show file tree

Hide file tree

Showing 12 changed files with 498 additions and 6 deletions.
diff --git a/.github/.cspell/project-dictionary.txt b/.github/.cspell/project-dictionary.txt
@@ -90,6 +90,7 @@ rcpc
 reentrancy
 sbcs
 seqlock
+setb
 sete
 sifive
 SIGILL

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -283,7 +283,7 @@ jobs:
           MIRIFLAGS: -Zmiri-strict-provenance -Zmiri-symbolic-alignment-check -Zmiri-retag-fields -Zmiri-disable-isolation
           RUSTDOCFLAGS: ${{ env.RUSTDOCFLAGS }} -Z randomize-layout
           RUSTFLAGS: ${{ env.RUSTFLAGS }} -Z randomize-layout
-          QUICKCHECK_TESTS: 20
+          QUICKCHECK_TESTS: 10
 
   san:
     strategy:

diff --git a/bench/benches/imp/spinlock_fallback.rs b/bench/benches/imp/spinlock_fallback.rs
@@ -85,6 +85,7 @@ macro_rules! atomic_int {
         unsafe impl Sync for $atomic_type {}
 
         no_fetch_ops_impl!($atomic_type, $int_type);
+        bit_opts_fetch_impl!($atomic_type, $int_type);
         impl $atomic_type {
             #[inline]
             pub(crate) const fn new(v: $int_type) -> Self {

diff --git a/src/imp/atomic128/intrinsics.rs b/src/imp/atomic128/intrinsics.rs
@@ -421,6 +421,7 @@ macro_rules! atomic128 {
         unsafe impl Sync for $atomic_type {}
 
         no_fetch_ops_impl!($atomic_type, $int_type);
+        bit_opts_fetch_impl!($atomic_type, $int_type);
         impl $atomic_type {
             #[inline]
             pub(crate) const fn new(v: $int_type) -> Self {

diff --git a/src/imp/atomic128/macros.rs b/src/imp/atomic128/macros.rs
@@ -11,6 +11,7 @@ macro_rules! atomic128 {
         unsafe impl Sync for $atomic_type {}
 
         no_fetch_ops_impl!($atomic_type, $int_type);
+        bit_opts_fetch_impl!($atomic_type, $int_type);
         impl $atomic_type {
             #[inline]
             pub(crate) const fn new(v: $int_type) -> Self {
@@ -258,6 +259,7 @@ macro_rules! atomic128 {
         unsafe impl Sync for $atomic_type {}
 
         no_fetch_ops_impl!($atomic_type, $int_type);
+        bit_opts_fetch_impl!($atomic_type, $int_type);
         impl $atomic_type {
             #[inline]
             pub(crate) const fn new(v: $int_type) -> Self {

diff --git a/src/imp/core_atomic.rs b/src/imp/core_atomic.rs
@@ -198,6 +198,17 @@ macro_rules! atomic_int {
         )]
         #[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg(target_has_atomic = "ptr"))]
         no_fetch_ops_impl!($atomic_type, $int_type);
+        #[cfg(not(all(
+            not(any(miri, portable_atomic_sanitize_thread)),
+            any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
+            any(target_arch = "x86", target_arch = "x86_64"),
+        )))]
+        #[cfg_attr(
+            portable_atomic_no_cfg_target_has_atomic,
+            cfg(not(portable_atomic_no_atomic_cas))
+        )]
+        #[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg(target_has_atomic = "ptr"))]
+        bit_opts_fetch_impl!($atomic_type, $int_type);
         impl $atomic_type {
             #[inline]
             pub(crate) const fn new(v: $int_type) -> Self {

diff --git a/src/imp/fallback/imp.rs b/src/imp/fallback/imp.rs
@@ -122,6 +122,8 @@ macro_rules! atomic {
 
         #[cfg(any(test, not(portable_atomic_unstable_cmpxchg16b_target_feature)))]
         no_fetch_ops_impl!($atomic_type, $int_type);
+        #[cfg(any(test, not(portable_atomic_unstable_cmpxchg16b_target_feature)))]
+        bit_opts_fetch_impl!($atomic_type, $int_type);
         impl $atomic_type {
             #[cfg(any(test, not(portable_atomic_unstable_cmpxchg16b_target_feature)))]
             #[inline]

diff --git a/src/imp/interrupt/mod.rs b/src/imp/interrupt/mod.rs
@@ -515,6 +515,7 @@ macro_rules! atomic_int {
 
         #[cfg(not(all(target_arch = "msp430", not(feature = "critical-section"))))]
         no_fetch_ops_impl!($atomic_type, $int_type);
+        bit_opts_fetch_impl!($atomic_type, $int_type);
         #[cfg(not(all(target_arch = "msp430", not(feature = "critical-section"))))]
         impl $atomic_type {
             #[inline]
@@ -578,6 +579,7 @@ macro_rules! atomic_int {
         atomic_int!(base, $atomic_type, $int_type, $align);
         atomic_int!($kind, cas, $atomic_type, $int_type);
         no_fetch_ops_impl!($atomic_type, $int_type);
+        bit_opts_fetch_impl!($atomic_type, $int_type);
         impl $atomic_type {
             #[inline]
             #[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)]

diff --git a/src/imp/x86.rs b/src/imp/x86.rs
@@ -7,7 +7,10 @@
 use core::arch::asm;
 use core::sync::atomic::Ordering;
 
-use super::core_atomic as imp;
+use super::core_atomic::{
+    AtomicI16, AtomicI32, AtomicI64, AtomicI8, AtomicIsize, AtomicU16, AtomicU32, AtomicU64,
+    AtomicU8, AtomicUsize,
+};
 
 #[cfg(target_pointer_width = "32")]
 macro_rules! ptr_modifier {
@@ -24,7 +27,7 @@ macro_rules! ptr_modifier {
 
 macro_rules! atomic_int {
     (uint, $atomic_type:ident, $int_type:ident, $ptr_size:tt) => {
-        impl imp::$atomic_type {
+        impl $atomic_type {
             #[inline]
             pub(crate) fn not(&self, _order: Ordering) {
                 let dst = self.as_ptr();
@@ -45,7 +48,7 @@ macro_rules! atomic_int {
     };
     (int, $atomic_type:ident, $int_type:ident, $ptr_size:tt) => {
         atomic_int!(uint, $atomic_type, $int_type, $ptr_size);
-        impl imp::$atomic_type {
+        impl $atomic_type {
             #[inline]
             pub(crate) fn neg(&self, _order: Ordering) {
                 let dst = self.as_ptr();
@@ -87,7 +90,7 @@ atomic_int!(int, AtomicIsize, isize, "qword");
 atomic_int!(uint, AtomicUsize, usize, "qword");
 
 #[cfg(target_arch = "x86")]
-impl imp::AtomicI64 {
+impl AtomicI64 {
     #[inline]
     pub(crate) fn not(&self, order: Ordering) {
         self.fetch_not(order);
@@ -98,9 +101,115 @@ impl imp::AtomicI64 {
     }
 }
 #[cfg(target_arch = "x86")]
-impl imp::AtomicU64 {
+impl AtomicU64 {
     #[inline]
     pub(crate) fn not(&self, order: Ordering) {
         self.fetch_not(order);
     }
 }
+
+// Note: As of LLVM 15, LLVM only supports generating `lock bt{s,r,c}` for immediate bit offsets.
+// https://godbolt.org/z/aM5rGPYn8
+macro_rules! atomic_bit_opts {
+    ($atomic_type:ident, $int_type:ident, $val_modifier:tt, $ptr_size:tt) => {
+        impl $atomic_type {
+            // `<integer>::BITS` is not available on old nightly.
+            const BITS: u32 = (core::mem::size_of::<$int_type>() * 8) as u32;
+            #[inline]
+            pub(crate) fn bit_set(&self, bit: u32, _order: Ordering) -> bool {
+                let dst = self.as_ptr();
+                // SAFETY: any data races are prevented by atomic intrinsics and the raw
+                // pointer passed in is valid because we got it from a reference.
+                // the masking by the bit size of the type ensures that we do not shift
+                // out of bounds.
+                //
+                // https://www.felixcloutier.com/x86/bts
+                unsafe {
+                    let out: u8;
+                    // atomic RMW is always SeqCst.
+                    asm!(
+                        concat!("lock bts ", $ptr_size, " ptr [{dst", ptr_modifier!(), "}], {bit", $val_modifier, "}"),
+                        "setb {out}",
+                        dst = in(reg) dst,
+                        bit = in(reg) (bit & (Self::BITS - 1)) as $int_type,
+                        out = out(reg_byte) out,
+                        // Do not use `preserves_flags` because BTS modifies the CF flag.
+                        options(nostack),
+                    );
+                    out != 0
+                }
+            }
+            #[inline]
+            pub(crate) fn bit_clear(&self, bit: u32, _order: Ordering) -> bool {
+                let dst = self.as_ptr();
+                // SAFETY: any data races are prevented by atomic intrinsics and the raw
+                // pointer passed in is valid because we got it from a reference.
+                // the masking by the bit size of the type ensures that we do not shift
+                // out of bounds.
+                //
+                // https://www.felixcloutier.com/x86/btr
+                unsafe {
+                    let out: u8;
+                    // atomic RMW is always SeqCst.
+                    asm!(
+                        concat!("lock btr ", $ptr_size, " ptr [{dst", ptr_modifier!(), "}], {bit", $val_modifier, "}"),
+                        "setb {out}",
+                        dst = in(reg) dst,
+                        bit = in(reg) (bit & (Self::BITS - 1)) as $int_type,
+                        out = out(reg_byte) out,
+                        // Do not use `preserves_flags` because BTR modifies the CF flag.
+                        options(nostack),
+                    );
+                    out != 0
+                }
+            }
+            #[inline]
+            pub(crate) fn bit_toggle(&self, bit: u32, _order: Ordering) -> bool {
+                let dst = self.as_ptr();
+                // SAFETY: any data races are prevented by atomic intrinsics and the raw
+                // pointer passed in is valid because we got it from a reference.
+                // the masking by the bit size of the type ensures that we do not shift
+                // out of bounds.
+                //
+                // https://www.felixcloutier.com/x86/btc
+                unsafe {
+                    let out: u8;
+                    // atomic RMW is always SeqCst.
+                    asm!(
+                        concat!("lock btc ", $ptr_size, " ptr [{dst", ptr_modifier!(), "}], {bit", $val_modifier, "}"),
+                        "setb {out}",
+                        dst = in(reg) dst,
+                        bit = in(reg) (bit & (Self::BITS - 1)) as $int_type,
+                        out = out(reg_byte) out,
+                        // Do not use `preserves_flags` because BTC modifies the CF flag.
+                        options(nostack),
+                    );
+                    out != 0
+                }
+            }
+        }
+    };
+}
+
+bit_opts_fetch_impl!(AtomicI8, i8);
+bit_opts_fetch_impl!(AtomicU8, u8);
+atomic_bit_opts!(AtomicI16, i16, ":x", "word");
+atomic_bit_opts!(AtomicU16, u16, ":x", "word");
+atomic_bit_opts!(AtomicI32, i32, ":e", "dword");
+atomic_bit_opts!(AtomicU32, u32, ":e", "dword");
+#[cfg(target_arch = "x86_64")]
+atomic_bit_opts!(AtomicI64, i64, "", "qword");
+#[cfg(target_arch = "x86_64")]
+atomic_bit_opts!(AtomicU64, u64, "", "qword");
+#[cfg(target_arch = "x86")]
+bit_opts_fetch_impl!(AtomicI64, i64);
+#[cfg(target_arch = "x86")]
+bit_opts_fetch_impl!(AtomicU64, u64);
+#[cfg(target_pointer_width = "32")]
+atomic_bit_opts!(AtomicIsize, isize, ":e", "dword");
+#[cfg(target_pointer_width = "32")]
+atomic_bit_opts!(AtomicUsize, usize, ":e", "dword");
+#[cfg(target_pointer_width = "64")]
+atomic_bit_opts!(AtomicIsize, isize, "", "qword");
+#[cfg(target_pointer_width = "64")]
+atomic_bit_opts!(AtomicUsize, usize, "", "qword");