From efacc89c210d7a34ef5e879821112189da5d1901 Mon Sep 17 00:00:00 2001 From: Taiki Endo Date: Sat, 1 Apr 2023 07:09:37 +0900 Subject: [PATCH] arm_linux: Do not use CAS loop for fallback This also removes the requirement to rely on the fallback implementation being seqlock. --- src/imp/arm_linux.rs | 303 ++++++++++++++++++---------- src/imp/atomic128/x86_64.rs | 34 +--- src/imp/fallback/outline_atomics.rs | 158 +++++++++++++++ src/utils.rs | 19 +- 4 files changed, 377 insertions(+), 137 deletions(-) create mode 100644 src/imp/fallback/outline_atomics.rs diff --git a/src/imp/arm_linux.rs b/src/imp/arm_linux.rs index 2ba735f8..ad06f2d8 100644 --- a/src/imp/arm_linux.rs +++ b/src/imp/arm_linux.rs @@ -11,6 +11,9 @@ // be possible to omit the dynamic kernel version check if the std feature is enabled on Rust 1.64+. // https://blog.rust-lang.org/2022/08/01/Increasing-glibc-kernel-requirements.html +#[path = "fallback/outline_atomics.rs"] +mod fallback; + #[cfg(not(portable_atomic_no_asm))] use core::arch::asm; use core::{mem, sync::atomic::Ordering}; @@ -34,43 +37,34 @@ struct Pair { // https://www.kernel.org/doc/Documentation/arm/kernel_user_helpers.txt const KUSER_HELPER_VERSION: usize = 0xFFFF0FFC; -// kuser_helper_version >= 5 (kernel version 3.1+) +// __kuser_helper_version >= 5 (kernel version 3.1+) const KUSER_CMPXCHG64: usize = 0xFFFF0F60; #[inline] -fn kuser_helper_version() -> i32 { - // SAFETY: KUSER_HELPER_VERSION is always available on ARM Linux/Android. - unsafe { (KUSER_HELPER_VERSION as *const i32).read() } +fn __kuser_helper_version() -> i32 { + use core::sync::atomic::AtomicI32; + + static CACHE: AtomicI32 = AtomicI32::new(0); + let mut v = CACHE.load(Ordering::Relaxed); + if v != 0 { + return v; + } + // SAFETY: core assumes that at least __kuser_cmpxchg (__kuser_helper_version >= 2) is available + // on this platform. __kuser_helper_version is always available on such a platform. + v = unsafe { (KUSER_HELPER_VERSION as *const i32).read() }; + CACHE.store(v, Ordering::Relaxed); + v } #[inline] -unsafe fn kuser_cmpxchg64(old_val: *const u64, new_val: *const u64, ptr: *mut u64) -> bool { - unsafe fn __kuser_cmpxchg64(old_val: *const u64, new_val: *const u64, ptr: *mut u64) -> bool { - // SAFETY: the caller must uphold the safety contract. - unsafe { - let f: extern "C" fn(*const u64, *const u64, *mut u64) -> u32 = - mem::transmute(KUSER_CMPXCHG64 as *const ()); - f(old_val, new_val, ptr) == 0 - } - } - #[cold] - unsafe fn _fallback(old_val: *const u64, new_val: *const u64, ptr: *mut u64) -> bool { - // SAFETY: the caller must uphold the safety contract. - unsafe { - // Use SeqCst because __kuser_cmpxchg64 is SeqCst. - // https://github.com/torvalds/linux/blob/v6.1/arch/arm/kernel/entry-armv.S#L918-L925 - (*(ptr as *const super::fallback::AtomicU64)) - .compare_exchange(*old_val, *new_val, Ordering::SeqCst, Ordering::SeqCst) - .is_ok() - } - } - // SAFETY: we only calls __kuser_cmpxchg64 if it is available. +fn has_kuser_cmpxchg64() -> bool { + __kuser_helper_version() >= 5 +} +#[inline] +unsafe fn __kuser_cmpxchg64(old_val: *const u64, new_val: *const u64, ptr: *mut u64) -> bool { + // SAFETY: the caller must uphold the safety contract. unsafe { - ifunc!(unsafe fn(old_val: *const u64, new_val: *const u64, ptr: *mut u64) -> bool { - if kuser_helper_version() >= 5 { - __kuser_cmpxchg64 - } else { - _fallback - } - }) + let f: extern "C" fn(*const u64, *const u64, *mut u64) -> u32 = + mem::transmute(KUSER_CMPXCHG64 as *const ()); + f(old_val, new_val, ptr) == 0 } } @@ -93,11 +87,12 @@ unsafe fn byte_wise_atomic_load(src: *const u64) -> u64 { } #[inline(always)] -unsafe fn atomic_update(dst: *mut u64, mut f: F) -> u64 +unsafe fn atomic_update_kuser_cmpxchg64(dst: *mut u64, mut f: F) -> u64 where F: FnMut(u64) -> u64, { debug_assert!(dst as usize % 8 == 0); + debug_assert!(has_kuser_cmpxchg64()); // SAFETY: the caller must uphold the safety contract. unsafe { @@ -105,53 +100,168 @@ where // This is not single-copy atomic reads, but this is ok because subsequent // CAS will check for consistency. // - // byte_wise_atomic_load works the same way as seqlock's byte-wise atomic memcpy, - // so it works well even when atomic_compare_exchange_weak calls global lock-based fallback. - // // Note that the C++20 memory model does not allow mixed-sized atomic access, // so we must use inline assembly to implement byte_wise_atomic_load. // (i.e., byte-wise atomic based on the standard library's atomic types - // cannot be used here). Since fallback's byte-wise atomic memcpy is per - // 32-bit on ARM, it's okay to use it together with this. + // cannot be used here). let old = byte_wise_atomic_load(dst); let next = f(old); - if kuser_cmpxchg64(&old, &next, dst) { + if __kuser_cmpxchg64(&old, &next, dst) { return old; } } } } -#[inline] -unsafe fn atomic_load(src: *mut u64) -> u64 { - // SAFETY: the caller must uphold the safety contract. - unsafe { atomic_update(src, |old| old) } +macro_rules! atomic_with_ifunc { + ( + unsafe fn $name:ident($($arg:tt)*) $(-> $ret_ty:ty)? { $($kuser_cmpxchg64_fn_body:tt)* } + fallback = $seqcst_fallback_fn:ident + ) => { + #[inline] + unsafe fn $name($($arg)*) $(-> $ret_ty)? { + unsafe fn kuser_cmpxchg64_fn($($arg)*) $(-> $ret_ty)? { + $($kuser_cmpxchg64_fn_body)* + } + // SAFETY: the caller must uphold the safety contract. + // we only calls __kuser_cmpxchg64 if it is available. + unsafe { + ifunc!(unsafe fn($($arg)*) $(-> $ret_ty)? { + if has_kuser_cmpxchg64() { + kuser_cmpxchg64_fn + } else { + // Use SeqCst because __kuser_cmpxchg64 is SeqCst. + // https://github.com/torvalds/linux/blob/v6.1/arch/arm/kernel/entry-armv.S#L918-L925 + fallback::$seqcst_fallback_fn + } + }) + } + } + }; } -#[inline] -unsafe fn atomic_store(dst: *mut u64, val: u64) { - // SAFETY: the caller must uphold the safety contract. - unsafe { - atomic_swap(dst, val); + +atomic_with_ifunc! { + unsafe fn atomic_load(src: *mut u64) -> u64 { + // SAFETY: the caller must uphold the safety contract. + unsafe { atomic_update_kuser_cmpxchg64(src, |old| old) } } + fallback = atomic_load_seqcst } -#[inline] -unsafe fn atomic_swap(dst: *mut u64, val: u64) -> u64 { - // SAFETY: the caller must uphold the safety contract. - unsafe { atomic_update(dst, |_| val) } +atomic_with_ifunc! { + unsafe fn atomic_store(dst: *mut u64, val: u64) { + // SAFETY: the caller must uphold the safety contract. + unsafe { atomic_update_kuser_cmpxchg64(dst, |_| val); } + } + fallback = atomic_store_seqcst } -#[inline] -unsafe fn atomic_compare_exchange(dst: *mut u64, old: u64, new: u64) -> Result { - // SAFETY: the caller must uphold the safety contract. - let res = unsafe { atomic_update(dst, |v| if v == old { new } else { v }) }; - if res == old { - Ok(res) - } else { - Err(res) +atomic_with_ifunc! { + unsafe fn atomic_swap(dst: *mut u64, val: u64) -> u64 { + // SAFETY: the caller must uphold the safety contract. + unsafe { atomic_update_kuser_cmpxchg64(dst, |_| val) } + } + fallback = atomic_swap_seqcst +} +atomic_with_ifunc! { + unsafe fn atomic_compare_exchange(dst: *mut u64, old: u64, new: u64) -> (u64, bool) { + // SAFETY: the caller must uphold the safety contract. + let res = unsafe { atomic_update_kuser_cmpxchg64(dst, |v| if v == old { new } else { v }) }; + (res, res == old) + } + fallback = atomic_compare_exchange_seqcst +} +atomic_with_ifunc! { + unsafe fn atomic_add(dst: *mut u64, val: u64) -> u64 { + // SAFETY: the caller must uphold the safety contract. + unsafe { atomic_update_kuser_cmpxchg64(dst, |x| x.wrapping_add(val)) } + } + fallback = atomic_add_seqcst +} +atomic_with_ifunc! { + unsafe fn atomic_sub(dst: *mut u64, val: u64) -> u64 { + // SAFETY: the caller must uphold the safety contract. + unsafe { atomic_update_kuser_cmpxchg64(dst, |x| x.wrapping_sub(val)) } + } + fallback = atomic_sub_seqcst +} +atomic_with_ifunc! { + unsafe fn atomic_and(dst: *mut u64, val: u64) -> u64 { + // SAFETY: the caller must uphold the safety contract. + unsafe { atomic_update_kuser_cmpxchg64(dst, |x| x & val) } + } + fallback = atomic_and_seqcst +} +atomic_with_ifunc! { + unsafe fn atomic_nand(dst: *mut u64, val: u64) -> u64 { + // SAFETY: the caller must uphold the safety contract. + unsafe { atomic_update_kuser_cmpxchg64(dst, |x| !(x & val)) } + } + fallback = atomic_nand_seqcst +} +atomic_with_ifunc! { + unsafe fn atomic_or(dst: *mut u64, val: u64) -> u64 { + // SAFETY: the caller must uphold the safety contract. + unsafe { atomic_update_kuser_cmpxchg64(dst, |x| x | val) } + } + fallback = atomic_or_seqcst +} +atomic_with_ifunc! { + unsafe fn atomic_xor(dst: *mut u64, val: u64) -> u64 { + // SAFETY: the caller must uphold the safety contract. + unsafe { atomic_update_kuser_cmpxchg64(dst, |x| x ^ val) } + } + fallback = atomic_xor_seqcst +} +atomic_with_ifunc! { + unsafe fn atomic_max(dst: *mut u64, val: u64) -> u64 { + #[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)] + // SAFETY: the caller must uphold the safety contract. + unsafe { + atomic_update_kuser_cmpxchg64(dst, |x| core::cmp::max(x as i64, val as i64) as u64) + } + } + fallback = atomic_max_seqcst +} +atomic_with_ifunc! { + unsafe fn atomic_umax(dst: *mut u64, val: u64) -> u64 { + // SAFETY: the caller must uphold the safety contract. + unsafe { atomic_update_kuser_cmpxchg64(dst, |x| core::cmp::max(x, val)) } + } + fallback = atomic_umax_seqcst +} +atomic_with_ifunc! { + unsafe fn atomic_min(dst: *mut u64, val: u64) -> u64 { + #[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)] + // SAFETY: the caller must uphold the safety contract. + unsafe { + atomic_update_kuser_cmpxchg64(dst, |x| core::cmp::min(x as i64, val as i64) as u64) + } + } + fallback = atomic_min_seqcst +} +atomic_with_ifunc! { + unsafe fn atomic_umin(dst: *mut u64, val: u64) -> u64 { + // SAFETY: the caller must uphold the safety contract. + unsafe { atomic_update_kuser_cmpxchg64(dst, |x| core::cmp::min(x, val)) } } + fallback = atomic_umin_seqcst +} +atomic_with_ifunc! { + unsafe fn atomic_not(dst: *mut u64) -> u64 { + // SAFETY: the caller must uphold the safety contract. + unsafe { atomic_update_kuser_cmpxchg64(dst, |x| !x) } + } + fallback = atomic_not_seqcst +} +atomic_with_ifunc! { + unsafe fn atomic_neg(dst: *mut u64) -> u64 { + // SAFETY: the caller must uphold the safety contract. + unsafe { atomic_update_kuser_cmpxchg64(dst, u64::wrapping_neg) } + } + fallback = atomic_neg_seqcst } macro_rules! atomic64 { - ($atomic_type:ident, $int_type:ident) => { + ($atomic_type:ident, $int_type:ident, $atomic_max:ident, $atomic_min:ident) => { #[repr(C, align(8))] pub(crate) struct $atomic_type { v: core::cell::UnsafeCell<$int_type>, @@ -171,7 +281,7 @@ macro_rules! atomic64 { #[inline] pub(crate) fn is_lock_free() -> bool { - kuser_helper_version() >= 5 + has_kuser_cmpxchg64() } #[inline] pub(crate) const fn is_always_lock_free() -> bool { @@ -228,13 +338,15 @@ macro_rules! atomic64 { // SAFETY: any data races are prevented by the kernel user helper or the lock // and the raw pointer passed in is valid because we got it from a reference. unsafe { - match atomic_compare_exchange( + let (res, ok) = atomic_compare_exchange( self.v.get().cast::(), current as u64, new as u64, - ) { - Ok(v) => Ok(v as $int_type), - Err(v) => Err(v as $int_type), + ); + if ok { + Ok(res as $int_type) + } else { + Err(res as $int_type) } } } @@ -255,88 +367,63 @@ macro_rules! atomic64 { pub(crate) fn fetch_add(&self, val: $int_type, _order: Ordering) -> $int_type { // SAFETY: any data races are prevented by the kernel user helper or the lock // and the raw pointer passed in is valid because we got it from a reference. - unsafe { - atomic_update(self.v.get().cast::(), |x| x.wrapping_add(val as u64)) - as $int_type - } + unsafe { atomic_add(self.v.get().cast::(), val as u64) as $int_type } } #[inline] pub(crate) fn fetch_sub(&self, val: $int_type, _order: Ordering) -> $int_type { // SAFETY: any data races are prevented by the kernel user helper or the lock // and the raw pointer passed in is valid because we got it from a reference. - unsafe { - atomic_update(self.v.get().cast::(), |x| x.wrapping_sub(val as u64)) - as $int_type - } + unsafe { atomic_sub(self.v.get().cast::(), val as u64) as $int_type } } #[inline] pub(crate) fn fetch_and(&self, val: $int_type, _order: Ordering) -> $int_type { // SAFETY: any data races are prevented by the kernel user helper or the lock // and the raw pointer passed in is valid because we got it from a reference. - unsafe { - atomic_update(self.v.get().cast::(), |x| x & val as u64) as $int_type - } + unsafe { atomic_and(self.v.get().cast::(), val as u64) as $int_type } } #[inline] pub(crate) fn fetch_nand(&self, val: $int_type, _order: Ordering) -> $int_type { // SAFETY: any data races are prevented by the kernel user helper or the lock // and the raw pointer passed in is valid because we got it from a reference. - unsafe { - atomic_update(self.v.get().cast::(), |x| !(x & val as u64)) as $int_type - } + unsafe { atomic_nand(self.v.get().cast::(), val as u64) as $int_type } } #[inline] pub(crate) fn fetch_or(&self, val: $int_type, _order: Ordering) -> $int_type { // SAFETY: any data races are prevented by the kernel user helper or the lock // and the raw pointer passed in is valid because we got it from a reference. - unsafe { - atomic_update(self.v.get().cast::(), |x| x | val as u64) as $int_type - } + unsafe { atomic_or(self.v.get().cast::(), val as u64) as $int_type } } #[inline] pub(crate) fn fetch_xor(&self, val: $int_type, _order: Ordering) -> $int_type { // SAFETY: any data races are prevented by the kernel user helper or the lock // and the raw pointer passed in is valid because we got it from a reference. - unsafe { - atomic_update(self.v.get().cast::(), |x| x ^ val as u64) as $int_type - } + unsafe { atomic_xor(self.v.get().cast::(), val as u64) as $int_type } } #[inline] pub(crate) fn fetch_max(&self, val: $int_type, _order: Ordering) -> $int_type { // SAFETY: any data races are prevented by the kernel user helper or the lock // and the raw pointer passed in is valid because we got it from a reference. - unsafe { - atomic_update(self.v.get().cast::(), |x| { - core::cmp::max(x as $int_type, val) as u64 - }) as $int_type - } + unsafe { $atomic_max(self.v.get().cast::(), val as u64) as $int_type } } #[inline] pub(crate) fn fetch_min(&self, val: $int_type, _order: Ordering) -> $int_type { // SAFETY: any data races are prevented by the kernel user helper or the lock // and the raw pointer passed in is valid because we got it from a reference. - unsafe { - atomic_update(self.v.get().cast::(), |x| { - core::cmp::min(x as $int_type, val) as u64 - }) as $int_type - } + unsafe { $atomic_min(self.v.get().cast::(), val as u64) as $int_type } } #[inline] pub(crate) fn fetch_not(&self, _order: Ordering) -> $int_type { // SAFETY: any data races are prevented by the kernel user helper or the lock // and the raw pointer passed in is valid because we got it from a reference. - unsafe { - atomic_update(self.v.get().cast::(), |x| !(x as $int_type) as u64) - as $int_type - } + unsafe { atomic_not(self.v.get().cast::()) as $int_type } } #[inline] pub(crate) fn not(&self, order: Ordering) { @@ -347,7 +434,7 @@ macro_rules! atomic64 { pub(crate) fn fetch_neg(&self, _order: Ordering) -> $int_type { // SAFETY: any data races are prevented by the kernel user helper or the lock // and the raw pointer passed in is valid because we got it from a reference. - unsafe { atomic_update(self.v.get().cast::(), u64::wrapping_neg) as $int_type } + unsafe { atomic_neg(self.v.get().cast::()) as $int_type } } #[inline] pub(crate) fn neg(&self, order: Ordering) { @@ -362,17 +449,25 @@ macro_rules! atomic64 { }; } -atomic64!(AtomicI64, i64); -atomic64!(AtomicU64, u64); +atomic64!(AtomicI64, i64, atomic_max, atomic_min); +atomic64!(AtomicU64, u64, atomic_umax, atomic_umin); +#[allow( + clippy::alloc_instead_of_core, + clippy::std_instead_of_alloc, + clippy::std_instead_of_core, + clippy::undocumented_unsafe_blocks, + clippy::wildcard_imports +)] #[cfg(test)] mod tests { use super::*; #[test] fn kuser_helper_version() { - let version = super::kuser_helper_version(); + let version = __kuser_helper_version(); assert!(version >= 5, "{:?}", version); + assert_eq!(version, unsafe { (KUSER_HELPER_VERSION as *const i32).read() }); } test_atomic_int!(i64); diff --git a/src/imp/atomic128/x86_64.rs b/src/imp/atomic128/x86_64.rs index 08bf78a9..e717a1b4 100644 --- a/src/imp/atomic128/x86_64.rs +++ b/src/imp/atomic128/x86_64.rs @@ -9,6 +9,13 @@ include!("macros.rs"); +#[cfg(any( + test, + not(any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b")), +))] +#[path = "../fallback/outline_atomics.rs"] +mod fallback; + #[path = "detect/x86_64.rs"] mod detect; @@ -369,7 +376,7 @@ unsafe fn atomic_compare_exchange( if detect::has_cmpxchg16b() { _cmpxchg16b } else { - _atomic_compare_exchange_fallback + fallback::atomic_compare_exchange } }) } @@ -380,29 +387,6 @@ unsafe fn atomic_compare_exchange( Err(res) } } -#[cfg(any( - test, - not(any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b")), -))] -#[cold] -unsafe fn _atomic_compare_exchange_fallback( - dst: *mut u128, - old: u128, - new: u128, - success: Ordering, - failure: Ordering, -) -> (u128, bool) { - #[allow(clippy::cast_ptr_alignment)] - // SAFETY: the caller must uphold the safety contract. - unsafe { - match (*(dst as *const super::fallback::AtomicU128)) - .compare_exchange(old, new, success, failure) - { - Ok(v) => (v, true), - Err(v) => (v, false), - } - } -} use atomic_compare_exchange as atomic_compare_exchange_weak; @@ -810,7 +794,7 @@ mod tests_no_cmpxchg16b { success: Ordering, failure: Ordering, ) -> (u128, bool) { - unsafe { super::_atomic_compare_exchange_fallback(dst, old, new, success, failure) } + unsafe { fallback::atomic_compare_exchange(dst, old, new, success, failure) } } #[inline] unsafe fn byte_wise_atomic_load(src: *mut u128) -> u128 { diff --git a/src/imp/fallback/outline_atomics.rs b/src/imp/fallback/outline_atomics.rs new file mode 100644 index 00000000..b553c2fa --- /dev/null +++ b/src/imp/fallback/outline_atomics.rs @@ -0,0 +1,158 @@ +// Helper for outline-atomics. +// +// On architectures where DW atomics are not supported on older CPUs, we use +// fallback implementation when DW atomic instructions are not supported and +// outline-atomics is enabled. +// +// This module provides helpers to implement them. + +use core::sync::atomic::Ordering; + +#[cfg(target_arch = "x86_64")] +#[allow(clippy::upper_case_acronyms)] +pub(crate) type UDW = u128; +#[cfg(target_arch = "x86_64")] +pub(crate) type AtomicUDW = super::super::fallback::AtomicU128; +// #[cfg(target_arch = "x86_64")] +// pub(crate) type AtomicIDW = super::super::fallback::AtomicI128; + +#[cfg(target_arch = "arm")] +#[allow(clippy::upper_case_acronyms)] +pub(crate) type UDW = u64; +#[cfg(target_arch = "arm")] +pub(crate) type AtomicUDW = super::super::fallback::AtomicU64; +#[cfg(target_arch = "arm")] +pub(crate) type AtomicIDW = super::super::fallback::AtomicI64; + +#[cfg(not(target_arch = "x86_64"))] +#[cold] +pub(crate) unsafe fn atomic_load(src: *mut UDW, order: Ordering) -> UDW { + #[allow(clippy::cast_ptr_alignment)] + // SAFETY: the caller must uphold the safety contract. + unsafe { + (*(src as *const AtomicUDW)).load(order) + } +} +#[cfg(not(target_arch = "x86_64"))] +fn_alias! { + #[cold] + pub(crate) unsafe fn(src: *mut UDW) -> UDW; + // fallback's atomic load has at least acquire semantics. + #[cfg(not(target_arch = "arm"))] + atomic_load_non_seqcst = atomic_load(Ordering::Acquire); + atomic_load_seqcst = atomic_load(Ordering::SeqCst); +} + +#[cfg(not(target_arch = "x86_64"))] +#[cold] +pub(crate) unsafe fn atomic_store(dst: *mut UDW, val: UDW, order: Ordering) { + #[allow(clippy::cast_ptr_alignment)] + // SAFETY: the caller must uphold the safety contract. + unsafe { + (*(dst as *const AtomicUDW)).store(val, order); + } +} +#[cfg(not(target_arch = "x86_64"))] +fn_alias! { + #[cold] + pub(crate) unsafe fn(dst: *mut UDW, val: UDW); + // fallback's atomic store has at least release semantics. + #[cfg(not(target_arch = "arm"))] + atomic_store_non_seqcst = atomic_store(Ordering::Release); + atomic_store_seqcst = atomic_store(Ordering::SeqCst); +} + +#[cold] +pub(crate) unsafe fn atomic_compare_exchange( + dst: *mut UDW, + old: UDW, + new: UDW, + success: Ordering, + failure: Ordering, +) -> (UDW, bool) { + #[allow(clippy::cast_ptr_alignment)] + // SAFETY: the caller must uphold the safety contract. + unsafe { + match (*(dst as *const AtomicUDW)).compare_exchange(old, new, success, failure) { + Ok(v) => (v, true), + Err(v) => (v, false), + } + } +} +#[cfg(not(target_arch = "x86_64"))] +fn_alias! { + #[cold] + pub(crate) unsafe fn(dst: *mut UDW, old: UDW, new: UDW) -> (UDW, bool); + // fallback's atomic RMW has at least AcqRel semantics. + #[cfg(not(target_arch = "arm"))] + atomic_compare_exchange_non_seqcst + = atomic_compare_exchange(Ordering::AcqRel, Ordering::Acquire); + atomic_compare_exchange_seqcst + = atomic_compare_exchange(Ordering::SeqCst, Ordering::SeqCst); +} + +macro_rules! atomic_rmw_3 { + ( + $name:ident($atomic_type:ident::$method_name:ident), + $non_seqcst_alias:ident, $seqcst_alias:ident + ) => { + #[cfg(not(target_arch = "x86_64"))] + #[cold] + pub(crate) unsafe fn $name(dst: *mut UDW, val: UDW, order: Ordering) -> UDW { + #[allow(clippy::cast_ptr_alignment)] + // SAFETY: the caller must uphold the safety contract. + unsafe { + (*(dst as *const $atomic_type)).$method_name(val as _, order) as UDW + } + } + #[cfg(not(target_arch = "x86_64"))] + fn_alias! { + #[cold] + pub(crate) unsafe fn(dst: *mut UDW, val: UDW) -> UDW; + // fallback's atomic RMW has at least AcqRel semantics. + #[cfg(not(target_arch = "arm"))] + $non_seqcst_alias = $name(Ordering::AcqRel); + $seqcst_alias = $name(Ordering::SeqCst); + } + }; +} +macro_rules! atomic_rmw_2 { + ( + $name:ident($atomic_type:ident::$method_name:ident), + $non_seqcst_alias:ident, $seqcst_alias:ident + ) => { + #[cfg(not(target_arch = "x86_64"))] + #[cold] + pub(crate) unsafe fn $name(dst: *mut UDW, order: Ordering) -> UDW { + #[allow(clippy::cast_ptr_alignment)] + // SAFETY: the caller must uphold the safety contract. + unsafe { + (*(dst as *const $atomic_type)).$method_name(order) as UDW + } + } + #[cfg(not(target_arch = "x86_64"))] + fn_alias! { + #[cold] + pub(crate) unsafe fn(dst: *mut UDW) -> UDW; + // fallback's atomic RMW has at least AcqRel semantics. + #[cfg(not(target_arch = "arm"))] + $non_seqcst_alias = $name(Ordering::AcqRel); + $seqcst_alias = $name(Ordering::SeqCst); + } + }; +} + +atomic_rmw_3!(atomic_swap(AtomicUDW::swap), atomic_swap_non_seqcst, atomic_swap_seqcst); +atomic_rmw_3!(atomic_add(AtomicUDW::fetch_add), atomic_add_non_seqcst, atomic_add_seqcst); +atomic_rmw_3!(atomic_sub(AtomicUDW::fetch_sub), atomic_sub_non_seqcst, atomic_sub_seqcst); +atomic_rmw_3!(atomic_and(AtomicUDW::fetch_and), atomic_and_non_seqcst, atomic_and_seqcst); +atomic_rmw_3!(atomic_nand(AtomicUDW::fetch_nand), atomic_nand_non_seqcst, atomic_nand_seqcst); +atomic_rmw_3!(atomic_or(AtomicUDW::fetch_or), atomic_or_non_seqcst, atomic_or_seqcst); +atomic_rmw_3!(atomic_xor(AtomicUDW::fetch_xor), atomic_xor_non_seqcst, atomic_xor_seqcst); +atomic_rmw_3!(atomic_max(AtomicIDW::fetch_max), atomic_max_non_seqcst, atomic_max_seqcst); +atomic_rmw_3!(atomic_umax(AtomicUDW::fetch_max), atomic_umax_non_seqcst, atomic_umax_seqcst); +atomic_rmw_3!(atomic_min(AtomicIDW::fetch_min), atomic_min_non_seqcst, atomic_min_seqcst); +atomic_rmw_3!(atomic_umin(AtomicUDW::fetch_min), atomic_umin_non_seqcst, atomic_umin_seqcst); + +atomic_rmw_2!(atomic_not(AtomicUDW::fetch_not), atomic_not_non_seqcst, atomic_not_seqcst); +atomic_rmw_2!(atomic_neg(AtomicUDW::fetch_neg), atomic_neg_non_seqcst, atomic_neg_seqcst); diff --git a/src/utils.rs b/src/utils.rs index 372302bb..de73f895 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -61,6 +61,7 @@ macro_rules! serde_impls { #[cfg(any( target_arch = "aarch64", target_arch = "arm", + target_arch = "powerpc64", all(target_arch = "x86_64", not(target_env = "sgx")), ))] macro_rules! ifunc { @@ -90,28 +91,30 @@ macro_rules! ifunc { #[allow(unused_macros)] #[cfg(not(portable_atomic_no_outline_atomics))] -#[cfg(any(target_arch = "aarch64", target_arch = "x86_64"))] +#[cfg(any(target_arch = "aarch64", target_arch = "arm", target_arch = "x86_64"))] macro_rules! fn_alias { ( - $(#[$($attr:tt)*])* - unsafe fn($($arg_pat:ident: $arg_ty:ty),*) $(-> $ret_ty:ty)?; + $(#[$($fn_attr:tt)*])* + $vis:vis unsafe fn($($arg_pat:ident: $arg_ty:ty),*) $(-> $ret_ty:ty)?; + $(#[$($alias_attr:tt)*])* $new:ident = $from:ident($($last_args:tt)*); $($rest:tt)* ) => { - $(#[$($attr)*])* - unsafe fn $new($($arg_pat: $arg_ty),*) $(-> $ret_ty)? { + $(#[$($fn_attr)*])* + $(#[$($alias_attr)*])* + $vis unsafe fn $new($($arg_pat: $arg_ty),*) $(-> $ret_ty)? { // SAFETY: the caller must uphold the safety contract. unsafe { $from($($arg_pat,)* $($last_args)*) } } fn_alias! { - $(#[$($attr)*])* - unsafe fn($($arg_pat: $arg_ty),*) $(-> $ret_ty)?; + $(#[$($fn_attr)*])* + $vis unsafe fn($($arg_pat: $arg_ty),*) $(-> $ret_ty)?; $($rest)* } }; ( $(#[$($attr:tt)*])* - unsafe fn($($arg_pat:ident: $arg_ty:ty),*) $(-> $ret_ty:ty)?; + $vis:vis unsafe fn($($arg_pat:ident: $arg_ty:ty),*) $(-> $ret_ty:ty)?; ) => {} }