Relax failure ordering in private CAS loop helpers

src/imp/atomic128/intrinsics.rs

@@ -284,13 +284,14 @@ unsafe fn atomic_update<F>(dst: *mut u128, order: Ordering, mut f: F) -> u128
 where
     F: FnMut(u128) -> u128,
 {
-    let failure = crate::utils::strongest_failure_ordering(order);
     // SAFETY: the caller must uphold the safety contract for `atomic_update`.
     unsafe {
-        let mut old = atomic_load(dst, failure);
+        // This is a private function and all instances of `f` only operate on the value
+        // loaded, so there is no need to synchronize the first load/failed CAS.
+        let mut old = atomic_load(dst, Ordering::Relaxed);
         loop {
             let next = f(old);
-            match atomic_compare_exchange_weak(dst, old, next, order, failure) {
+            match atomic_compare_exchange_weak(dst, old, next, order, Ordering::Relaxed) {
                 Ok(x) => return x,
                 Err(x) => old = x,
             }

src/imp/atomic128/s390x.rs

@@ -196,13 +196,14 @@ unsafe fn atomic_update<F>(dst: *mut u128, order: Ordering, mut f: F) -> u128
 where
     F: FnMut(u128) -> u128,
 {
-    let failure = crate::utils::strongest_failure_ordering(order);
     // SAFETY: the caller must uphold the safety contract for `atomic_update`.
     unsafe {
-        let mut old = atomic_load(dst, failure);
+        // This is a private function and all instances of `f` only operate on the value
+        // loaded, so there is no need to synchronize the first load/failed CAS.
+        let mut old = atomic_load(dst, Ordering::Relaxed);
         loop {
             let next = f(old);
-            match atomic_compare_exchange_weak(dst, old, next, order, failure) {
+            match atomic_compare_exchange_weak(dst, old, next, order, Ordering::Relaxed) {
                 Ok(x) => return x,
                 Err(x) => old = x,
             }

src/imp/atomic128/x86_64.rs

@@ -367,7 +367,6 @@ unsafe fn atomic_update<F>(dst: *mut u128, order: Ordering, mut f: F) -> u128
 where
     F: FnMut(u128) -> u128,
 {
-    let failure = crate::utils::strongest_failure_ordering(order);
     // SAFETY: the caller must uphold the safety contract for `atomic_update`.
     unsafe {
         // This is based on the code generated for the first load in DW RMWs by LLVM,
@@ -387,7 +386,9 @@ where
         let mut old = byte_wise_atomic_load(dst);
         loop {
             let next = f(old);
-            match atomic_compare_exchange_weak(dst, old, next, order, failure) {
+            // This is a private function and all instances of `f` only operate on the value
+            // loaded, so there is no need to synchronize the first load/failed CAS.
+            match atomic_compare_exchange_weak(dst, old, next, order, Ordering::Relaxed) {
                 Ok(x) => return x,
                 Err(x) => old = x,
             }
@@ -561,12 +562,11 @@ mod tests_no_cmpxchg16b {
     where
         F: FnMut(u128) -> u128,
     {
-        let failure = crate::utils::strongest_failure_ordering(order);
         unsafe {
             let mut old = byte_wise_atomic_load(dst);
             loop {
                 let next = f(old);
-                match atomic_compare_exchange_weak(dst, old, next, order, failure) {
+                match atomic_compare_exchange_weak(dst, old, next, order, Ordering::Relaxed) {
                     Ok(x) => return x,
                     Err(x) => old = x,
                 }

src/imp/core_atomic.rs

@@ -291,15 +291,16 @@ macro_rules! atomic_int {
             }
             #[allow(dead_code)]
             #[inline]
-            fn fetch_update_<F>(&self, set_order: Ordering, mut f: F) -> $int_type
+            fn fetch_update_<F>(&self, order: Ordering, mut f: F) -> $int_type
             where
                 F: FnMut($int_type) -> $int_type,
             {
-                let fetch_order = crate::utils::strongest_failure_ordering(set_order);
-                let mut prev = self.load(fetch_order);
+                // This is a private function and all instances of `f` only operate on the value
+                // loaded, so there is no need to synchronize the first load/failed CAS.
+                let mut prev = self.load(Ordering::Relaxed);
                 loop {
                     let next = f(prev);
-                    match self.compare_exchange_weak(prev, next, set_order, fetch_order) {
+                    match self.compare_exchange_weak(prev, next, order, Ordering::Relaxed) {
                         Ok(x) => return x,
                         Err(next_prev) => prev = next_prev,
                     }

src/imp/float.rs

@@ -153,15 +153,16 @@ macro_rules! atomic_float {
             }
 
             #[inline]
-            fn fetch_update_<F>(&self, set_order: Ordering, mut f: F) -> $float_type
+            fn fetch_update_<F>(&self, order: Ordering, mut f: F) -> $float_type
             where
                 F: FnMut($float_type) -> $float_type,
             {
-                let fetch_order = crate::utils::strongest_failure_ordering(set_order);
-                let mut prev = self.load(fetch_order);
+                // This is a private function and all instances of `f` only operate on the value
+                // loaded, so there is no need to synchronize the first load/failed CAS.
+                let mut prev = self.load(Ordering::Relaxed);
                 loop {
                     let next = f(prev);
-                    match self.compare_exchange_weak(prev, next, set_order, fetch_order) {
+                    match self.compare_exchange_weak(prev, next, order, Ordering::Relaxed) {
                         Ok(x) => return x,
                         Err(next_prev) => prev = next_prev,
                     }

src/lib.rs

@@ -1917,15 +1917,16 @@ impl<T> AtomicPtr<T> {
 
     #[cfg(miri)]
     #[inline]
-    fn fetch_update_<F>(&self, set_order: Ordering, mut f: F) -> *mut T
+    fn fetch_update_<F>(&self, order: Ordering, mut f: F) -> *mut T
     where
         F: FnMut(*mut T) -> *mut T,
     {
-        let fetch_order = crate::utils::strongest_failure_ordering(set_order);
-        let mut prev = self.load(fetch_order);
+        // This is a private function and all instances of `f` only operate on the value
+        // loaded, so there is no need to synchronize the first load/failed CAS.
+        let mut prev = self.load(Ordering::Relaxed);
         loop {
             let next = f(prev);
-            match self.compare_exchange_weak(prev, next, set_order, fetch_order) {
+            match self.compare_exchange_weak(prev, next, order, Ordering::Relaxed) {
                 Ok(x) => return x,
                 Err(next_prev) => prev = next_prev,
             }

src/utils.rs

@@ -269,6 +269,7 @@ pub(crate) struct NoRefUnwindSafe(UnsafeCell<()>);
 unsafe impl Sync for NoRefUnwindSafe {}
 
 // https://github.com/rust-lang/rust/blob/1.67.0/library/core/src/sync/atomic.rs#L2956
+#[allow(dead_code)]
 #[inline]
 pub(crate) fn strongest_failure_ordering(order: Ordering) -> Ordering {
     match order {