crossbeam_utils/atomic/
consume.rs

1#[cfg(not(crossbeam_no_atomic))]
2use core::sync::atomic::Ordering;
3
4/// Trait which allows reading from primitive atomic types with "consume" ordering.
5pub trait AtomicConsume {
6    /// Type returned by `load_consume`.
7    type Val;
8
9    /// Loads a value from the atomic using a "consume" memory ordering.
10    ///
11    /// This is similar to the "acquire" ordering, except that an ordering is
12    /// only guaranteed with operations that "depend on" the result of the load.
13    /// However consume loads are usually much faster than acquire loads on
14    /// architectures with a weak memory model since they don't require memory
15    /// fence instructions.
16    ///
17    /// The exact definition of "depend on" is a bit vague, but it works as you
18    /// would expect in practice since a lot of software, especially the Linux
19    /// kernel, rely on this behavior.
20    ///
21    /// This is currently only implemented on ARM and AArch64, where a fence
22    /// can be avoided. On other architectures this will fall back to a simple
23    /// `load(Ordering::Acquire)`.
24    fn load_consume(&self) -> Self::Val;
25}
26
27#[cfg(not(crossbeam_no_atomic))]
28// Miri and Loom don't support "consume" ordering and ThreadSanitizer doesn't treat
29// load(Relaxed) + compiler_fence(Acquire) as "consume" load.
30// LLVM generates machine code equivalent to fence(Acquire) in compiler_fence(Acquire)
31// on PowerPC, MIPS, etc. (https://godbolt.org/z/hffvjvW7h), so for now the fence
32// can be actually avoided here only on ARM and AArch64. See also
33// https://github.com/rust-lang/rust/issues/62256.
34#[cfg(all(
35    any(target_arch = "arm", target_arch = "aarch64"),
36    not(any(miri, crossbeam_loom, crossbeam_sanitize_thread)),
37))]
38macro_rules! impl_consume {
39    () => {
40        #[inline]
41        fn load_consume(&self) -> Self::Val {
42            use crate::primitive::sync::atomic::compiler_fence;
43            let result = self.load(Ordering::Relaxed);
44            compiler_fence(Ordering::Acquire);
45            result
46        }
47    };
48}
49
50#[cfg(not(crossbeam_no_atomic))]
51#[cfg(not(all(
52    any(target_arch = "arm", target_arch = "aarch64"),
53    not(any(miri, crossbeam_loom, crossbeam_sanitize_thread)),
54)))]
55macro_rules! impl_consume {
56    () => {
57        #[inline]
58        fn load_consume(&self) -> Self::Val {
59            self.load(Ordering::Acquire)
60        }
61    };
62}
63
64macro_rules! impl_atomic {
65    ($atomic:ident, $val:ty) => {
66        #[cfg(not(crossbeam_no_atomic))]
67        impl AtomicConsume for core::sync::atomic::$atomic {
68            type Val = $val;
69            impl_consume!();
70        }
71        #[cfg(crossbeam_loom)]
72        impl AtomicConsume for loom::sync::atomic::$atomic {
73            type Val = $val;
74            impl_consume!();
75        }
76    };
77}
78
79impl_atomic!(AtomicBool, bool);
80impl_atomic!(AtomicUsize, usize);
81impl_atomic!(AtomicIsize, isize);
82impl_atomic!(AtomicU8, u8);
83impl_atomic!(AtomicI8, i8);
84impl_atomic!(AtomicU16, u16);
85impl_atomic!(AtomicI16, i16);
86#[cfg(any(target_has_atomic = "32", not(target_pointer_width = "16")))]
87impl_atomic!(AtomicU32, u32);
88#[cfg(any(target_has_atomic = "32", not(target_pointer_width = "16")))]
89impl_atomic!(AtomicI32, i32);
90#[cfg(any(
91    target_has_atomic = "64",
92    not(any(target_pointer_width = "16", target_pointer_width = "32")),
93))]
94impl_atomic!(AtomicU64, u64);
95#[cfg(any(
96    target_has_atomic = "64",
97    not(any(target_pointer_width = "16", target_pointer_width = "32")),
98))]
99impl_atomic!(AtomicI64, i64);
100
101#[cfg(not(crossbeam_no_atomic))]
102impl<T> AtomicConsume for core::sync::atomic::AtomicPtr<T> {
103    type Val = *mut T;
104    impl_consume!();
105}
106
107#[cfg(crossbeam_loom)]
108impl<T> AtomicConsume for loom::sync::atomic::AtomicPtr<T> {
109    type Val = *mut T;
110    impl_consume!();
111}
crossbeam_utils/atomic/consume.rs

crossbeam_utils/atomic/
consume.rs
