Add atomic support for pre-ARMv6 on Linux
This commit is contained in:
parent
fe8d893a6b
commit
73e38dcf2f
5
build.rs
5
build.rs
|
@ -40,6 +40,11 @@ fn main() {
|
||||||
if llvm_target[0] == "thumbv6m" {
|
if llvm_target[0] == "thumbv6m" {
|
||||||
println!("cargo:rustc-cfg=thumbv6m")
|
println!("cargo:rustc-cfg=thumbv6m")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Only emit the ARM Linux atomic emulation on pre-ARMv6 architectures.
|
||||||
|
if llvm_target[0] == "armv5te" {
|
||||||
|
println!("cargo:rustc-cfg=armv5te")
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(feature = "gen-tests")]
|
#[cfg(feature = "gen-tests")]
|
||||||
|
|
|
@ -0,0 +1,170 @@
|
||||||
|
use core::intrinsics;
|
||||||
|
use core::mem;
|
||||||
|
|
||||||
|
// Kernel-provided user-mode helper functions:
|
||||||
|
// https://www.kernel.org/doc/Documentation/arm/kernel_user_helpers.txt
|
||||||
|
unsafe fn __kuser_cmpxchg(oldval: u32, newval: u32, ptr: *mut u32) -> bool {
|
||||||
|
let out: u32;
|
||||||
|
// FIXME: we can't use BLX on ARMv4
|
||||||
|
asm!("blx ${0}"
|
||||||
|
: "={r0}" (out)
|
||||||
|
: "r" (0xffff0fc0u32)
|
||||||
|
"{r0}" (oldval),
|
||||||
|
"{r1}" (newval),
|
||||||
|
"{r2}" (ptr)
|
||||||
|
: "r3", "r12", "lr", "cc", "memory");
|
||||||
|
out == 0
|
||||||
|
}
|
||||||
|
unsafe fn __kuser_memory_barrier() {
|
||||||
|
// FIXME: we can't use BLX on ARMv4
|
||||||
|
asm!("blx ${0}"
|
||||||
|
:
|
||||||
|
: "r" (0xffff0fa0u32)
|
||||||
|
: "lr", "memory");
|
||||||
|
}
|
||||||
|
|
||||||
|
// Word-align a pointer
|
||||||
|
fn align_ptr<T>(ptr: *mut T) -> *mut u32 {
|
||||||
|
// This gives us a mask of 0 when T == u32 since the pointer is already
|
||||||
|
// supposed to be aligned, which avoids any masking in that case.
|
||||||
|
let ptr_mask = 3 & (4 - mem::size_of::<T>());
|
||||||
|
(ptr as usize & !ptr_mask) as *mut u32
|
||||||
|
}
|
||||||
|
|
||||||
|
// Calculate the shift and mask of a value inside an aligned word
|
||||||
|
fn get_shift_mask<T>(ptr: *mut T) -> (u32, u32) {
|
||||||
|
// Mask to get the low byte/halfword/word
|
||||||
|
let mask = match mem::size_of::<T>() {
|
||||||
|
1 => 0xff,
|
||||||
|
2 => 0xffff,
|
||||||
|
4 => 0xffffffff,
|
||||||
|
_ => unreachable!(),
|
||||||
|
};
|
||||||
|
|
||||||
|
// If we are on big-endian then we need to adjust the shift accordingly
|
||||||
|
let endian_adjust = if cfg!(target_endian = "little") {
|
||||||
|
0
|
||||||
|
} else {
|
||||||
|
4 - mem::size_of::<T>() as u32
|
||||||
|
};
|
||||||
|
|
||||||
|
// Shift to get the desired element in the word
|
||||||
|
let ptr_mask = 3 & (4 - mem::size_of::<T>());
|
||||||
|
let shift = ((ptr as usize & ptr_mask) as u32 ^ endian_adjust) * 8;
|
||||||
|
|
||||||
|
(shift, mask)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Extract a value from an aligned word
|
||||||
|
fn extract_aligned(aligned: u32, shift: u32, mask: u32) -> u32 {
|
||||||
|
(aligned >> shift) & mask
|
||||||
|
}
|
||||||
|
|
||||||
|
// Insert a value into an aligned word
|
||||||
|
fn insert_aligned(aligned: u32, val: u32, shift: u32, mask: u32) -> u32 {
|
||||||
|
(aligned & !(mask << shift)) | ((val & mask) << shift)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Generic atomic read-modify-write operation
|
||||||
|
unsafe fn atomic_rmw<T, F: Fn(u32) -> u32>(ptr: *mut T, f: F) -> u32 {
|
||||||
|
let aligned_ptr = align_ptr(ptr);
|
||||||
|
let (shift, mask) = get_shift_mask(ptr);
|
||||||
|
|
||||||
|
loop {
|
||||||
|
let curval_aligned = intrinsics::atomic_load_unordered(aligned_ptr);
|
||||||
|
let curval = extract_aligned(curval_aligned, shift, mask);
|
||||||
|
let newval = f(curval);
|
||||||
|
let newval_aligned = insert_aligned(curval_aligned, newval, shift, mask);
|
||||||
|
if __kuser_cmpxchg(curval_aligned, newval_aligned, aligned_ptr) {
|
||||||
|
return curval;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Generic atomic compare-exchange operation
|
||||||
|
unsafe fn atomic_cmpxchg<T>(oldval: u32, newval: u32, ptr: *mut T) -> u32 {
|
||||||
|
let aligned_ptr = align_ptr(ptr);
|
||||||
|
let (shift, mask) = get_shift_mask(ptr);
|
||||||
|
|
||||||
|
loop {
|
||||||
|
let curval_aligned = intrinsics::atomic_load_unordered(aligned_ptr);
|
||||||
|
let curval = extract_aligned(curval_aligned, shift, mask);
|
||||||
|
if curval != oldval {
|
||||||
|
return curval;
|
||||||
|
}
|
||||||
|
let newval_aligned = insert_aligned(curval_aligned, newval, shift, mask);
|
||||||
|
if __kuser_cmpxchg(curval_aligned, newval_aligned, aligned_ptr) {
|
||||||
|
return oldval;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
macro_rules! atomic_rmw {
|
||||||
|
($name:ident, $ty:ty, $op:expr) => {
|
||||||
|
#[cfg_attr(not(feature = "mangled-names"), no_mangle)]
|
||||||
|
pub unsafe extern "C" fn $name(ptr: *mut $ty, val: $ty) -> $ty {
|
||||||
|
atomic_rmw(ptr, |x| $op(x as $ty, val) as u32) as $ty
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
macro_rules! atomic_cmpxchg {
|
||||||
|
($name:ident, $ty:ty) => {
|
||||||
|
#[cfg_attr(not(feature = "mangled-names"), no_mangle)]
|
||||||
|
pub unsafe extern "C" fn $name(oldval: $ty, newval: $ty, ptr: *mut $ty) -> $ty {
|
||||||
|
atomic_cmpxchg(oldval as u32, newval as u32, ptr) as $ty
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
atomic_rmw!(__sync_fetch_and_add_1, u8, |a: u8, b: u8| a.wrapping_add(b));
|
||||||
|
atomic_rmw!(__sync_fetch_and_add_2, u16, |a: u16, b: u16| a.wrapping_add(b));
|
||||||
|
atomic_rmw!(__sync_fetch_and_add_4, u32, |a: u32, b: u32| a.wrapping_add(b));
|
||||||
|
|
||||||
|
atomic_rmw!(__sync_fetch_and_sub_1, u8, |a: u8, b: u8| a.wrapping_sub(b));
|
||||||
|
atomic_rmw!(__sync_fetch_and_sub_2, u16, |a: u16, b: u16| a.wrapping_sub(b));
|
||||||
|
atomic_rmw!(__sync_fetch_and_sub_4, u32, |a: u32, b: u32| a.wrapping_sub(b));
|
||||||
|
|
||||||
|
atomic_rmw!(__sync_fetch_and_and_1, u8, |a: u8, b: u8| a & b);
|
||||||
|
atomic_rmw!(__sync_fetch_and_and_2, u16, |a: u16, b: u16| a & b);
|
||||||
|
atomic_rmw!(__sync_fetch_and_and_4, u32, |a: u32, b: u32| a & b);
|
||||||
|
|
||||||
|
atomic_rmw!(__sync_fetch_and_or_1, u8, |a: u8, b: u8| a | b);
|
||||||
|
atomic_rmw!(__sync_fetch_and_or_2, u16, |a: u16, b: u16| a | b);
|
||||||
|
atomic_rmw!(__sync_fetch_and_or_4, u32, |a: u32, b: u32| a | b);
|
||||||
|
|
||||||
|
atomic_rmw!(__sync_fetch_and_xor_1, u8, |a: u8, b: u8| a ^ b);
|
||||||
|
atomic_rmw!(__sync_fetch_and_xor_2, u16, |a: u16, b: u16| a ^ b);
|
||||||
|
atomic_rmw!(__sync_fetch_and_xor_4, u32, |a: u32, b: u32| a ^ b);
|
||||||
|
|
||||||
|
atomic_rmw!(__sync_fetch_and_nand_1, u8, |a: u8, b: u8| !a & b);
|
||||||
|
atomic_rmw!(__sync_fetch_and_nand_2, u16, |a: u16, b: u16| !a & b);
|
||||||
|
atomic_rmw!(__sync_fetch_and_nand_4, u32, |a: u32, b: u32| !a & b);
|
||||||
|
|
||||||
|
atomic_rmw!(__sync_fetch_and_max_1, i8, |a: i8, b: i8| if a > b { a } else { b });
|
||||||
|
atomic_rmw!(__sync_fetch_and_max_2, i16, |a: i16, b: i16| if a > b { a } else { b });
|
||||||
|
atomic_rmw!(__sync_fetch_and_max_4, i32, |a: i32, b: i32| if a > b { a } else { b });
|
||||||
|
|
||||||
|
atomic_rmw!(__sync_fetch_and_umax_1, u8, |a: u8, b: u8| if a > b { a } else { b });
|
||||||
|
atomic_rmw!(__sync_fetch_and_umax_2, u16, |a: u16, b: u16| if a > b { a } else { b });
|
||||||
|
atomic_rmw!(__sync_fetch_and_umax_4, u32, |a: u32, b: u32| if a > b { a } else { b });
|
||||||
|
|
||||||
|
atomic_rmw!(__sync_fetch_and_min_1, i8, |a: i8, b: i8| if a < b { a } else { b });
|
||||||
|
atomic_rmw!(__sync_fetch_and_min_2, i16, |a: i16, b: i16| if a < b { a } else { b });
|
||||||
|
atomic_rmw!(__sync_fetch_and_min_4, i32, |a: i32, b: i32| if a < b { a } else { b });
|
||||||
|
|
||||||
|
atomic_rmw!(__sync_fetch_and_umin_1, u8, |a: u8, b: u8| if a < b { a } else { b });
|
||||||
|
atomic_rmw!(__sync_fetch_and_umin_2, u16, |a: u16, b: u16| if a < b { a } else { b });
|
||||||
|
atomic_rmw!(__sync_fetch_and_umin_4, u32, |a: u32, b: u32| if a < b { a } else { b });
|
||||||
|
|
||||||
|
atomic_rmw!(__sync_lock_test_and_set_1, u8, |_: u8, b: u8| b);
|
||||||
|
atomic_rmw!(__sync_lock_test_and_set_2, u16, |_: u16, b: u16| b);
|
||||||
|
atomic_rmw!(__sync_lock_test_and_set_4, u32, |_: u32, b: u32| b);
|
||||||
|
|
||||||
|
atomic_cmpxchg!(__sync_val_compare_and_swap_1, u8);
|
||||||
|
atomic_cmpxchg!(__sync_val_compare_and_swap_2, u16);
|
||||||
|
atomic_cmpxchg!(__sync_val_compare_and_swap_4, u32);
|
||||||
|
|
||||||
|
#[cfg_attr(not(feature = "mangled-names"), no_mangle)]
|
||||||
|
pub unsafe extern "C" fn __sync_synchronize() {
|
||||||
|
__kuser_memory_barrier();
|
||||||
|
}
|
|
@ -51,6 +51,9 @@ pub mod mem;
|
||||||
#[cfg(target_arch = "arm")]
|
#[cfg(target_arch = "arm")]
|
||||||
pub mod arm;
|
pub mod arm;
|
||||||
|
|
||||||
|
#[cfg(all(armv5te, target_os = "linux", target_arch = "arm"))]
|
||||||
|
pub mod arm_linux;
|
||||||
|
|
||||||
#[cfg(target_arch = "x86")]
|
#[cfg(target_arch = "x86")]
|
||||||
pub mod x86;
|
pub mod x86;
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue