Aaron Kutch
11 months ago
committed by
GitHub
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 206 additions and 89 deletions

143src/int/leading_zeros.rs

69src/int/mod.rs

6testcrate/Cargo.toml

23testcrate/tests/count_leading_zeros.rs

54testcrate/tests/leading_zeros.rs
@ 0,0 +1,143 @@ 

// Note: these functions happen to produce the correct `usize::leading_zeros(0)` value


// without a explicit zero check. Zero is probably common enough that it could warrant


// adding a zero check at the beginning, but `__clzsi2` has a precondition that `x != 0`.


// Compilers will insert the check for zero in cases where it is needed.




/// Returns the number of leading binary zeros in `x`.


pub fn usize_leading_zeros_default(x: usize) > usize {


// The basic idea is to test if the higher bits of `x` are zero and bisect the number


// of leading zeros. It is possible for all branches of the bisection to use the same


// code path by conditionally shifting the higher parts down to let the next bisection


// step work on the higher or lower parts of `x`. Instead of starting with `z == 0`


// and adding to the number of zeros, it is slightly faster to start with


// `z == usize::MAX.count_ones()` and subtract from the potential number of zeros,


// because it simplifies the final bisection step.


let mut x = x;


// the number of potential leading zeros


let mut z = usize::MAX.count_ones() as usize;


// a temporary


let mut t: usize;


#[cfg(target_pointer_width = "64")]


{


t = x >> 32;


if t != 0 {


z = 32;


x = t;


}


}


#[cfg(any(target_pointer_width = "32", target_pointer_width = "64"))]


{


t = x >> 16;


if t != 0 {


z = 16;


x = t;


}


}


t = x >> 8;


if t != 0 {


z = 8;


x = t;


}


t = x >> 4;


if t != 0 {


z = 4;


x = t;


}


t = x >> 2;


if t != 0 {


z = 2;


x = t;


}


// the last two bisections are combined into one conditional


t = x >> 1;


if t != 0 {


z  2


} else {


z  x


}




// We could potentially save a few cycles by using the LUT trick from


// "https://embeddedgurus.com/statespace/2014/09/


// fastdeterministicandportablecountingleadingzeros/".


// However, 256 bytes for a LUT is too large for embedded use cases. We could remove


// the last 3 bisections and use this 16 byte LUT for the rest of the work:


//const LUT: [u8; 16] = [0, 1, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4];


//z = LUT[x] as usize;


//z


// However, it ends up generating about the same number of instructions. When benchmarked


// on x86_64, it is slightly faster to use the LUT, but this is probably because of OOO


// execution effects. Changing to using a LUT and branching is risky for smaller cores.


}




// The above method does not compile well on RISCV (because of the lack of predicated


// instructions), producing code with many branches or using an excessively long


// branchless solution. This method takes advantage of the setiflessthan instruction on


// RISCV that allows `(x >= poweroftwo) as usize` to be branchless.




/// Returns the number of leading binary zeros in `x`.


pub fn usize_leading_zeros_riscv(x: usize) > usize {


let mut x = x;


// the number of potential leading zeros


let mut z = usize::MAX.count_ones() as usize;


// a temporary


let mut t: usize;




// RISCV does not have a setifgreaterthanorequal instruction and


// `(x >= poweroftwo) as usize` will get compiled into two instructions, but this is


// still the most optimal method. A conditional set can only be turned into a single


// immediate instruction if `x` is compared with an immediate `imm` (that can fit into


// 12 bits) like `x < imm` but not `imm < x` (because the immediate is always on the


// right). If we try to save an instruction by using `x < imm` for each bisection, we


// have to shift `x` left and compare with powers of two approaching `usize::MAX + 1`,


// but the immediate will never fit into 12 bits and never save an instruction.


#[cfg(target_pointer_width = "64")]


{


// If the upper 32 bits of `x` are not all 0, `t` is set to `1 << 5`, otherwise


// `t` is set to 0.


t = ((x >= (1 << 32)) as usize) << 5;


// If `t` was set to `1 << 5`, then the upper 32 bits are shifted down for the


// next step to process.


x >>= t;


// If `t` was set to `1 << 5`, then we subtract 32 from the number of potential


// leading zeros


z = t;


}


#[cfg(any(target_pointer_width = "32", target_pointer_width = "64"))]


{


t = ((x >= (1 << 16)) as usize) << 4;


x >>= t;


z = t;


}


t = ((x >= (1 << 8)) as usize) << 3;


x >>= t;


z = t;


t = ((x >= (1 << 4)) as usize) << 2;


x >>= t;


z = t;


t = ((x >= (1 << 2)) as usize) << 1;


x >>= t;


z = t;


t = (x >= (1 << 1)) as usize;


x >>= t;


z = t;


// All bits except the LSB are guaranteed to be zero for this final bisection step.


// If `x != 0` then `x == 1` and subtracts one potential zero from `z`.


z  x


}




intrinsics! {


#[maybe_use_optimized_c_shim]


#[cfg(any(


target_pointer_width = "16",


target_pointer_width = "32",


target_pointer_width = "64"


))]


/// Returns the number of leading binary zeros in `x`.


pub extern "C" fn __clzsi2(x: usize) > usize {


if cfg!(any(target_arch = "riscv32", target_arch = "riscv64")) {


usize_leading_zeros_riscv(x)


} else {


usize_leading_zeros_default(x)


}


}


}

@ 1,23 +0,0 @@ 

extern crate compiler_builtins;




use compiler_builtins::int::__clzsi2;




#[test]


fn __clzsi2_test() {


let mut i: usize = core::usize::MAX;


// Check all values above 0


while i > 0 {


assert_eq!(__clzsi2(i) as u32, i.leading_zeros());


i >>= 1;


}


// check 0 also


i = 0;


assert_eq!(__clzsi2(i) as u32, i.leading_zeros());


// double check for bit patterns that aren't just solid 1s


i = 1;


for _ in 0..63 {


assert_eq!(__clzsi2(i) as u32, i.leading_zeros());


i <<= 2;


i += 1;


}


}

@ 0,0 +1,54 @@ 

use rand_xoshiro::rand_core::{RngCore, SeedableRng};


use rand_xoshiro::Xoshiro128StarStar;




use compiler_builtins::int::__clzsi2;


use compiler_builtins::int::leading_zeros::{


usize_leading_zeros_default, usize_leading_zeros_riscv,


};




#[test]


fn __clzsi2_test() {


// Binary fuzzer. We cannot just send a random number directly to `__clzsi2()`, because we need


// large sequences of zeros to test. This XORs, ANDs, and ORs random length strings of 1s to


// `x`. ORs insure sequences of ones, ANDs insures sequences of zeros, and XORs are not often


// destructive but add entropy.


let mut rng = Xoshiro128StarStar::seed_from_u64(0);


let mut x = 0usize;


// creates a mask for indexing the bits of the type


let bit_indexing_mask = usize::MAX.count_ones()  1;


// 10000 iterations is enough to make sure edge cases like single set bits are tested and to go


// through many paths.


for _ in 0..10_000 {


let r0 = bit_indexing_mask & rng.next_u32();


// random length of ones


let ones: usize = !0 >> r0;


let r1 = bit_indexing_mask & rng.next_u32();


// random circular shift


let mask = ones.rotate_left(r1);


match rng.next_u32() % 4 {


0 => x = mask,


1 => x &= mask,


// both 2 and 3 to make XORs as common as ORs and ANDs combined


_ => x ^= mask,


}


let lz = x.leading_zeros() as usize;


let lz0 = __clzsi2(x);


let lz1 = usize_leading_zeros_default(x);


let lz2 = usize_leading_zeros_riscv(x);


if lz0 != lz {


panic!("__clzsi2({}): expected: {}, found: {}", x, lz, lz0);


}


if lz1 != lz {


panic!(


"usize_leading_zeros_default({}): expected: {}, found: {}",


x, lz, lz1


);


}


if lz2 != lz {


panic!(


"usize_leading_zeros_riscv({}): expected: {}, found: {}",


x, lz, lz2


);


}


}


}

Write
Preview
Loading…
Cancel
Save
Reference in new issue