More inlining.

This commit is contained in:
Sébastien Crozet 2021-08-04 11:19:57 +02:00
parent 6d57396a42
commit 65b299557c
6 changed files with 16 additions and 46 deletions

View File

@ -30,7 +30,7 @@ use std::mem;
* *
*/ */
/// A array-based statically sized matrix data storage. /// A array-based statically sized matrix data storage.
#[repr(C)] #[repr(transparent)]
#[derive(Copy, Clone, PartialEq, Eq, Hash)] #[derive(Copy, Clone, PartialEq, Eq, Hash)]
pub struct ArrayStorage<T, const R: usize, const C: usize>(pub [[T; R]; C]); pub struct ArrayStorage<T, const R: usize, const C: usize>(pub [[T; R]; C]);

View File

@ -20,7 +20,7 @@ use crate::base::constraint::{
}; };
use crate::base::dimension::{Dim, Dynamic, U1}; use crate::base::dimension::{Dim, Dynamic, U1};
use crate::base::storage::{RawStorage, RawStorageMut}; use crate::base::storage::{RawStorage, RawStorageMut};
use crate::base::uninit::{InitStatus, Initialized}; use crate::base::uninit::InitStatus;
use crate::base::{Matrix, Scalar, Vector}; use crate::base::{Matrix, Scalar, Vector};
use std::any::TypeId; use std::any::TypeId;
@ -79,8 +79,8 @@ fn array_axc<Status, T>(
/// If `b` is zero, `y` is never read from and may be uninitialized. /// If `b` is zero, `y` is never read from and may be uninitialized.
/// ///
/// # Safety /// # Safety
/// This is UB if `Status == Uninit && b != 0`. /// This is UB if b != 0 and any component of `y` is uninitialized.
#[inline] #[inline(always)]
#[allow(clippy::many_single_char_names)] #[allow(clippy::many_single_char_names)]
pub unsafe fn axcpy_uninit<Status, T, D1: Dim, D2: Dim, SA, SB>( pub unsafe fn axcpy_uninit<Status, T, D1: Dim, D2: Dim, SA, SB>(
status: Status, status: Status,
@ -119,8 +119,8 @@ pub unsafe fn axcpy_uninit<Status, T, D1: Dim, D2: Dim, SA, SB>(
/// If `beta` is zero, `y` is never read from and may be uninitialized. /// If `beta` is zero, `y` is never read from and may be uninitialized.
/// ///
/// # Safety /// # Safety
/// This is UB if `Status == Uninit && beta != 0`. /// This is UB if beta != 0 and any component of `y` is uninitialized.
#[inline] #[inline(always)]
pub unsafe fn gemv_uninit<Status, T, D1: Dim, R2: Dim, C2: Dim, D3: Dim, SA, SB, SC>( pub unsafe fn gemv_uninit<Status, T, D1: Dim, R2: Dim, C2: Dim, D3: Dim, SA, SB, SC>(
status: Status, status: Status,
y: &mut Vector<Status::Value, D1, SA>, y: &mut Vector<Status::Value, D1, SA>,
@ -166,15 +166,8 @@ pub unsafe fn gemv_uninit<Status, T, D1: Dim, R2: Dim, C2: Dim, D3: Dim, SA, SB,
let col2 = a.column(j); let col2 = a.column(j);
let val = x.vget_unchecked(j).inlined_clone(); let val = x.vget_unchecked(j).inlined_clone();
// SAFETY: because y was initialized above, we can use the initialized status. // SAFETY: safe because y was initialized above.
axcpy_uninit( axcpy_uninit(status, y, alpha.inlined_clone(), &col2, val, T::one());
Initialized(status),
y,
alpha.inlined_clone(),
&col2,
val,
T::one(),
);
} }
} }
@ -184,8 +177,8 @@ pub unsafe fn gemv_uninit<Status, T, D1: Dim, R2: Dim, C2: Dim, D3: Dim, SA, SB,
/// If `beta` is zero, `y` is never read from and may be uninitialized. /// If `beta` is zero, `y` is never read from and may be uninitialized.
/// ///
/// # Safety /// # Safety
/// This is UB if `Status == Uninit && beta != 0`. /// This is UB if beta != 0 and any component of `y` is uninitialized.
#[inline] #[inline(always)]
pub unsafe fn gemm_uninit< pub unsafe fn gemm_uninit<
Status, Status,
T, T,

View File

@ -32,6 +32,7 @@ where
DefaultAllocator: Allocator<T, R, C>, DefaultAllocator: Allocator<T, R, C>,
{ {
/// Builds a matrix with uninitialized elements of type `MaybeUninit<T>`. /// Builds a matrix with uninitialized elements of type `MaybeUninit<T>`.
#[inline(always)]
pub fn uninit(nrows: R, ncols: C) -> Self { pub fn uninit(nrows: R, ncols: C) -> Self {
// SAFETY: this is OK because the dimension automatically match the storage // SAFETY: this is OK because the dimension automatically match the storage
// because we are building an owned storage. // because we are building an owned storage.

View File

@ -38,19 +38,19 @@ impl<T: Scalar, const R: usize, const C: usize> Allocator<T, Const<R>, Const<C>>
type Buffer = ArrayStorage<T, R, C>; type Buffer = ArrayStorage<T, R, C>;
type BufferUninit = ArrayStorage<MaybeUninit<T>, R, C>; type BufferUninit = ArrayStorage<MaybeUninit<T>, R, C>;
#[inline] #[inline(always)]
fn allocate_uninit(_: Const<R>, _: Const<C>) -> ArrayStorage<MaybeUninit<T>, R, C> { fn allocate_uninit(_: Const<R>, _: Const<C>) -> ArrayStorage<MaybeUninit<T>, R, C> {
// SAFETY: An uninitialized `[MaybeUninit<_>; _]` is valid. // SAFETY: An uninitialized `[MaybeUninit<_>; _]` is valid.
let array: [[MaybeUninit<T>; R]; C] = unsafe { MaybeUninit::uninit().assume_init() }; let array: [[MaybeUninit<T>; R]; C] = unsafe { MaybeUninit::uninit().assume_init() };
ArrayStorage(array) ArrayStorage(array)
} }
#[inline] #[inline(always)]
unsafe fn assume_init(uninit: ArrayStorage<MaybeUninit<T>, R, C>) -> ArrayStorage<T, R, C> { unsafe fn assume_init(uninit: ArrayStorage<MaybeUninit<T>, R, C>) -> ArrayStorage<T, R, C> {
// Safety: // Safety:
// * The caller guarantees that all elements of the array are initialized // * The caller guarantees that all elements of the array are initialized
// * `MaybeUninit<T>` and T are guaranteed to have the same layout // * `MaybeUninit<T>` and T are guaranteed to have the same layout
// * `MaybeUnint` does not drop, so there are no double-frees // * `MaybeUninit` does not drop, so there are no double-frees
// And thus the conversion is safe // And thus the conversion is safe
ArrayStorage((&uninit as *const _ as *const [_; C]).read()) ArrayStorage((&uninit as *const _ as *const [_; C]).read())
} }

View File

@ -422,6 +422,7 @@ where
/// # Safety /// # Safety
/// The user must make sure that every single entry of the buffer has been initialized, /// The user must make sure that every single entry of the buffer has been initialized,
/// or Undefined Behavior will immediately occur. /// or Undefined Behavior will immediately occur.
#[inline(always)]
pub unsafe fn assume_init(self) -> OMatrix<T, R, C> { pub unsafe fn assume_init(self) -> OMatrix<T, R, C> {
OMatrix::from_data(<DefaultAllocator as Allocator<T, R, C>>::assume_init( OMatrix::from_data(<DefaultAllocator as Allocator<T, R, C>>::assume_init(
self.data, self.data,

View File

@ -36,10 +36,6 @@ pub struct Init;
#[derive(Copy, Clone, Debug, PartialEq, Eq)] #[derive(Copy, Clone, Debug, PartialEq, Eq)]
/// A type implementing `InitStatus` indicating that the value is completely unitialized. /// A type implementing `InitStatus` indicating that the value is completely unitialized.
pub struct Uninit; pub struct Uninit;
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
/// A type implementing `InitStatus` indicating that the value is initialized even if the value
/// has the type `MaybeUninit` (i.e. when `Status == Uninit`).
pub struct Initialized<Status>(pub Status);
unsafe impl<T> InitStatus<T> for Init { unsafe impl<T> InitStatus<T> for Init {
type Value = T; type Value = T;
@ -78,24 +74,3 @@ unsafe impl<T> InitStatus<T> for Uninit {
std::mem::transmute(t.as_mut_ptr()) // TODO: use t.assume_init_mut() std::mem::transmute(t.as_mut_ptr()) // TODO: use t.assume_init_mut()
} }
} }
unsafe impl<T, Status: InitStatus<T>> InitStatus<T> for Initialized<Status> {
type Value = Status::Value;
#[inline(always)]
fn init(out: &mut Status::Value, t: T) {
unsafe {
*Status::assume_init_mut(out) = t;
}
}
#[inline(always)]
unsafe fn assume_init_ref(t: &Status::Value) -> &T {
Status::assume_init_ref(t)
}
#[inline(always)]
unsafe fn assume_init_mut(t: &mut Status::Value) -> &mut T {
Status::assume_init_mut(t)
}
}