More inlining.
This commit is contained in:
parent
6d57396a42
commit
65b299557c
|
@ -30,7 +30,7 @@ use std::mem;
|
||||||
*
|
*
|
||||||
*/
|
*/
|
||||||
/// A array-based statically sized matrix data storage.
|
/// A array-based statically sized matrix data storage.
|
||||||
#[repr(C)]
|
#[repr(transparent)]
|
||||||
#[derive(Copy, Clone, PartialEq, Eq, Hash)]
|
#[derive(Copy, Clone, PartialEq, Eq, Hash)]
|
||||||
pub struct ArrayStorage<T, const R: usize, const C: usize>(pub [[T; R]; C]);
|
pub struct ArrayStorage<T, const R: usize, const C: usize>(pub [[T; R]; C]);
|
||||||
|
|
||||||
|
|
|
@ -20,7 +20,7 @@ use crate::base::constraint::{
|
||||||
};
|
};
|
||||||
use crate::base::dimension::{Dim, Dynamic, U1};
|
use crate::base::dimension::{Dim, Dynamic, U1};
|
||||||
use crate::base::storage::{RawStorage, RawStorageMut};
|
use crate::base::storage::{RawStorage, RawStorageMut};
|
||||||
use crate::base::uninit::{InitStatus, Initialized};
|
use crate::base::uninit::InitStatus;
|
||||||
use crate::base::{Matrix, Scalar, Vector};
|
use crate::base::{Matrix, Scalar, Vector};
|
||||||
use std::any::TypeId;
|
use std::any::TypeId;
|
||||||
|
|
||||||
|
@ -79,8 +79,8 @@ fn array_axc<Status, T>(
|
||||||
/// If `b` is zero, `y` is never read from and may be uninitialized.
|
/// If `b` is zero, `y` is never read from and may be uninitialized.
|
||||||
///
|
///
|
||||||
/// # Safety
|
/// # Safety
|
||||||
/// This is UB if `Status == Uninit && b != 0`.
|
/// This is UB if b != 0 and any component of `y` is uninitialized.
|
||||||
#[inline]
|
#[inline(always)]
|
||||||
#[allow(clippy::many_single_char_names)]
|
#[allow(clippy::many_single_char_names)]
|
||||||
pub unsafe fn axcpy_uninit<Status, T, D1: Dim, D2: Dim, SA, SB>(
|
pub unsafe fn axcpy_uninit<Status, T, D1: Dim, D2: Dim, SA, SB>(
|
||||||
status: Status,
|
status: Status,
|
||||||
|
@ -119,8 +119,8 @@ pub unsafe fn axcpy_uninit<Status, T, D1: Dim, D2: Dim, SA, SB>(
|
||||||
/// If `beta` is zero, `y` is never read from and may be uninitialized.
|
/// If `beta` is zero, `y` is never read from and may be uninitialized.
|
||||||
///
|
///
|
||||||
/// # Safety
|
/// # Safety
|
||||||
/// This is UB if `Status == Uninit && beta != 0`.
|
/// This is UB if beta != 0 and any component of `y` is uninitialized.
|
||||||
#[inline]
|
#[inline(always)]
|
||||||
pub unsafe fn gemv_uninit<Status, T, D1: Dim, R2: Dim, C2: Dim, D3: Dim, SA, SB, SC>(
|
pub unsafe fn gemv_uninit<Status, T, D1: Dim, R2: Dim, C2: Dim, D3: Dim, SA, SB, SC>(
|
||||||
status: Status,
|
status: Status,
|
||||||
y: &mut Vector<Status::Value, D1, SA>,
|
y: &mut Vector<Status::Value, D1, SA>,
|
||||||
|
@ -166,15 +166,8 @@ pub unsafe fn gemv_uninit<Status, T, D1: Dim, R2: Dim, C2: Dim, D3: Dim, SA, SB,
|
||||||
let col2 = a.column(j);
|
let col2 = a.column(j);
|
||||||
let val = x.vget_unchecked(j).inlined_clone();
|
let val = x.vget_unchecked(j).inlined_clone();
|
||||||
|
|
||||||
// SAFETY: because y was initialized above, we can use the initialized status.
|
// SAFETY: safe because y was initialized above.
|
||||||
axcpy_uninit(
|
axcpy_uninit(status, y, alpha.inlined_clone(), &col2, val, T::one());
|
||||||
Initialized(status),
|
|
||||||
y,
|
|
||||||
alpha.inlined_clone(),
|
|
||||||
&col2,
|
|
||||||
val,
|
|
||||||
T::one(),
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -184,8 +177,8 @@ pub unsafe fn gemv_uninit<Status, T, D1: Dim, R2: Dim, C2: Dim, D3: Dim, SA, SB,
|
||||||
/// If `beta` is zero, `y` is never read from and may be uninitialized.
|
/// If `beta` is zero, `y` is never read from and may be uninitialized.
|
||||||
///
|
///
|
||||||
/// # Safety
|
/// # Safety
|
||||||
/// This is UB if `Status == Uninit && beta != 0`.
|
/// This is UB if beta != 0 and any component of `y` is uninitialized.
|
||||||
#[inline]
|
#[inline(always)]
|
||||||
pub unsafe fn gemm_uninit<
|
pub unsafe fn gemm_uninit<
|
||||||
Status,
|
Status,
|
||||||
T,
|
T,
|
||||||
|
|
|
@ -32,6 +32,7 @@ where
|
||||||
DefaultAllocator: Allocator<T, R, C>,
|
DefaultAllocator: Allocator<T, R, C>,
|
||||||
{
|
{
|
||||||
/// Builds a matrix with uninitialized elements of type `MaybeUninit<T>`.
|
/// Builds a matrix with uninitialized elements of type `MaybeUninit<T>`.
|
||||||
|
#[inline(always)]
|
||||||
pub fn uninit(nrows: R, ncols: C) -> Self {
|
pub fn uninit(nrows: R, ncols: C) -> Self {
|
||||||
// SAFETY: this is OK because the dimension automatically match the storage
|
// SAFETY: this is OK because the dimension automatically match the storage
|
||||||
// because we are building an owned storage.
|
// because we are building an owned storage.
|
||||||
|
|
|
@ -38,19 +38,19 @@ impl<T: Scalar, const R: usize, const C: usize> Allocator<T, Const<R>, Const<C>>
|
||||||
type Buffer = ArrayStorage<T, R, C>;
|
type Buffer = ArrayStorage<T, R, C>;
|
||||||
type BufferUninit = ArrayStorage<MaybeUninit<T>, R, C>;
|
type BufferUninit = ArrayStorage<MaybeUninit<T>, R, C>;
|
||||||
|
|
||||||
#[inline]
|
#[inline(always)]
|
||||||
fn allocate_uninit(_: Const<R>, _: Const<C>) -> ArrayStorage<MaybeUninit<T>, R, C> {
|
fn allocate_uninit(_: Const<R>, _: Const<C>) -> ArrayStorage<MaybeUninit<T>, R, C> {
|
||||||
// SAFETY: An uninitialized `[MaybeUninit<_>; _]` is valid.
|
// SAFETY: An uninitialized `[MaybeUninit<_>; _]` is valid.
|
||||||
let array: [[MaybeUninit<T>; R]; C] = unsafe { MaybeUninit::uninit().assume_init() };
|
let array: [[MaybeUninit<T>; R]; C] = unsafe { MaybeUninit::uninit().assume_init() };
|
||||||
ArrayStorage(array)
|
ArrayStorage(array)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[inline]
|
#[inline(always)]
|
||||||
unsafe fn assume_init(uninit: ArrayStorage<MaybeUninit<T>, R, C>) -> ArrayStorage<T, R, C> {
|
unsafe fn assume_init(uninit: ArrayStorage<MaybeUninit<T>, R, C>) -> ArrayStorage<T, R, C> {
|
||||||
// Safety:
|
// Safety:
|
||||||
// * The caller guarantees that all elements of the array are initialized
|
// * The caller guarantees that all elements of the array are initialized
|
||||||
// * `MaybeUninit<T>` and T are guaranteed to have the same layout
|
// * `MaybeUninit<T>` and T are guaranteed to have the same layout
|
||||||
// * `MaybeUnint` does not drop, so there are no double-frees
|
// * `MaybeUninit` does not drop, so there are no double-frees
|
||||||
// And thus the conversion is safe
|
// And thus the conversion is safe
|
||||||
ArrayStorage((&uninit as *const _ as *const [_; C]).read())
|
ArrayStorage((&uninit as *const _ as *const [_; C]).read())
|
||||||
}
|
}
|
||||||
|
|
|
@ -422,6 +422,7 @@ where
|
||||||
/// # Safety
|
/// # Safety
|
||||||
/// The user must make sure that every single entry of the buffer has been initialized,
|
/// The user must make sure that every single entry of the buffer has been initialized,
|
||||||
/// or Undefined Behavior will immediately occur.
|
/// or Undefined Behavior will immediately occur.
|
||||||
|
#[inline(always)]
|
||||||
pub unsafe fn assume_init(self) -> OMatrix<T, R, C> {
|
pub unsafe fn assume_init(self) -> OMatrix<T, R, C> {
|
||||||
OMatrix::from_data(<DefaultAllocator as Allocator<T, R, C>>::assume_init(
|
OMatrix::from_data(<DefaultAllocator as Allocator<T, R, C>>::assume_init(
|
||||||
self.data,
|
self.data,
|
||||||
|
|
|
@ -36,10 +36,6 @@ pub struct Init;
|
||||||
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
|
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
|
||||||
/// A type implementing `InitStatus` indicating that the value is completely unitialized.
|
/// A type implementing `InitStatus` indicating that the value is completely unitialized.
|
||||||
pub struct Uninit;
|
pub struct Uninit;
|
||||||
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
|
|
||||||
/// A type implementing `InitStatus` indicating that the value is initialized even if the value
|
|
||||||
/// has the type `MaybeUninit` (i.e. when `Status == Uninit`).
|
|
||||||
pub struct Initialized<Status>(pub Status);
|
|
||||||
|
|
||||||
unsafe impl<T> InitStatus<T> for Init {
|
unsafe impl<T> InitStatus<T> for Init {
|
||||||
type Value = T;
|
type Value = T;
|
||||||
|
@ -78,24 +74,3 @@ unsafe impl<T> InitStatus<T> for Uninit {
|
||||||
std::mem::transmute(t.as_mut_ptr()) // TODO: use t.assume_init_mut()
|
std::mem::transmute(t.as_mut_ptr()) // TODO: use t.assume_init_mut()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
unsafe impl<T, Status: InitStatus<T>> InitStatus<T> for Initialized<Status> {
|
|
||||||
type Value = Status::Value;
|
|
||||||
|
|
||||||
#[inline(always)]
|
|
||||||
fn init(out: &mut Status::Value, t: T) {
|
|
||||||
unsafe {
|
|
||||||
*Status::assume_init_mut(out) = t;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[inline(always)]
|
|
||||||
unsafe fn assume_init_ref(t: &Status::Value) -> &T {
|
|
||||||
Status::assume_init_ref(t)
|
|
||||||
}
|
|
||||||
|
|
||||||
#[inline(always)]
|
|
||||||
unsafe fn assume_init_mut(t: &mut Status::Value) -> &mut T {
|
|
||||||
Status::assume_init_mut(t)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
Loading…
Reference in New Issue