From 65b299557c23702c2c772e82640b5642e56bde98 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?S=C3=A9bastien=20Crozet?= <developer@crozet.re>
Date: Wed, 4 Aug 2021 11:19:57 +0200
Subject: [PATCH] More inlining.

---
 src/base/array_storage.rs     |  2 +-
 src/base/blas_uninit.rs       | 25 +++++++++----------------
 src/base/construction.rs      |  1 +
 src/base/default_allocator.rs |  6 +++---
 src/base/matrix.rs            |  3 ++-
 src/base/uninit.rs            | 25 -------------------------
 6 files changed, 16 insertions(+), 46 deletions(-)

diff --git a/src/base/array_storage.rs b/src/base/array_storage.rs
index 7b2bb799..3fc88ade 100644
--- a/src/base/array_storage.rs
+++ b/src/base/array_storage.rs
@@ -30,7 +30,7 @@ use std::mem;
  *
  */
 /// A array-based statically sized matrix data storage.
-#[repr(C)]
+#[repr(transparent)]
 #[derive(Copy, Clone, PartialEq, Eq, Hash)]
 pub struct ArrayStorage<T, const R: usize, const C: usize>(pub [[T; R]; C]);
 
diff --git a/src/base/blas_uninit.rs b/src/base/blas_uninit.rs
index a50ec97d..6f4fde7b 100644
--- a/src/base/blas_uninit.rs
+++ b/src/base/blas_uninit.rs
@@ -20,7 +20,7 @@ use crate::base::constraint::{
 };
 use crate::base::dimension::{Dim, Dynamic, U1};
 use crate::base::storage::{RawStorage, RawStorageMut};
-use crate::base::uninit::{InitStatus, Initialized};
+use crate::base::uninit::InitStatus;
 use crate::base::{Matrix, Scalar, Vector};
 use std::any::TypeId;
 
@@ -79,8 +79,8 @@ fn array_axc<Status, T>(
 /// If `b` is zero, `y` is never read from and may be uninitialized.
 ///
 /// # Safety
-/// This is UB if `Status == Uninit && b != 0`.
-#[inline]
+/// This is UB if b != 0 and any component of `y` is uninitialized.
+#[inline(always)]
 #[allow(clippy::many_single_char_names)]
 pub unsafe fn axcpy_uninit<Status, T, D1: Dim, D2: Dim, SA, SB>(
     status: Status,
@@ -119,8 +119,8 @@ pub unsafe fn axcpy_uninit<Status, T, D1: Dim, D2: Dim, SA, SB>(
 /// If `beta` is zero, `y` is never read from and may be uninitialized.
 ///
 /// # Safety
-/// This is UB if `Status == Uninit && beta != 0`.
-#[inline]
+/// This is UB if beta != 0 and any component of `y` is uninitialized.
+#[inline(always)]
 pub unsafe fn gemv_uninit<Status, T, D1: Dim, R2: Dim, C2: Dim, D3: Dim, SA, SB, SC>(
     status: Status,
     y: &mut Vector<Status::Value, D1, SA>,
@@ -166,15 +166,8 @@ pub unsafe fn gemv_uninit<Status, T, D1: Dim, R2: Dim, C2: Dim, D3: Dim, SA, SB,
         let col2 = a.column(j);
         let val = x.vget_unchecked(j).inlined_clone();
 
-        // SAFETY: because y was initialized above, we can use the initialized status.
-        axcpy_uninit(
-            Initialized(status),
-            y,
-            alpha.inlined_clone(),
-            &col2,
-            val,
-            T::one(),
-        );
+        // SAFETY: safe because y was initialized above.
+        axcpy_uninit(status, y, alpha.inlined_clone(), &col2, val, T::one());
     }
 }
 
@@ -184,8 +177,8 @@ pub unsafe fn gemv_uninit<Status, T, D1: Dim, R2: Dim, C2: Dim, D3: Dim, SA, SB,
 /// If `beta` is zero, `y` is never read from and may be uninitialized.
 ///
 /// # Safety
-/// This is UB if `Status == Uninit && beta != 0`.
-#[inline]
+/// This is UB if beta != 0 and any component of `y` is uninitialized.
+#[inline(always)]
 pub unsafe fn gemm_uninit<
     Status,
     T,
diff --git a/src/base/construction.rs b/src/base/construction.rs
index 0806743c..3deb66c2 100644
--- a/src/base/construction.rs
+++ b/src/base/construction.rs
@@ -32,6 +32,7 @@ where
     DefaultAllocator: Allocator<T, R, C>,
 {
     /// Builds a matrix with uninitialized elements of type `MaybeUninit<T>`.
+    #[inline(always)]
     pub fn uninit(nrows: R, ncols: C) -> Self {
         // SAFETY: this is OK because the dimension automatically match the storage
         //         because we are building an owned storage.
diff --git a/src/base/default_allocator.rs b/src/base/default_allocator.rs
index 23c80153..b676b5e3 100644
--- a/src/base/default_allocator.rs
+++ b/src/base/default_allocator.rs
@@ -38,19 +38,19 @@ impl<T: Scalar, const R: usize, const C: usize> Allocator<T, Const<R>, Const<C>>
     type Buffer = ArrayStorage<T, R, C>;
     type BufferUninit = ArrayStorage<MaybeUninit<T>, R, C>;
 
-    #[inline]
+    #[inline(always)]
     fn allocate_uninit(_: Const<R>, _: Const<C>) -> ArrayStorage<MaybeUninit<T>, R, C> {
         // SAFETY: An uninitialized `[MaybeUninit<_>; _]` is valid.
         let array: [[MaybeUninit<T>; R]; C] = unsafe { MaybeUninit::uninit().assume_init() };
         ArrayStorage(array)
     }
 
-    #[inline]
+    #[inline(always)]
     unsafe fn assume_init(uninit: ArrayStorage<MaybeUninit<T>, R, C>) -> ArrayStorage<T, R, C> {
         // Safety:
         // * The caller guarantees that all elements of the array are initialized
         // * `MaybeUninit<T>` and T are guaranteed to have the same layout
-        // * `MaybeUnint` does not drop, so there are no double-frees
+        // * `MaybeUninit` does not drop, so there are no double-frees
         // And thus the conversion is safe
         ArrayStorage((&uninit as *const _ as *const [_; C]).read())
     }
diff --git a/src/base/matrix.rs b/src/base/matrix.rs
index 392ea343..ce5f2f18 100644
--- a/src/base/matrix.rs
+++ b/src/base/matrix.rs
@@ -421,7 +421,8 @@ where
     ///
     /// # Safety
     /// The user must make sure that every single entry of the buffer has been initialized,
-    /// or Undefined Behavior will immediately occur.    
+    /// or Undefined Behavior will immediately occur.
+    #[inline(always)]
     pub unsafe fn assume_init(self) -> OMatrix<T, R, C> {
         OMatrix::from_data(<DefaultAllocator as Allocator<T, R, C>>::assume_init(
             self.data,
diff --git a/src/base/uninit.rs b/src/base/uninit.rs
index 5d37a249..92d246df 100644
--- a/src/base/uninit.rs
+++ b/src/base/uninit.rs
@@ -36,10 +36,6 @@ pub struct Init;
 #[derive(Copy, Clone, Debug, PartialEq, Eq)]
 /// A type implementing `InitStatus` indicating that the value is completely unitialized.
 pub struct Uninit;
-#[derive(Copy, Clone, Debug, PartialEq, Eq)]
-/// A type implementing `InitStatus` indicating that the value is initialized even if the value
-/// has the type `MaybeUninit` (i.e. when `Status == Uninit`).
-pub struct Initialized<Status>(pub Status);
 
 unsafe impl<T> InitStatus<T> for Init {
     type Value = T;
@@ -78,24 +74,3 @@ unsafe impl<T> InitStatus<T> for Uninit {
         std::mem::transmute(t.as_mut_ptr()) // TODO: use t.assume_init_mut()
     }
 }
-
-unsafe impl<T, Status: InitStatus<T>> InitStatus<T> for Initialized<Status> {
-    type Value = Status::Value;
-
-    #[inline(always)]
-    fn init(out: &mut Status::Value, t: T) {
-        unsafe {
-            *Status::assume_init_mut(out) = t;
-        }
-    }
-
-    #[inline(always)]
-    unsafe fn assume_init_ref(t: &Status::Value) -> &T {
-        Status::assume_init_ref(t)
-    }
-
-    #[inline(always)]
-    unsafe fn assume_init_mut(t: &mut Status::Value) -> &mut T {
-        Status::assume_init_mut(t)
-    }
-}