diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..7ed41da --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "sfkv" +version = "0.1.0" +description = "Simple Flash Key Value store" +documentation = "https://docs.rs/sfkv/" +homepage = "https://git.m-labs.hk/M-Labs/sfkv" +repository = "https://git.m-labs.hk/M-Labs/sfkv.git" +readme = "README.md" +keywords = ["flash", "config"] +categories = ["embedded", "database"] +license = "GPL-3" +edition = "2018" + +[dependencies] +byteorder = { version = "1.0", default-features = false } diff --git a/src/error.rs b/src/error.rs new file mode 100644 index 0000000..fd0f97d --- /dev/null +++ b/src/error.rs @@ -0,0 +1,59 @@ +use core::{fmt, str}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Error { + Read(ReadError), + Write(WriteError), +} + +impl From for Error { + fn from(e: ReadError) -> Self { + Error::Read(e) + } +} + +impl From> for Error { + fn from(e: WriteError) -> Self { + Error::Write(e) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ReadError { + Truncated { offset: usize }, + InvalidSize { offset: usize, size: usize }, + MissingSeparator { offset: usize }, + Utf8Error(str::Utf8Error), +} + +impl fmt::Display for ReadError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + &ReadError::Truncated { offset }=> + write!(f, "truncated record at offset {}", offset), + &ReadError::InvalidSize { offset, size } => + write!(f, "invalid record size {} at offset {}", size, offset), + &ReadError::MissingSeparator { offset } => + write!(f, "missing separator at offset {}", offset), + &ReadError::Utf8Error(err) => + write!(f, "{}", err), + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum WriteError { + SpaceExhausted, + Backend(B), +} + +impl fmt::Display for WriteError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + &WriteError::SpaceExhausted => + write!(f, "space exhausted"), + &WriteError::Backend(ref e) => + e.fmt(f), + } + } +} diff --git a/src/fmt.rs b/src/fmt.rs new file mode 100644 index 0000000..dcb9ebb --- /dev/null +++ b/src/fmt.rs @@ -0,0 +1,30 @@ +use core::fmt; + +pub struct FmtWrapper<'a> { + buf: &'a mut [u8], + offset: usize, +} + +impl<'a> FmtWrapper<'a> { + pub fn new(buf: &'a mut [u8]) -> Self { + FmtWrapper { + buf: buf, + offset: 0, + } + } + + pub fn contents(&self) -> &[u8] { + &self.buf[..self.offset] + } +} + +impl<'a> fmt::Write for FmtWrapper<'a> { + fn write_str(&mut self, s: &str) -> fmt::Result { + let bytes = s.as_bytes(); + let remainder = &mut self.buf[self.offset..]; + let remainder = &mut remainder[..bytes.len()]; + remainder.copy_from_slice(bytes); + self.offset += bytes.len(); + Ok(()) + } +} diff --git a/src/iter.rs b/src/iter.rs new file mode 100644 index 0000000..743ad51 --- /dev/null +++ b/src/iter.rs @@ -0,0 +1,46 @@ +use byteorder::{ByteOrder, BigEndian}; +use crate::ReadError; + +#[derive(Clone)] +pub struct Iter<'a> { + data: &'a [u8], + pub offset: usize +} + +impl<'a> Iter<'a> { + pub fn new(data: &'a [u8]) -> Iter<'a> { + Iter { data: data, offset: 0 } + } +} + +impl<'a> Iterator for Iter<'a> { + type Item = Result<(&'a [u8], &'a [u8]), ReadError>; + + fn next(&mut self) -> Option { + let data = &self.data[self.offset..]; + + if data.len() < 4 { + return Some(Err(ReadError::Truncated { offset: self.offset })) + } + + let record_size = BigEndian::read_u32(data) as usize; + if record_size == !0u32 as usize /* all ones; erased flash */ { + return None + } else if record_size < 4 || record_size > data.len() { + return Some(Err(ReadError::InvalidSize { offset: self.offset, size: record_size })) + } + + let record_body = &data[4..record_size]; + match record_body.iter().position(|&x| x == 0) { + None => { + return Some(Err(ReadError::MissingSeparator { offset: self.offset })) + } + Some(pos) => { + self.offset += record_size; + + let (key, zero_and_value) = record_body.split_at(pos); + Some(Ok((key, &zero_and_value[1..]))) + } + } + } +} diff --git a/src/lib.rs b/src/lib.rs index 7ed368e..299d93b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,222 +1,113 @@ -use core::{str, fmt}; +#![no_std] -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum Error { - AlreadyLocked, - SpaceExhausted, - Truncated { offset: usize }, - InvalidSize { offset: usize, size: usize }, - MissingSeparator { offset: usize }, - Utf8Error(str::Utf8Error), - NoFlash, +use core::{fmt::Write, mem::size_of, str}; +use byteorder::{ByteOrder, BigEndian}; + +mod error; +pub use error::{Error, ReadError, WriteError}; +mod iter; +use iter::Iter; +mod fmt; +use fmt::FmtWrapper; +mod test; + +pub trait StoreBackend { + type Data: Sized + AsRef<[u8]> + Clone; + + /// Memory-mapped + fn data(&self) -> &Self::Data; + fn len(&self) -> usize { + size_of::() + } + + /// utility for compaction, requires compile-time knowledge of the data length + + type Error; + fn erase(&mut self) -> Result<(), Self::Error>; + fn program(&mut self, offset: usize, payload: &[u8]) -> Result<(), Self::Error>; + /// called after many `program()` invocations to allow for eg. cache flushing + fn program_done(&mut self) {} } -impl fmt::Display for Error { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - match self { - &Error::AlreadyLocked => - write!(f, "attempt at reentrant access"), - &Error::SpaceExhausted => - write!(f, "space exhausted"), - &Error::Truncated { offset }=> - write!(f, "truncated record at offset {}", offset), - &Error::InvalidSize { offset, size } => - write!(f, "invalid record size {} at offset {}", size, offset), - &Error::MissingSeparator { offset } => - write!(f, "missing separator at offset {}", offset), - &Error::Utf8Error(err) => - write!(f, "{}", err), - &Error::NoFlash => - write!(f, "flash memory is not present"), - } - } +// TODO: NoFlash + +pub struct Store { + backend: B, } -#[cfg(has_spiflash)] -mod imp { - use core::str; - use byteorder::{ByteOrder, BigEndian}; - use cache; - use spiflash; - use super::Error; - use core::fmt; - use core::fmt::Write; - - struct FmtWrapper<'a> { - buf: &'a mut [u8], - offset: usize, +impl Store { + pub fn new(backend: B) -> Self { + Store { backend } } - impl<'a> FmtWrapper<'a> { - fn new(buf: &'a mut [u8]) -> Self { - FmtWrapper { - buf: buf, - offset: 0, + pub fn read(&self, key: &str) -> Result, ReadError> { + let mut iter = Iter::new(self.backend.data().as_ref()); + let mut value = None; + while let Some(result) = iter.next() { + let (record_key, record_value) = result?; + if key.as_bytes() == record_key { + // last write wins + value = Some(record_value) } } - - fn contents(&self) -> &[u8] { - &self.buf[..self.offset] - } + Ok(value) } - impl<'a> fmt::Write for FmtWrapper<'a> { - fn write_str(&mut self, s: &str) -> fmt::Result { - let bytes = s.as_bytes(); - let remainder = &mut self.buf[self.offset..]; - let remainder = &mut remainder[..bytes.len()]; - remainder.copy_from_slice(bytes); - self.offset += bytes.len(); - Ok(()) - } + pub fn read_str(&self, key: &str) -> Result, ReadError> { + self.read(key) + .and_then(|value| value + .map(|value| str::from_utf8(value) + .map_err(ReadError::Utf8Error) + ) + .transpose()) } - // One flash sector immediately before the firmware. - const ADDR: usize = ::mem::FLASH_BOOT_ADDRESS - spiflash::SECTOR_SIZE; - const SIZE: usize = spiflash::SECTOR_SIZE; - - mod lock { - use core::slice; - use core::sync::atomic::{AtomicUsize, Ordering, ATOMIC_USIZE_INIT}; - use super::Error; - - static LOCKED: AtomicUsize = ATOMIC_USIZE_INIT; - - pub struct Lock; - - impl Lock { - pub fn take() -> Result { - if LOCKED.swap(1, Ordering::SeqCst) != 0 { - Err(Error::AlreadyLocked) - } else { - Ok(Lock) - } - } - - pub fn data(&self) -> &'static [u8] { - unsafe { slice::from_raw_parts(super::ADDR as *const u8, super::SIZE) } - } - } - - impl Drop for Lock { - fn drop(&mut self) { - LOCKED.store(0, Ordering::SeqCst) - } - } - } - - use self::lock::Lock; - - #[derive(Clone)] - struct Iter<'a> { - data: &'a [u8], - offset: usize - } - - impl<'a> Iter<'a> { - fn new(data: &'a [u8]) -> Iter<'a> { - Iter { data: data, offset: 0 } - } - } - - impl<'a> Iterator for Iter<'a> { - type Item = Result<(&'a [u8], &'a [u8]), Error>; - - fn next(&mut self) -> Option { - let data = &self.data[self.offset..]; - - if data.len() < 4 { - // error!("offset {}: truncated record", self.offset); - return Some(Err(Error::Truncated { offset: self.offset })) - } - - let record_size = BigEndian::read_u32(data) as usize; - if record_size == !0 /* all ones; erased flash */ { - return None - } else if record_size < 4 || record_size > data.len() { - return Some(Err(Error::InvalidSize { offset: self.offset, size: record_size })) - } - - let record_body = &data[4..record_size]; - match record_body.iter().position(|&x| x == 0) { - None => { - return Some(Err(Error::MissingSeparator { offset: self.offset })) - } - Some(pos) => { - self.offset += record_size; - - let (key, zero_and_value) = record_body.split_at(pos); - Some(Ok((key, &zero_and_value[1..]))) - } - } - } - } - - pub fn read) -> R, R>(key: &str, f: F) -> R { - f(Lock::take().and_then(|lock| { - let mut iter = Iter::new(lock.data()); - let mut value = &[][..]; - while let Some(result) = iter.next() { - let (record_key, record_value) = result?; - if key.as_bytes() == record_key { - // last write wins - value = record_value - } - } - Ok(value) - })) - } - - pub fn read_str) -> R, R>(key: &str, f: F) -> R { - read(key, |result| { - f(result.and_then(|value| str::from_utf8(value).map_err(Error::Utf8Error))) - }) - } - - unsafe fn append_at(data: &[u8], mut offset: usize, - key: &[u8], value: &[u8]) -> Result { + unsafe fn append_at(&mut self, mut offset: usize, + key: &[u8], value: &[u8]) -> Result> { let record_size = 4 + key.len() + 1 + value.len(); - if offset + record_size > data.len() { - return Err(Error::SpaceExhausted) + if offset + record_size > self.backend.len() { + return Err(WriteError::SpaceExhausted) } let mut record_size_bytes = [0u8; 4]; BigEndian::write_u32(&mut record_size_bytes[..], record_size as u32); { - let mut write = |payload| { - spiflash::write(data.as_ptr().offset(offset as isize) as usize, payload); + let mut write = |payload| -> Result<(), WriteError> { + self.backend.program(offset, payload) + .map_err(WriteError::Backend)?; offset += payload.len(); + Ok(()) }; - write(&record_size_bytes[..]); - write(key); - write(&[0]); - write(value); - cache::flush_l2_cache(); + write(&record_size_bytes[..])?; + write(key)?; + write(&[0])?; + write(value)?; } + self.backend.program_done(); Ok(offset) } - fn compact() -> Result<(), Error> { - let lock = Lock::take()?; - let data = lock.data(); + fn compact(&mut self) -> Result<(), Error> { + // static mut OLD_DATA: B::Data; + // let mut old_data = MaybeUninit::::zeroed(); + // let old_data = unsafe { + // let mut old_data = old_data.assume_init(); + // old_data.copy_from_slice(self.backend.data().as_ref()); + // old_data + // }; + let old_data = self.backend.data().clone(); - static mut OLD_DATA: [u8; SIZE] = [0; SIZE]; - let old_data = unsafe { - OLD_DATA.copy_from_slice(data); - &OLD_DATA[..] - }; - - unsafe { spiflash::erase_sector(data.as_ptr() as usize) }; + self.erase()?; // This is worst-case quadratic, but we're limited by a small SPI flash sector size, // so it does not really matter. let mut offset = 0; - let mut iter = Iter::new(old_data); + let mut iter = Iter::new(old_data.as_ref()); 'iter: while let Some(result) = iter.next() { - let (key, mut value) = result?; + let (key, value) = result?; if value.is_empty() { // This is a removed entry, ignore it. continue @@ -230,84 +121,53 @@ mod imp { continue 'iter } } - offset = unsafe { append_at(data, offset, key, value)? }; + offset = unsafe { self.append_at(offset, key, value)? }; } Ok(()) } - fn append(key: &str, value: &[u8]) -> Result<(), Error> { - let lock = Lock::take()?; - let data = lock.data(); - + fn append(&mut self, key: &str, value: &[u8]) -> Result<(), Error> { let free_offset = { - let mut iter = Iter::new(data); + let mut iter = Iter::new(self.backend.data().as_ref()); while let Some(result) = iter.next() { let _ = result?; } iter.offset }; - unsafe { append_at(data, free_offset, key.as_bytes(), value)? }; + unsafe { self.append_at(free_offset, key.as_bytes(), value)? }; Ok(()) } - pub fn write(key: &str, value: &[u8]) -> Result<(), Error> { - match append(key, value) { - Err(Error::SpaceExhausted) => { - compact()?; - append(key, value) + pub fn write(&mut self, key: &str, value: &[u8]) -> Result<(), Error> { + match self.append(key, value) { + Err(Error::Write(WriteError::SpaceExhausted)) => { + self.compact()?; + self.append(key, value) } res => res } } - pub fn write_int(key: &str, value: u32) -> Result<(), Error> { + pub fn write_int(&mut self, key: &str, value: u32) -> Result<(), Error> { let mut buf = [0; 16]; let mut wrapper = FmtWrapper::new(&mut buf); write!(&mut wrapper, "{}", value).unwrap(); - write(key, wrapper.contents()) + self.write(key, wrapper.contents()) } - pub fn remove(key: &str) -> Result<(), Error> { - write(key, &[]) + pub fn remove(&mut self, key: &str) -> Result<(), Error> { + self.write(key, &[]) } - pub fn erase() -> Result<(), Error> { - let lock = Lock::take()?; - let data = lock.data(); - - unsafe { spiflash::erase_sector(data.as_ptr() as usize) }; - cache::flush_l2_cache(); + pub fn erase(&mut self) -> Result<(), WriteError> { + self.backend.erase() + .map_err(WriteError::Backend)?; Ok(()) } } -#[cfg(not(has_spiflash))] -mod imp { - use super::Error; - - pub fn read) -> R, R>(_key: &str, f: F) -> R { - f(Err(Error::NoFlash)) - } - - pub fn read_str) -> R, R>(_key: &str, f: F) -> R { - f(Err(Error::NoFlash)) - } - - pub fn write(_key: &str, _value: &[u8]) -> Result<(), Error> { - Err(Error::NoFlash) - } - - pub fn remove(_key: &str) -> Result<(), Error> { - Err(Error::NoFlash) - } - - pub fn erase() -> Result<(), Error> { - Err(Error::NoFlash) - } -} - -pub use self::imp::*; +// TODO: quadratic iterator diff --git a/src/test.rs b/src/test.rs new file mode 100644 index 0000000..0d3272a --- /dev/null +++ b/src/test.rs @@ -0,0 +1,79 @@ +#![cfg(test)] + +use crate::{Store, StoreBackend}; + +const SIZE: usize = 1024; + +pub struct TestBackend { + data: [u8; SIZE], +} + +impl TestBackend { + pub fn new() -> Self { + TestBackend { + data: [!0; SIZE], + } + } +} + +impl StoreBackend for TestBackend { + type Data = [u8; SIZE]; + + fn data(&self) -> &Self::Data { + &self.data + } + + type Error = (); + fn erase(&mut self) -> Result<(), Self::Error> { + for b in &mut self.data { + *b = !0; + } + Ok(()) + } + + fn program(&mut self, offset: usize, payload: &[u8]) -> Result<(), Self::Error> { + let end = offset + payload.len(); + if end > self.len() { + return Err(()); + } + self.data[offset..end].copy_from_slice(payload); + Ok(()) + } +} + +fn make_store() -> Store { + let backend = TestBackend::new(); + Store::new(backend) +} + +#[test] +fn empty_read_not_found() { + let store = make_store(); + assert_eq!(store.read("foo").unwrap(), None) +} + + +#[test] +fn write_read() { + let mut store = make_store(); + store.write("foo", b"bar").unwrap(); + assert_eq!(store.read("foo").unwrap().unwrap(), b"bar"); +} + +#[test] +fn write_int_read_str() { + let mut store = make_store(); + store.write_int("foo", 42).unwrap(); + assert_eq!(store.read_str("foo").unwrap().unwrap(), "42"); +} + +#[test] +fn write_many() { + let mut store = make_store(); + store.write("foo", b"do not compact away").unwrap(); + for _ in 0..10000 { + store.write("bar", b"SPAM").unwrap(); + } + assert_eq!(store.read("foo").unwrap().unwrap(), b"do not compact away"); + assert_eq!(store.read("bar").unwrap().unwrap(), b"SPAM"); +}