refactor for abstract StoreBackend

master
Astro 2020-12-11 00:28:24 +01:00
parent 2d09805b42
commit 449f7f5924
6 changed files with 322 additions and 233 deletions

15
Cargo.toml Normal file
View File

@ -0,0 +1,15 @@
[package]
name = "sfkv"
version = "0.1.0"
description = "Simple Flash Key Value store"
documentation = "https://docs.rs/sfkv/"
homepage = "https://git.m-labs.hk/M-Labs/sfkv"
repository = "https://git.m-labs.hk/M-Labs/sfkv.git"
readme = "README.md"
keywords = ["flash", "config"]
categories = ["embedded", "database"]
license = "GPL-3"
edition = "2018"
[dependencies]
byteorder = { version = "1.0", default-features = false }

59
src/error.rs Normal file
View File

@ -0,0 +1,59 @@
use core::{fmt, str};
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Error<B> {
Read(ReadError),
Write(WriteError<B>),
}
impl<B> From<ReadError> for Error<B> {
fn from(e: ReadError) -> Self {
Error::Read(e)
}
}
impl<B> From<WriteError<B>> for Error<B> {
fn from(e: WriteError<B>) -> Self {
Error::Write(e)
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ReadError {
Truncated { offset: usize },
InvalidSize { offset: usize, size: usize },
MissingSeparator { offset: usize },
Utf8Error(str::Utf8Error),
}
impl fmt::Display for ReadError {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match self {
&ReadError::Truncated { offset }=>
write!(f, "truncated record at offset {}", offset),
&ReadError::InvalidSize { offset, size } =>
write!(f, "invalid record size {} at offset {}", size, offset),
&ReadError::MissingSeparator { offset } =>
write!(f, "missing separator at offset {}", offset),
&ReadError::Utf8Error(err) =>
write!(f, "{}", err),
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum WriteError<B> {
SpaceExhausted,
Backend(B),
}
impl<B: fmt::Display> fmt::Display for WriteError<B> {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match self {
&WriteError::SpaceExhausted =>
write!(f, "space exhausted"),
&WriteError::Backend(ref e) =>
e.fmt(f),
}
}
}

30
src/fmt.rs Normal file
View File

@ -0,0 +1,30 @@
use core::fmt;
pub struct FmtWrapper<'a> {
buf: &'a mut [u8],
offset: usize,
}
impl<'a> FmtWrapper<'a> {
pub fn new(buf: &'a mut [u8]) -> Self {
FmtWrapper {
buf: buf,
offset: 0,
}
}
pub fn contents(&self) -> &[u8] {
&self.buf[..self.offset]
}
}
impl<'a> fmt::Write for FmtWrapper<'a> {
fn write_str(&mut self, s: &str) -> fmt::Result {
let bytes = s.as_bytes();
let remainder = &mut self.buf[self.offset..];
let remainder = &mut remainder[..bytes.len()];
remainder.copy_from_slice(bytes);
self.offset += bytes.len();
Ok(())
}
}

46
src/iter.rs Normal file
View File

@ -0,0 +1,46 @@
use byteorder::{ByteOrder, BigEndian};
use crate::ReadError;
#[derive(Clone)]
pub struct Iter<'a> {
data: &'a [u8],
pub offset: usize
}
impl<'a> Iter<'a> {
pub fn new(data: &'a [u8]) -> Iter<'a> {
Iter { data: data, offset: 0 }
}
}
impl<'a> Iterator for Iter<'a> {
type Item = Result<(&'a [u8], &'a [u8]), ReadError>;
fn next(&mut self) -> Option<Self::Item> {
let data = &self.data[self.offset..];
if data.len() < 4 {
return Some(Err(ReadError::Truncated { offset: self.offset }))
}
let record_size = BigEndian::read_u32(data) as usize;
if record_size == !0u32 as usize /* all ones; erased flash */ {
return None
} else if record_size < 4 || record_size > data.len() {
return Some(Err(ReadError::InvalidSize { offset: self.offset, size: record_size }))
}
let record_body = &data[4..record_size];
match record_body.iter().position(|&x| x == 0) {
None => {
return Some(Err(ReadError::MissingSeparator { offset: self.offset }))
}
Some(pos) => {
self.offset += record_size;
let (key, zero_and_value) = record_body.split_at(pos);
Some(Ok((key, &zero_and_value[1..])))
}
}
}
}

View File

@ -1,222 +1,113 @@
use core::{str, fmt};
#![no_std]
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Error {
AlreadyLocked,
SpaceExhausted,
Truncated { offset: usize },
InvalidSize { offset: usize, size: usize },
MissingSeparator { offset: usize },
Utf8Error(str::Utf8Error),
NoFlash,
use core::{fmt::Write, mem::size_of, str};
use byteorder::{ByteOrder, BigEndian};
mod error;
pub use error::{Error, ReadError, WriteError};
mod iter;
use iter::Iter;
mod fmt;
use fmt::FmtWrapper;
mod test;
pub trait StoreBackend {
type Data: Sized + AsRef<[u8]> + Clone;
/// Memory-mapped
fn data(&self) -> &Self::Data;
fn len(&self) -> usize {
size_of::<Self::Data>()
}
/// utility for compaction, requires compile-time knowledge of the data length
type Error;
fn erase(&mut self) -> Result<(), Self::Error>;
fn program(&mut self, offset: usize, payload: &[u8]) -> Result<(), Self::Error>;
/// called after many `program()` invocations to allow for eg. cache flushing
fn program_done(&mut self) {}
}
impl fmt::Display for Error {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match self {
&Error::AlreadyLocked =>
write!(f, "attempt at reentrant access"),
&Error::SpaceExhausted =>
write!(f, "space exhausted"),
&Error::Truncated { offset }=>
write!(f, "truncated record at offset {}", offset),
&Error::InvalidSize { offset, size } =>
write!(f, "invalid record size {} at offset {}", size, offset),
&Error::MissingSeparator { offset } =>
write!(f, "missing separator at offset {}", offset),
&Error::Utf8Error(err) =>
write!(f, "{}", err),
&Error::NoFlash =>
write!(f, "flash memory is not present"),
}
}
// TODO: NoFlash
pub struct Store<B: StoreBackend> {
backend: B,
}
#[cfg(has_spiflash)]
mod imp {
use core::str;
use byteorder::{ByteOrder, BigEndian};
use cache;
use spiflash;
use super::Error;
use core::fmt;
use core::fmt::Write;
struct FmtWrapper<'a> {
buf: &'a mut [u8],
offset: usize,
impl<B: StoreBackend> Store<B> {
pub fn new(backend: B) -> Self {
Store { backend }
}
impl<'a> FmtWrapper<'a> {
fn new(buf: &'a mut [u8]) -> Self {
FmtWrapper {
buf: buf,
offset: 0,
pub fn read(&self, key: &str) -> Result<Option<&[u8]>, ReadError> {
let mut iter = Iter::new(self.backend.data().as_ref());
let mut value = None;
while let Some(result) = iter.next() {
let (record_key, record_value) = result?;
if key.as_bytes() == record_key {
// last write wins
value = Some(record_value)
}
}
fn contents(&self) -> &[u8] {
&self.buf[..self.offset]
}
Ok(value)
}
impl<'a> fmt::Write for FmtWrapper<'a> {
fn write_str(&mut self, s: &str) -> fmt::Result {
let bytes = s.as_bytes();
let remainder = &mut self.buf[self.offset..];
let remainder = &mut remainder[..bytes.len()];
remainder.copy_from_slice(bytes);
self.offset += bytes.len();
Ok(())
}
pub fn read_str(&self, key: &str) -> Result<Option<&str>, ReadError> {
self.read(key)
.and_then(|value| value
.map(|value| str::from_utf8(value)
.map_err(ReadError::Utf8Error)
)
.transpose())
}
// One flash sector immediately before the firmware.
const ADDR: usize = ::mem::FLASH_BOOT_ADDRESS - spiflash::SECTOR_SIZE;
const SIZE: usize = spiflash::SECTOR_SIZE;
mod lock {
use core::slice;
use core::sync::atomic::{AtomicUsize, Ordering, ATOMIC_USIZE_INIT};
use super::Error;
static LOCKED: AtomicUsize = ATOMIC_USIZE_INIT;
pub struct Lock;
impl Lock {
pub fn take() -> Result<Lock, Error> {
if LOCKED.swap(1, Ordering::SeqCst) != 0 {
Err(Error::AlreadyLocked)
} else {
Ok(Lock)
}
}
pub fn data(&self) -> &'static [u8] {
unsafe { slice::from_raw_parts(super::ADDR as *const u8, super::SIZE) }
}
}
impl Drop for Lock {
fn drop(&mut self) {
LOCKED.store(0, Ordering::SeqCst)
}
}
}
use self::lock::Lock;
#[derive(Clone)]
struct Iter<'a> {
data: &'a [u8],
offset: usize
}
impl<'a> Iter<'a> {
fn new(data: &'a [u8]) -> Iter<'a> {
Iter { data: data, offset: 0 }
}
}
impl<'a> Iterator for Iter<'a> {
type Item = Result<(&'a [u8], &'a [u8]), Error>;
fn next(&mut self) -> Option<Self::Item> {
let data = &self.data[self.offset..];
if data.len() < 4 {
// error!("offset {}: truncated record", self.offset);
return Some(Err(Error::Truncated { offset: self.offset }))
}
let record_size = BigEndian::read_u32(data) as usize;
if record_size == !0 /* all ones; erased flash */ {
return None
} else if record_size < 4 || record_size > data.len() {
return Some(Err(Error::InvalidSize { offset: self.offset, size: record_size }))
}
let record_body = &data[4..record_size];
match record_body.iter().position(|&x| x == 0) {
None => {
return Some(Err(Error::MissingSeparator { offset: self.offset }))
}
Some(pos) => {
self.offset += record_size;
let (key, zero_and_value) = record_body.split_at(pos);
Some(Ok((key, &zero_and_value[1..])))
}
}
}
}
pub fn read<F: FnOnce(Result<&[u8], Error>) -> R, R>(key: &str, f: F) -> R {
f(Lock::take().and_then(|lock| {
let mut iter = Iter::new(lock.data());
let mut value = &[][..];
while let Some(result) = iter.next() {
let (record_key, record_value) = result?;
if key.as_bytes() == record_key {
// last write wins
value = record_value
}
}
Ok(value)
}))
}
pub fn read_str<F: FnOnce(Result<&str, Error>) -> R, R>(key: &str, f: F) -> R {
read(key, |result| {
f(result.and_then(|value| str::from_utf8(value).map_err(Error::Utf8Error)))
})
}
unsafe fn append_at(data: &[u8], mut offset: usize,
key: &[u8], value: &[u8]) -> Result<usize, Error> {
unsafe fn append_at(&mut self, mut offset: usize,
key: &[u8], value: &[u8]) -> Result<usize, WriteError<B::Error>> {
let record_size = 4 + key.len() + 1 + value.len();
if offset + record_size > data.len() {
return Err(Error::SpaceExhausted)
if offset + record_size > self.backend.len() {
return Err(WriteError::SpaceExhausted)
}
let mut record_size_bytes = [0u8; 4];
BigEndian::write_u32(&mut record_size_bytes[..], record_size as u32);
{
let mut write = |payload| {
spiflash::write(data.as_ptr().offset(offset as isize) as usize, payload);
let mut write = |payload| -> Result<(), WriteError<B::Error>> {
self.backend.program(offset, payload)
.map_err(WriteError::Backend)?;
offset += payload.len();
Ok(())
};
write(&record_size_bytes[..]);
write(key);
write(&[0]);
write(value);
cache::flush_l2_cache();
write(&record_size_bytes[..])?;
write(key)?;
write(&[0])?;
write(value)?;
}
self.backend.program_done();
Ok(offset)
}
fn compact() -> Result<(), Error> {
let lock = Lock::take()?;
let data = lock.data();
fn compact(&mut self) -> Result<(), Error<B::Error>> {
// static mut OLD_DATA: B::Data;
// let mut old_data = MaybeUninit::<B::Data>::zeroed();
// let old_data = unsafe {
// let mut old_data = old_data.assume_init();
// old_data.copy_from_slice(self.backend.data().as_ref());
// old_data
// };
let old_data = self.backend.data().clone();
static mut OLD_DATA: [u8; SIZE] = [0; SIZE];
let old_data = unsafe {
OLD_DATA.copy_from_slice(data);
&OLD_DATA[..]
};
unsafe { spiflash::erase_sector(data.as_ptr() as usize) };
self.erase()?;
// This is worst-case quadratic, but we're limited by a small SPI flash sector size,
// so it does not really matter.
let mut offset = 0;
let mut iter = Iter::new(old_data);
let mut iter = Iter::new(old_data.as_ref());
'iter: while let Some(result) = iter.next() {
let (key, mut value) = result?;
let (key, value) = result?;
if value.is_empty() {
// This is a removed entry, ignore it.
continue
@ -230,84 +121,53 @@ mod imp {
continue 'iter
}
}
offset = unsafe { append_at(data, offset, key, value)? };
offset = unsafe { self.append_at(offset, key, value)? };
}
Ok(())
}
fn append(key: &str, value: &[u8]) -> Result<(), Error> {
let lock = Lock::take()?;
let data = lock.data();
fn append(&mut self, key: &str, value: &[u8]) -> Result<(), Error<B::Error>> {
let free_offset = {
let mut iter = Iter::new(data);
let mut iter = Iter::new(self.backend.data().as_ref());
while let Some(result) = iter.next() {
let _ = result?;
}
iter.offset
};
unsafe { append_at(data, free_offset, key.as_bytes(), value)? };
unsafe { self.append_at(free_offset, key.as_bytes(), value)? };
Ok(())
}
pub fn write(key: &str, value: &[u8]) -> Result<(), Error> {
match append(key, value) {
Err(Error::SpaceExhausted) => {
compact()?;
append(key, value)
pub fn write(&mut self, key: &str, value: &[u8]) -> Result<(), Error<B::Error>> {
match self.append(key, value) {
Err(Error::Write(WriteError::SpaceExhausted)) => {
self.compact()?;
self.append(key, value)
}
res => res
}
}
pub fn write_int(key: &str, value: u32) -> Result<(), Error> {
pub fn write_int(&mut self, key: &str, value: u32) -> Result<(), Error<B::Error>> {
let mut buf = [0; 16];
let mut wrapper = FmtWrapper::new(&mut buf);
write!(&mut wrapper, "{}", value).unwrap();
write(key, wrapper.contents())
self.write(key, wrapper.contents())
}
pub fn remove(key: &str) -> Result<(), Error> {
write(key, &[])
pub fn remove(&mut self, key: &str) -> Result<(), Error<B::Error>> {
self.write(key, &[])
}
pub fn erase() -> Result<(), Error> {
let lock = Lock::take()?;
let data = lock.data();
unsafe { spiflash::erase_sector(data.as_ptr() as usize) };
cache::flush_l2_cache();
pub fn erase(&mut self) -> Result<(), WriteError<B::Error>> {
self.backend.erase()
.map_err(WriteError::Backend)?;
Ok(())
}
}
#[cfg(not(has_spiflash))]
mod imp {
use super::Error;
pub fn read<F: FnOnce(Result<&[u8], Error>) -> R, R>(_key: &str, f: F) -> R {
f(Err(Error::NoFlash))
}
pub fn read_str<F: FnOnce(Result<&str, Error>) -> R, R>(_key: &str, f: F) -> R {
f(Err(Error::NoFlash))
}
pub fn write(_key: &str, _value: &[u8]) -> Result<(), Error> {
Err(Error::NoFlash)
}
pub fn remove(_key: &str) -> Result<(), Error> {
Err(Error::NoFlash)
}
pub fn erase() -> Result<(), Error> {
Err(Error::NoFlash)
}
}
pub use self::imp::*;
// TODO: quadratic iterator

79
src/test.rs Normal file
View File

@ -0,0 +1,79 @@
#![cfg(test)]
use crate::{Store, StoreBackend};
const SIZE: usize = 1024;
pub struct TestBackend {
data: [u8; SIZE],
}
impl TestBackend {
pub fn new() -> Self {
TestBackend {
data: [!0; SIZE],
}
}
}
impl StoreBackend for TestBackend {
type Data = [u8; SIZE];
fn data(&self) -> &Self::Data {
&self.data
}
type Error = ();
fn erase(&mut self) -> Result<(), Self::Error> {
for b in &mut self.data {
*b = !0;
}
Ok(())
}
fn program(&mut self, offset: usize, payload: &[u8]) -> Result<(), Self::Error> {
let end = offset + payload.len();
if end > self.len() {
return Err(());
}
self.data[offset..end].copy_from_slice(payload);
Ok(())
}
}
fn make_store() -> Store<TestBackend> {
let backend = TestBackend::new();
Store::new(backend)
}
#[test]
fn empty_read_not_found() {
let store = make_store();
assert_eq!(store.read("foo").unwrap(), None)
}
#[test]
fn write_read() {
let mut store = make_store();
store.write("foo", b"bar").unwrap();
assert_eq!(store.read("foo").unwrap().unwrap(), b"bar");
}
#[test]
fn write_int_read_str() {
let mut store = make_store();
store.write_int("foo", 42).unwrap();
assert_eq!(store.read_str("foo").unwrap().unwrap(), "42");
}
#[test]
fn write_many() {
let mut store = make_store();
store.write("foo", b"do not compact away").unwrap();
for _ in 0..10000 {
store.write("bar", b"SPAM").unwrap();
}
assert_eq!(store.read("foo").unwrap().unwrap(), b"do not compact away");
assert_eq!(store.read("bar").unwrap().unwrap(), b"SPAM");
}