diff --git a/artiq/firmware/Cargo.lock b/artiq/firmware/Cargo.lock index f4bd9b88a..80933a416 100644 --- a/artiq/firmware/Cargo.lock +++ b/artiq/firmware/Cargo.lock @@ -352,6 +352,9 @@ dependencies = [ "board_artiq", "board_misoc", "build_misoc", + "cslice", + "eh", + "io", "log", "proto_artiq", "riscv", diff --git a/artiq/firmware/satman/Cargo.toml b/artiq/firmware/satman/Cargo.toml index 20dec311f..f8016f576 100644 --- a/artiq/firmware/satman/Cargo.toml +++ b/artiq/firmware/satman/Cargo.toml @@ -14,8 +14,11 @@ build_misoc = { path = "../libbuild_misoc" } [dependencies] log = { version = "0.4", default-features = false } +io = { path = "../libio", features = ["byteorder", "alloc"] } +cslice = { version = "0.3" } board_misoc = { path = "../libboard_misoc", features = ["uart_console", "log"] } board_artiq = { path = "../libboard_artiq", features = ["alloc"] } alloc_list = { path = "../liballoc_list" } riscv = { version = "0.6.0", features = ["inline-asm"] } proto_artiq = { path = "../libproto_artiq", features = ["log", "alloc"] } +eh = { path = "../libeh" } \ No newline at end of file diff --git a/artiq/firmware/satman/Makefile b/artiq/firmware/satman/Makefile index 82e65d730..55befda95 100644 --- a/artiq/firmware/satman/Makefile +++ b/artiq/firmware/satman/Makefile @@ -1,9 +1,14 @@ include ../include/generated/variables.mak include $(MISOC_DIRECTORY)/software/common.mak -LDFLAGS += -L../libbase +CFLAGS += \ + -I$(LIBUNWIND_DIRECTORY) \ + -I$(LIBUNWIND_DIRECTORY)/../unwinder/include -RUSTFLAGS += -Cpanic=abort +LDFLAGS += \ + -L../libunwind + +RUSTFLAGS += -Cpanic=unwind export XBUILD_SYSROOT_PATH=$(BUILDINC_DIRECTORY)/../sysroot @@ -15,8 +20,12 @@ $(RUSTOUT)/libsatman.a: --manifest-path $(SATMAN_DIRECTORY)/Cargo.toml \ --target $(SATMAN_DIRECTORY)/../$(CARGO_TRIPLE).json -satman.elf: $(RUSTOUT)/libsatman.a - $(link) -T $(SATMAN_DIRECTORY)/satman.ld +satman.elf: $(RUSTOUT)/libsatman.a ksupport_data.o + $(link) -T $(SATMAN_DIRECTORY)/satman.ld \ + -lunwind-vexriscv-bare -m elf32lriscv + +ksupport_data.o: ../ksupport/ksupport.elf + $(LD) -r -m elf32lriscv -b binary -o $@ $< %.bin: %.elf $(objcopy) -O binary diff --git a/artiq/firmware/satman/cache.rs b/artiq/firmware/satman/cache.rs new file mode 100644 index 000000000..fa73364cb --- /dev/null +++ b/artiq/firmware/satman/cache.rs @@ -0,0 +1,83 @@ +use alloc::{vec, vec::Vec, string::String, collections::btree_map::BTreeMap}; +use cslice::{CSlice, AsCSlice}; +use core::mem::transmute; + +struct Entry { + data: Vec, + slice: CSlice<'static, i32>, + borrowed: bool +} + +impl core::fmt::Debug for Entry { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("Entry") + .field("data", &self.data) + .field("borrowed", &self.borrowed) + .finish() + } +} + +pub struct Cache { + entries: BTreeMap, + empty: CSlice<'static, i32>, +} + +impl core::fmt::Debug for Cache { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("Cache") + .field("entries", &self.entries) + .finish() + } +} + +impl Cache { + pub fn new() -> Cache { + let empty_vec = vec![]; + let empty = unsafe { + transmute::, CSlice<'static, i32>>(empty_vec.as_c_slice()) + }; + Cache { entries: BTreeMap::new(), empty } + } + + pub fn get(&mut self, key: &str) -> *const CSlice<'static, i32> { + match self.entries.get_mut(key) { + None => &self.empty, + Some(ref mut entry) => { + entry.borrowed = true; + &entry.slice + } + } + } + + pub fn put(&mut self, key: &str, data: &[i32]) -> Result<(), ()> { + match self.entries.get_mut(key) { + None => (), + Some(ref mut entry) => { + if entry.borrowed { return Err(()) } + entry.data = Vec::from(data); + unsafe { + entry.slice = transmute::, CSlice<'static, i32>>( + entry.data.as_c_slice()); + } + return Ok(()) + } + } + + let data = Vec::from(data); + let slice = unsafe { + transmute::, CSlice<'static, i32>>(data.as_c_slice()) + }; + self.entries.insert(String::from(key), Entry { + data, + slice, + borrowed: false + }); + Ok(()) + } + + pub unsafe fn unborrow(&mut self) { + for (_key, entry) in self.entries.iter_mut() { + entry.borrowed = false; + } + } +} diff --git a/artiq/firmware/satman/dma.rs b/artiq/firmware/satman/dma.rs index 133bfd3c0..34dcaf475 100644 --- a/artiq/firmware/satman/dma.rs +++ b/artiq/firmware/satman/dma.rs @@ -1,5 +1,6 @@ use board_misoc::{csr, cache::flush_l2_cache}; use alloc::{vec::Vec, collections::btree_map::BTreeMap}; +use ::{cricon_select, RtioMaster}; const ALIGNMENT: usize = 64; @@ -126,14 +127,14 @@ impl Manager { csr::rtio_dma::base_address_write(ptr as u64); csr::rtio_dma::time_offset_write(timestamp as u64); - csr::cri_con::selected_write(1); + cricon_select(RtioMaster::Dma); csr::rtio_dma::enable_write(1); // playback has begun here, for status call check_state } Ok(()) } - pub fn check_state(&mut self) -> Option { + pub fn get_status(&mut self) -> Option { if self.state != ManagerState::Playback { // nothing to report return None; @@ -141,12 +142,11 @@ impl Manager { let dma_enable = unsafe { csr::rtio_dma::enable_read() }; if dma_enable != 0 { return None; - } - else { + } else { self.state = ManagerState::Idle; unsafe { - csr::cri_con::selected_write(0); - let error = csr::rtio_dma::error_read(); + cricon_select(RtioMaster::Drtio); + let error = csr::rtio_dma::error_read(); let channel = csr::rtio_dma::error_channel_read(); let timestamp = csr::rtio_dma::error_timestamp_read(); if error != 0 { @@ -161,4 +161,8 @@ impl Manager { } } + pub fn running(&self) -> bool { + self.state == ManagerState::Playback + } + } \ No newline at end of file diff --git a/artiq/firmware/satman/kernel.rs b/artiq/firmware/satman/kernel.rs new file mode 100644 index 000000000..cc401a134 --- /dev/null +++ b/artiq/firmware/satman/kernel.rs @@ -0,0 +1,823 @@ +use core::{mem, option::NoneError, cmp::min}; +use alloc::{string::String, format, vec::Vec, collections::{btree_map::BTreeMap, vec_deque::VecDeque}}; +use cslice::AsCSlice; + +use board_artiq::{mailbox, spi}; +use board_misoc::{csr, clock, i2c}; +use proto_artiq::{kernel_proto as kern, session_proto::Reply::KernelException as HostKernelException, rpc_proto as rpc}; +use eh::eh_artiq; +use io::{Cursor, ProtoRead}; +use kernel::eh_artiq::StackPointerBacktrace; + +use ::{cricon_select, RtioMaster}; +use cache::Cache; +use SAT_PAYLOAD_MAX_SIZE; +use MASTER_PAYLOAD_MAX_SIZE; + +mod kernel_cpu { + use super::*; + use core::ptr; + + use proto_artiq::kernel_proto::{KERNELCPU_EXEC_ADDRESS, KERNELCPU_LAST_ADDRESS, KSUPPORT_HEADER_SIZE}; + + pub unsafe fn start() { + if csr::kernel_cpu::reset_read() == 0 { + panic!("attempted to start kernel CPU when it is already running") + } + + stop(); + + extern { + static _binary____ksupport_ksupport_elf_start: u8; + static _binary____ksupport_ksupport_elf_end: u8; + } + let ksupport_start = &_binary____ksupport_ksupport_elf_start as *const _; + let ksupport_end = &_binary____ksupport_ksupport_elf_end as *const _; + ptr::copy_nonoverlapping(ksupport_start, + (KERNELCPU_EXEC_ADDRESS - KSUPPORT_HEADER_SIZE) as *mut u8, + ksupport_end as usize - ksupport_start as usize); + + csr::kernel_cpu::reset_write(0); + } + + pub unsafe fn stop() { + csr::kernel_cpu::reset_write(1); + cricon_select(RtioMaster::Drtio); + + mailbox::acknowledge(); + } + + pub fn validate(ptr: usize) -> bool { + ptr >= KERNELCPU_EXEC_ADDRESS && ptr <= KERNELCPU_LAST_ADDRESS + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum KernelState { + Absent, + Loaded, + Running, + MsgAwait { max_time: u64 }, + MsgSending +} + +#[derive(Debug)] +pub enum Error { + Load(String), + KernelNotFound, + InvalidPointer(usize), + Unexpected(String), + NoMessage, + AwaitingMessage, + SubkernelIoError, + KernelException(Sliceable) +} + +impl From for Error { + fn from(_: NoneError) -> Error { + Error::KernelNotFound + } +} + +impl From> for Error { + fn from(_value: io::Error) -> Error { + Error::SubkernelIoError + } +} + +macro_rules! unexpected { + ($($arg:tt)*) => (return Err(Error::Unexpected(format!($($arg)*)))); +} + +/* represents data that has to be sent to Master */ +#[derive(Debug)] +pub struct Sliceable { + it: usize, + data: Vec +} + +/* represents interkernel messages */ +struct Message { + count: u8, + tag: u8, + data: Vec +} + +#[derive(PartialEq)] +enum OutMessageState { + NoMessage, + MessageReady, + MessageBeingSent, + MessageSent, + MessageAcknowledged +} + +/* for dealing with incoming and outgoing interkernel messages */ +struct MessageManager { + out_message: Option, + out_state: OutMessageState, + in_queue: VecDeque, + in_buffer: Option, +} + +// Per-run state +struct Session { + kernel_state: KernelState, + log_buffer: String, + last_exception: Option, + messages: MessageManager +} + +#[derive(Debug)] +struct KernelLibrary { + library: Vec, + complete: bool +} + +pub struct Manager { + kernels: BTreeMap, + current_id: u32, + session: Session, + cache: Cache, + last_finished: Option +} + +pub struct SubkernelFinished { + pub id: u32, + pub with_exception: bool +} + +pub struct SliceMeta { + pub len: u16, + pub last: bool +} + +macro_rules! get_slice_fn { + ( $name:tt, $size:expr ) => { + pub fn $name(&mut self, data_slice: &mut [u8; $size]) -> SliceMeta { + if self.data.len() == 0 { + return SliceMeta { len: 0, last: true }; + } + let len = min($size, self.data.len() - self.it); + let last = self.it + len == self.data.len(); + + data_slice[..len].clone_from_slice(&self.data[self.it..self.it+len]); + self.it += len; + + SliceMeta { + len: len as u16, + last: last + } + } + }; +} + +impl Sliceable { + pub fn new(data: Vec) -> Sliceable { + Sliceable { + it: 0, + data: data + } + } + + get_slice_fn!(get_slice_sat, SAT_PAYLOAD_MAX_SIZE); + get_slice_fn!(get_slice_master, MASTER_PAYLOAD_MAX_SIZE); +} + +impl MessageManager { + pub fn new() -> MessageManager { + MessageManager { + out_message: None, + out_state: OutMessageState::NoMessage, + in_queue: VecDeque::new(), + in_buffer: None + } + } + + pub fn handle_incoming(&mut self, last: bool, length: usize, data: &[u8; MASTER_PAYLOAD_MAX_SIZE]) { + // called when receiving a message from master + match self.in_buffer.as_mut() { + Some(message) => message.data.extend(&data[..length]), + None => { + self.in_buffer = Some(Message { + count: data[0], + tag: data[1], + data: data[2..length].to_vec() + }); + } + }; + if last { + // when done, remove from working queue + self.in_queue.push_back(self.in_buffer.take().unwrap()); + } + } + + pub fn is_outgoing_ready(&mut self) -> bool { + // called by main loop, to see if there's anything to send, will send it afterwards + match self.out_state { + OutMessageState::MessageReady => { + self.out_state = OutMessageState::MessageBeingSent; + true + }, + _ => false + } + } + + pub fn was_message_acknowledged(&mut self) -> bool { + match self.out_state { + OutMessageState::MessageAcknowledged => { + self.out_state = OutMessageState::NoMessage; + true + }, + _ => false + } + } + + pub fn get_outgoing_slice(&mut self, data_slice: &mut [u8; MASTER_PAYLOAD_MAX_SIZE]) -> Option { + if self.out_state != OutMessageState::MessageBeingSent { + return None; + } + let meta = self.out_message.as_mut()?.get_slice_master(data_slice); + if meta.last { + // clear the message slot + self.out_message = None; + // notify kernel with a flag that message is sent + self.out_state = OutMessageState::MessageSent; + } + Some(meta) + } + + pub fn ack_slice(&mut self) -> bool { + // returns whether or not there's more to be sent + match self.out_state { + OutMessageState::MessageBeingSent => true, + OutMessageState::MessageSent => { + self.out_state = OutMessageState::MessageAcknowledged; + false + }, + _ => { + warn!("received unsolicited SubkernelMessageAck"); + false + } + } + } + + pub fn accept_outgoing(&mut self, count: u8, tag: &[u8], data: *const *const ()) -> Result<(), Error> { + let mut writer = Cursor::new(Vec::new()); + rpc::send_args(&mut writer, 0, tag, data)?; + // skip service tag, but write the count + let mut data = writer.into_inner().split_off(3); + data[0] = count; + self.out_message = Some(Sliceable::new(data)); + self.out_state = OutMessageState::MessageReady; + Ok(()) + } + + pub fn get_incoming(&mut self) -> Option { + self.in_queue.pop_front() + } +} + +impl Session { + pub fn new() -> Session { + Session { + kernel_state: KernelState::Absent, + log_buffer: String::new(), + last_exception: None, + messages: MessageManager::new() + } + } + + fn running(&self) -> bool { + match self.kernel_state { + KernelState::Absent | KernelState::Loaded => false, + KernelState::Running | KernelState::MsgAwait { .. } | + KernelState::MsgSending => true + } + } + + fn flush_log_buffer(&mut self) { + if &self.log_buffer[self.log_buffer.len() - 1..] == "\n" { + for line in self.log_buffer.lines() { + info!(target: "kernel", "{}", line); + } + self.log_buffer.clear() + } + } +} + +impl Manager { + pub fn new() -> Manager { + Manager { + kernels: BTreeMap::new(), + current_id: 0, + session: Session::new(), + cache: Cache::new(), + last_finished: None, + } + } + + pub fn add(&mut self, id: u32, last: bool, data: &[u8], data_len: usize) -> Result<(), Error> { + let kernel = match self.kernels.get_mut(&id) { + Some(kernel) => { + if kernel.complete { + // replace entry + self.kernels.remove(&id); + self.kernels.insert(id, KernelLibrary { + library: Vec::new(), + complete: false }); + self.kernels.get_mut(&id)? + } else { + kernel + } + }, + None => { + self.kernels.insert(id, KernelLibrary { + library: Vec::new(), + complete: false }); + self.kernels.get_mut(&id)? + }, + }; + kernel.library.extend(&data[0..data_len]); + + kernel.complete = last; + Ok(()) + } + + pub fn is_running(&self) -> bool { + self.session.running() + } + + pub fn get_current_id(&self) -> Option { + match self.is_running() { + true => Some(self.current_id), + false => None + } + } + + pub fn stop(&mut self) { + unsafe { kernel_cpu::stop() } + self.session.kernel_state = KernelState::Absent; + unsafe { self.cache.unborrow() } + } + + pub fn run(&mut self, id: u32) -> Result<(), Error> { + info!("starting subkernel #{}", id); + if self.session.kernel_state != KernelState::Loaded + || self.current_id != id { + self.load(id)?; + } + self.session.kernel_state = KernelState::Running; + cricon_select(RtioMaster::Kernel); + + kern_acknowledge() + } + + pub fn message_handle_incoming(&mut self, last: bool, length: usize, slice: &[u8; MASTER_PAYLOAD_MAX_SIZE]) { + if !self.is_running() { + return; + } + self.session.messages.handle_incoming(last, length, slice); + } + + pub fn message_get_slice(&mut self, slice: &mut [u8; MASTER_PAYLOAD_MAX_SIZE]) -> Option { + if !self.is_running() { + return None; + } + self.session.messages.get_outgoing_slice(slice) + } + + pub fn message_ack_slice(&mut self) -> bool { + if !self.is_running() { + warn!("received unsolicited SubkernelMessageAck"); + return false; + } + self.session.messages.ack_slice() + } + + pub fn message_is_ready(&mut self) -> bool { + self.session.messages.is_outgoing_ready() + } + + pub fn get_last_finished(&mut self) -> Option { + self.last_finished.take() + } + + pub fn load(&mut self, id: u32) -> Result<(), Error> { + if self.current_id == id && self.session.kernel_state == KernelState::Loaded { + return Ok(()) + } + if !self.kernels.get(&id)?.complete { + return Err(Error::KernelNotFound) + } + self.current_id = id; + self.session = Session::new(); + self.stop(); + + unsafe { + kernel_cpu::start(); + + kern_send(&kern::LoadRequest(&self.kernels.get(&id)?.library)).unwrap(); + kern_recv(|reply| { + match reply { + kern::LoadReply(Ok(())) => { + self.session.kernel_state = KernelState::Loaded; + Ok(()) + } + kern::LoadReply(Err(error)) => { + kernel_cpu::stop(); + Err(Error::Load(format!("{}", error))) + } + other => { + unexpected!("unexpected kernel CPU reply to load request: {:?}", other) + } + } + }) + } + } + + pub fn exception_get_slice(&mut self, data_slice: &mut [u8; SAT_PAYLOAD_MAX_SIZE]) -> SliceMeta { + match self.session.last_exception.as_mut() { + Some(exception) => exception.get_slice_sat(data_slice), + None => SliceMeta { len: 0, last: true } + } + } + + fn runtime_exception(&mut self, cause: Error) { + let raw_exception: Vec = Vec::new(); + let mut writer = Cursor::new(raw_exception); + match (HostKernelException { + exceptions: &[Some(eh_artiq::Exception { + id: 11, // SubkernelError, defined in ksupport + message: format!("in subkernel id {}: {:?}", self.current_id, cause).as_c_slice(), + param: [0, 0, 0], + file: file!().as_c_slice(), + line: line!(), + column: column!(), + function: format!("subkernel id {}", self.current_id).as_c_slice(), + })], + stack_pointers: &[StackPointerBacktrace { + stack_pointer: 0, + initial_backtrace_size: 0, + current_backtrace_size: 0 + }], + backtrace: &[], + async_errors: 0 + }).write_to(&mut writer) { + Ok(_) => self.session.last_exception = Some(Sliceable::new(writer.into_inner())), + Err(_) => error!("Error writing exception data") + } + } + + pub fn process_kern_requests(&mut self, rank: u8) { + if !self.is_running() { + return; + } + + match self.process_external_messages() { + Ok(()) => (), + Err(Error::AwaitingMessage) => return, // kernel still waiting, do not process kernel messages + Err(Error::KernelException(exception)) => { + unsafe { kernel_cpu::stop() } + self.session.kernel_state = KernelState::Absent; + unsafe { self.cache.unborrow() } + self.session.last_exception = Some(exception); + self.last_finished = Some(SubkernelFinished { id: self.current_id, with_exception: true }) + }, + Err(e) => { + error!("Error while running processing external messages: {:?}", e); + self.stop(); + self.runtime_exception(e); + self.last_finished = Some(SubkernelFinished { id: self.current_id, with_exception: true }) + } + } + + match self.process_kern_message(rank) { + Ok(Some(with_exception)) => { + self.last_finished = Some(SubkernelFinished { id: self.current_id, with_exception: with_exception }) + }, + Ok(None) | Err(Error::NoMessage) => (), + Err(e) => { + error!("Error while running kernel: {:?}", e); + self.stop(); + self.runtime_exception(e); + self.last_finished = Some(SubkernelFinished { id: self.current_id, with_exception: true }) + } + } + } + + fn process_external_messages(&mut self) -> Result<(), Error> { + match self.session.kernel_state { + KernelState::MsgAwait { max_time } => { + if clock::get_ms() > max_time { + kern_send(&kern::SubkernelMsgRecvReply { status: kern::SubkernelStatus::Timeout, count: 0 })?; + self.session.kernel_state = KernelState::Running; + return Ok(()) + } + if let Some(message) = self.session.messages.get_incoming() { + kern_send(&kern::SubkernelMsgRecvReply { status: kern::SubkernelStatus::NoError, count: message.count })?; + self.session.kernel_state = KernelState::Running; + pass_message_to_kernel(&message) + } else { + Err(Error::AwaitingMessage) + } + }, + KernelState::MsgSending => { + if self.session.messages.was_message_acknowledged() { + self.session.kernel_state = KernelState::Running; + kern_acknowledge() + } else { + Err(Error::AwaitingMessage) + } + }, + _ => Ok(()) + } + } + + fn process_kern_message(&mut self, rank: u8) -> Result, Error> { + // returns Ok(with_exception) on finish + // None if the kernel is still running + kern_recv(|request| { + match (request, self.session.kernel_state) { + (&kern::LoadReply(_), KernelState::Loaded) => { + // We're standing by; ignore the message. + return Ok(None) + } + (_, KernelState::Running) => (), + _ => { + unexpected!("unexpected request {:?} from kernel CPU in {:?} state", + request, self.session.kernel_state) + }, + } + + if process_kern_hwreq(request, rank)? { + return Ok(None) + } + + match request { + &kern::Log(args) => { + use core::fmt::Write; + self.session.log_buffer + .write_fmt(args) + .unwrap_or_else(|_| warn!("cannot append to session log buffer")); + self.session.flush_log_buffer(); + kern_acknowledge() + } + + &kern::LogSlice(arg) => { + self.session.log_buffer += arg; + self.session.flush_log_buffer(); + kern_acknowledge() + } + + &kern::RpcFlush => { + // we do not have to do anything about this request, + // it is sent by the kernel firmware regardless of RPC being used + kern_acknowledge() + } + + &kern::CacheGetRequest { key } => { + let value = self.cache.get(key); + kern_send(&kern::CacheGetReply { + value: unsafe { mem::transmute(value) } + }) + } + + &kern::CachePutRequest { key, value } => { + let succeeded = self.cache.put(key, value).is_ok(); + kern_send(&kern::CachePutReply { succeeded: succeeded }) + } + + &kern::RunFinished => { + unsafe { kernel_cpu::stop() } + self.session.kernel_state = KernelState::Absent; + unsafe { self.cache.unborrow() } + + return Ok(Some(false)) + } + &kern::RunException { exceptions, stack_pointers, backtrace } => { + unsafe { kernel_cpu::stop() } + self.session.kernel_state = KernelState::Absent; + unsafe { self.cache.unborrow() } + let exception = slice_kernel_exception(&exceptions, &stack_pointers, &backtrace)?; + self.session.last_exception = Some(exception); + return Ok(Some(true)) + } + + &kern::SubkernelMsgSend { id: _, count, tag, data } => { + self.session.messages.accept_outgoing(count, tag, data)?; + // acknowledge after the message is sent + self.session.kernel_state = KernelState::MsgSending; + Ok(()) + } + + &kern::SubkernelMsgRecvRequest { id: _, timeout } => { + let max_time = clock::get_ms() + timeout as u64; + self.session.kernel_state = KernelState::MsgAwait { max_time: max_time }; + Ok(()) + }, + + request => unexpected!("unexpected request {:?} from kernel CPU", request) + }.and(Ok(None)) + }) + } +} + +impl Drop for Manager { + fn drop(&mut self) { + cricon_select(RtioMaster::Drtio); + unsafe { + kernel_cpu::stop() + }; + } +} + +fn kern_recv(f: F) -> Result + where F: FnOnce(&kern::Message) -> Result { + if mailbox::receive() == 0 { + return Err(Error::NoMessage); + }; + if !kernel_cpu::validate(mailbox::receive()) { + return Err(Error::InvalidPointer(mailbox::receive())) + } + f(unsafe { &*(mailbox::receive() as *const kern::Message) }) +} + +fn kern_recv_w_timeout(timeout: u64, f: F) -> Result + where F: FnOnce(&kern::Message) -> Result + Copy { + // sometimes kernel may be too slow to respond immediately + // (e.g. when receiving external messages) + // we cannot wait indefinitely to keep the satellite responsive + // so a timeout is used instead + let max_time = clock::get_ms() + timeout; + while clock::get_ms() < max_time { + match kern_recv(f) { + Err(Error::NoMessage) => continue, + anything_else => return anything_else + } + } + Err(Error::NoMessage) +} + +fn kern_acknowledge() -> Result<(), Error> { + mailbox::acknowledge(); + Ok(()) +} + +fn kern_send(request: &kern::Message) -> Result<(), Error> { + unsafe { mailbox::send(request as *const _ as usize) } + while !mailbox::acknowledged() {} + Ok(()) +} + +fn slice_kernel_exception(exceptions: &[Option], + stack_pointers: &[eh_artiq::StackPointerBacktrace], + backtrace: &[(usize, usize)] +) -> Result { + error!("exception in kernel"); + for exception in exceptions { + error!("{:?}", exception.unwrap()); + } + error!("stack pointers: {:?}", stack_pointers); + error!("backtrace: {:?}", backtrace); + // master will only pass the exception data back to the host: + let raw_exception: Vec = Vec::new(); + let mut writer = Cursor::new(raw_exception); + match (HostKernelException { + exceptions: exceptions, + stack_pointers: stack_pointers, + backtrace: backtrace, + async_errors: 0 + }).write_to(&mut writer) { + // save last exception data to be received by master + Ok(_) => Ok(Sliceable::new(writer.into_inner())), + Err(_) => Err(Error::SubkernelIoError) + } +} + +fn pass_message_to_kernel(message: &Message) -> Result<(), Error> { + let mut reader = Cursor::new(&message.data); + let mut tag: [u8; 1] = [message.tag]; + let count = message.count; + let mut i = 0; + loop { + let slot = kern_recv_w_timeout(100, |reply| { + match reply { + &kern::RpcRecvRequest(slot) => Ok(slot), + &kern::RunException { exceptions, stack_pointers, backtrace } => { + let exception = slice_kernel_exception(&exceptions, &stack_pointers, &backtrace)?; + Err(Error::KernelException(exception)) + }, + other => unexpected!( + "expected root value slot from kernel CPU, not {:?}", other) + } + })?; + + let res = rpc::recv_return(&mut reader, &tag, slot, &|size| -> Result<_, Error> { + if size == 0 { + return Ok(0 as *mut ()) + } + kern_send(&kern::RpcRecvReply(Ok(size)))?; + Ok(kern_recv_w_timeout(100, |reply| { + match reply { + &kern::RpcRecvRequest(slot) => Ok(slot), + &kern::RunException { + exceptions, + stack_pointers, + backtrace + }=> { + let exception = slice_kernel_exception(&exceptions, &stack_pointers, &backtrace)?; + Err(Error::KernelException(exception)) + }, + other => unexpected!( + "expected nested value slot from kernel CPU, not {:?}", other) + } + })?) + }); + match res { + Ok(_) => kern_send(&kern::RpcRecvReply(Ok(0)))?, + Err(_) => unexpected!("expected valid subkernel message data") + }; + i += 1; + if i < count { + // update the tag for next read + tag[0] = reader.read_u8()?; + } else { + // should be done by then + break; + } + } + Ok(()) +} + +fn process_kern_hwreq(request: &kern::Message, rank: u8) -> Result { + match request { + &kern::RtioInitRequest => { + unsafe { + csr::drtiosat::reset_write(1); + clock::spin_us(100); + csr::drtiosat::reset_write(0); + } + kern_acknowledge() + } + + &kern::RtioDestinationStatusRequest { destination } => { + // only local destination is considered "up" + // no access to other DRTIO destinations + kern_send(&kern::RtioDestinationStatusReply { + up: destination == rank }) + } + + &kern::I2cStartRequest { busno } => { + let succeeded = i2c::start(busno as u8).is_ok(); + kern_send(&kern::I2cBasicReply { succeeded: succeeded }) + } + &kern::I2cRestartRequest { busno } => { + let succeeded = i2c::restart(busno as u8).is_ok(); + kern_send(&kern::I2cBasicReply { succeeded: succeeded }) + } + &kern::I2cStopRequest { busno } => { + let succeeded = i2c::stop(busno as u8).is_ok(); + kern_send(&kern::I2cBasicReply { succeeded: succeeded }) + } + &kern::I2cWriteRequest { busno, data } => { + match i2c::write(busno as u8, data) { + Ok(ack) => kern_send( + &kern::I2cWriteReply { succeeded: true, ack: ack }), + Err(_) => kern_send( + &kern::I2cWriteReply { succeeded: false, ack: false }) + } + } + &kern::I2cReadRequest { busno, ack } => { + match i2c::read(busno as u8, ack) { + Ok(data) => kern_send( + &kern::I2cReadReply { succeeded: true, data: data }), + Err(_) => kern_send( + &kern::I2cReadReply { succeeded: false, data: 0xff }) + } + } + &kern::I2cSwitchSelectRequest { busno, address, mask } => { + let succeeded = i2c::switch_select(busno as u8, address, mask).is_ok(); + kern_send(&kern::I2cBasicReply { succeeded: succeeded }) + } + + &kern::SpiSetConfigRequest { busno, flags, length, div, cs } => { + let succeeded = spi::set_config(busno as u8, flags, length, div, cs).is_ok(); + kern_send(&kern::SpiBasicReply { succeeded: succeeded }) + }, + &kern::SpiWriteRequest { busno, data } => { + let succeeded = spi::write(busno as u8, data).is_ok(); + kern_send(&kern::SpiBasicReply { succeeded: succeeded }) + } + &kern::SpiReadRequest { busno } => { + match spi::read(busno as u8) { + Ok(data) => kern_send( + &kern::SpiReadReply { succeeded: true, data: data }), + Err(_) => kern_send( + &kern::SpiReadReply { succeeded: false, data: 0 }) + } + } + + _ => return Ok(false) + }.and(Ok(true)) +} \ No newline at end of file diff --git a/artiq/firmware/satman/main.rs b/artiq/firmware/satman/main.rs index 4384e68d3..a44919933 100644 --- a/artiq/firmware/satman/main.rs +++ b/artiq/firmware/satman/main.rs @@ -1,4 +1,4 @@ -#![feature(never_type, panic_info_message, llvm_asm, default_alloc_error_handler)] +#![feature(never_type, panic_info_message, llvm_asm, default_alloc_error_handler, try_trait)] #![no_std] #[macro_use] @@ -9,12 +9,15 @@ extern crate board_artiq; extern crate riscv; extern crate alloc; extern crate proto_artiq; +extern crate cslice; +extern crate io; +extern crate eh; use core::convert::TryFrom; use board_misoc::{csr, ident, clock, uart_logger, i2c, pmp}; #[cfg(has_si5324)] use board_artiq::si5324; -use board_artiq::{spi, drtioaux}; +use board_artiq::{spi, drtioaux, drtio_routing}; #[cfg(soc_platform = "efc")] use board_artiq::ad9117; use proto_artiq::drtioaux_proto::{SAT_PAYLOAD_MAX_SIZE, MASTER_PAYLOAD_MAX_SIZE}; @@ -22,6 +25,7 @@ use proto_artiq::drtioaux_proto::{SAT_PAYLOAD_MAX_SIZE, MASTER_PAYLOAD_MAX_SIZE} use board_artiq::drtio_eem; use riscv::register::{mcause, mepc, mtval}; use dma::Manager as DmaManager; +use kernel::Manager as KernelManager; use analyzer::Analyzer; #[global_allocator] @@ -30,6 +34,8 @@ static mut ALLOC: alloc_list::ListAlloc = alloc_list::EMPTY; mod repeater; mod dma; mod analyzer; +mod kernel; +mod cache; fn drtiosat_reset(reset: bool) { unsafe { @@ -59,6 +65,22 @@ fn drtiosat_tsc_loaded() -> bool { } } +pub enum RtioMaster { + Drtio, + Dma, + Kernel +} + +pub fn cricon_select(master: RtioMaster) { + let val = match master { + RtioMaster::Drtio => 0, + RtioMaster::Dma => 1, + RtioMaster::Kernel => 2 + }; + unsafe { + csr::cri_con::selected_write(val); + } +} #[cfg(has_drtio_routing)] macro_rules! forward { @@ -80,8 +102,8 @@ macro_rules! forward { ($routing_table:expr, $destination:expr, $rank:expr, $repeaters:expr, $packet:expr) => {} } -fn process_aux_packet(_manager: &mut DmaManager, analyzer: &mut Analyzer, _repeaters: &mut [repeater::Repeater], - _routing_table: &mut drtio_routing::RoutingTable, _rank: &mut u8, +fn process_aux_packet(dmamgr: &mut DmaManager, analyzer: &mut Analyzer, kernelmgr: &mut KernelManager, + _repeaters: &mut [repeater::Repeater], _routing_table: &mut drtio_routing::RoutingTable, _rank: &mut u8, packet: drtioaux::Packet) -> Result<(), drtioaux::Error> { // In the code below, *_chan_sel_write takes an u8 if there are fewer than 256 channels, // and u16 otherwise; hence the `as _` conversion. @@ -101,18 +123,30 @@ fn process_aux_packet(_manager: &mut DmaManager, analyzer: &mut Analyzer, _repea drtioaux::send(0, &drtioaux::Packet::ResetAck) }, - drtioaux::Packet::DestinationStatusRequest { destination: _destination } => { + drtioaux::Packet::DestinationStatusRequest { destination } => { #[cfg(has_drtio_routing)] - let hop = _routing_table.0[_destination as usize][*_rank as usize]; + let hop = _routing_table.0[destination as usize][*_rank as usize]; #[cfg(not(has_drtio_routing))] let hop = 0; if hop == 0 { // async messages - if let Some(status) = _manager.check_state() { + if let Some(status) = dmamgr.get_status() { info!("playback done, error: {}, channel: {}, timestamp: {}", status.error, status.channel, status.timestamp); drtioaux::send(0, &drtioaux::Packet::DmaPlaybackStatus { - destination: _destination, id: status.id, error: status.error, channel: status.channel, timestamp: status.timestamp })?; + destination: destination, id: status.id, error: status.error, channel: status.channel, timestamp: status.timestamp })?; + } else if let Some(subkernel_finished) = kernelmgr.get_last_finished() { + info!("subkernel {} finished, with exception: {}", subkernel_finished.id, subkernel_finished.with_exception); + drtioaux::send(0, &drtioaux::Packet::SubkernelFinished { + id: subkernel_finished.id, with_exception: subkernel_finished.with_exception + })?; + } else if kernelmgr.message_is_ready() { + let mut data_slice: [u8; MASTER_PAYLOAD_MAX_SIZE] = [0; MASTER_PAYLOAD_MAX_SIZE]; + let meta = kernelmgr.message_get_slice(&mut data_slice).unwrap(); + drtioaux::send(0, &drtioaux::Packet::SubkernelMessage { + destination: destination, id: kernelmgr.get_current_id().unwrap(), + last: meta.last, length: meta.len as u16, data: data_slice + })?; } else { let errors; unsafe { @@ -156,7 +190,7 @@ fn process_aux_packet(_manager: &mut DmaManager, analyzer: &mut Analyzer, _repea if hop <= csr::DRTIOREP.len() { let repno = hop - 1; match _repeaters[repno].aux_forward(&drtioaux::Packet::DestinationStatusRequest { - destination: _destination + destination: destination }) { Ok(()) => (), Err(drtioaux::Error::LinkDown) => drtioaux::send(0, &drtioaux::Packet::DestinationDownReply)?, @@ -336,28 +370,80 @@ fn process_aux_packet(_manager: &mut DmaManager, analyzer: &mut Analyzer, _repea }) } - #[cfg(has_rtio_dma)] drtioaux::Packet::DmaAddTraceRequest { destination: _destination, id, last, length, trace } => { forward!(_routing_table, _destination, *_rank, _repeaters, &packet); - let succeeded = _manager.add(id, last, &trace, length as usize).is_ok(); + let succeeded = dmamgr.add(id, last, &trace, length as usize).is_ok(); drtioaux::send(0, &drtioaux::Packet::DmaAddTraceReply { succeeded: succeeded }) } - #[cfg(has_rtio_dma)] drtioaux::Packet::DmaRemoveTraceRequest { destination: _destination, id } => { forward!(_routing_table, _destination, *_rank, _repeaters, &packet); - let succeeded = _manager.erase(id).is_ok(); + let succeeded = dmamgr.erase(id).is_ok(); drtioaux::send(0, &drtioaux::Packet::DmaRemoveTraceReply { succeeded: succeeded }) } - #[cfg(has_rtio_dma)] drtioaux::Packet::DmaPlaybackRequest { destination: _destination, id, timestamp } => { forward!(_routing_table, _destination, *_rank, _repeaters, &packet); - let succeeded = _manager.playback(id, timestamp).is_ok(); + // no DMA with a running kernel + let succeeded = !kernelmgr.is_running() && dmamgr.playback(id, timestamp).is_ok(); drtioaux::send(0, &drtioaux::Packet::DmaPlaybackReply { succeeded: succeeded }) } + drtioaux::Packet::SubkernelAddDataRequest { destination: _destination, id, last, length, data } => { + forward!(_routing_table, _destination, *_rank, _repeaters, &packet); + let succeeded = kernelmgr.add(id, last, &data, length as usize).is_ok(); + drtioaux::send(0, + &drtioaux::Packet::SubkernelAddDataReply { succeeded: succeeded }) + } + drtioaux::Packet::SubkernelLoadRunRequest { destination: _destination, id, run } => { + forward!(_routing_table, _destination, *_rank, _repeaters, &packet); + let mut succeeded = kernelmgr.load(id).is_ok(); + // allow preloading a kernel with delayed run + if run { + if dmamgr.running() { + // cannot run kernel while DDMA is running + succeeded = false; + } else { + succeeded |= kernelmgr.run(id).is_ok(); + } + } + drtioaux::send(0, + &drtioaux::Packet::SubkernelLoadRunReply { succeeded: succeeded }) + } + drtioaux::Packet::SubkernelExceptionRequest { destination: _destination } => { + forward!(_routing_table, _destination, *_rank, _repeaters, &packet); + let mut data_slice: [u8; SAT_PAYLOAD_MAX_SIZE] = [0; SAT_PAYLOAD_MAX_SIZE]; + let meta = kernelmgr.exception_get_slice(&mut data_slice); + drtioaux::send(0, &drtioaux::Packet::SubkernelException { + last: meta.last, + length: meta.len, + data: data_slice, + }) + } + drtioaux::Packet::SubkernelMessage { destination, id: _id, last, length, data } => { + forward!(_routing_table, destination, *_rank, _repeaters, &packet); + kernelmgr.message_handle_incoming(last, length as usize, &data); + drtioaux::send(0, &drtioaux::Packet::SubkernelMessageAck { + destination: destination + }) + } + drtioaux::Packet::SubkernelMessageAck { destination: _destination } => { + forward!(_routing_table, _destination, *_rank, _repeaters, &packet); + if kernelmgr.message_ack_slice() { + let mut data_slice: [u8; MASTER_PAYLOAD_MAX_SIZE] = [0; MASTER_PAYLOAD_MAX_SIZE]; + if let Some(meta) = kernelmgr.message_get_slice(&mut data_slice) { + drtioaux::send(0, &drtioaux::Packet::SubkernelMessage { + destination: *_rank, id: kernelmgr.get_current_id().unwrap(), + last: meta.last, length: meta.len as u16, data: data_slice + })? + } else { + error!("Error receiving message slice"); + } + } + Ok(()) + } + _ => { warn!("received unexpected aux packet"); Ok(()) @@ -366,12 +452,12 @@ fn process_aux_packet(_manager: &mut DmaManager, analyzer: &mut Analyzer, _repea } fn process_aux_packets(dma_manager: &mut DmaManager, analyzer: &mut Analyzer, - repeaters: &mut [repeater::Repeater], + kernelmgr: &mut KernelManager, repeaters: &mut [repeater::Repeater], routing_table: &mut drtio_routing::RoutingTable, rank: &mut u8) { let result = drtioaux::recv(0).and_then(|packet| { if let Some(packet) = packet { - process_aux_packet(dma_manager, analyzer, repeaters, routing_table, rank, packet) + process_aux_packet(dma_manager, analyzer, kernelmgr, repeaters, routing_table, rank, packet) } else { Ok(()) } @@ -383,10 +469,7 @@ fn process_aux_packets(dma_manager: &mut DmaManager, analyzer: &mut Analyzer, } fn drtiosat_process_errors() { - let errors; - unsafe { - errors = csr::drtiosat::protocol_error_read(); - } + let errors = unsafe { csr::drtiosat::protocol_error_read() }; if errors & 1 != 0 { error!("received packet of an unknown type"); } @@ -394,26 +477,29 @@ fn drtiosat_process_errors() { error!("received truncated packet"); } if errors & 4 != 0 { - let destination; - unsafe { - destination = csr::drtiosat::buffer_space_timeout_dest_read(); - } + let destination = unsafe { + csr::drtiosat::buffer_space_timeout_dest_read() + }; error!("timeout attempting to get buffer space from CRI, destination=0x{:02x}", destination) } - if errors & 8 != 0 { - let channel; - let timestamp_event; - let timestamp_counter; - unsafe { - channel = csr::drtiosat::underflow_channel_read(); - timestamp_event = csr::drtiosat::underflow_timestamp_event_read() as i64; - timestamp_counter = csr::drtiosat::underflow_timestamp_counter_read() as i64; + let drtiosat_active = unsafe { csr::cri_con::selected_read() == 0 }; + if drtiosat_active { + // RTIO errors are handled by ksupport and dma manager + if errors & 8 != 0 { + let channel; + let timestamp_event; + let timestamp_counter; + unsafe { + channel = csr::drtiosat::underflow_channel_read(); + timestamp_event = csr::drtiosat::underflow_timestamp_event_read() as i64; + timestamp_counter = csr::drtiosat::underflow_timestamp_counter_read() as i64; + } + error!("write underflow, channel={}, timestamp={}, counter={}, slack={}", + channel, timestamp_event, timestamp_counter, timestamp_event-timestamp_counter); + } + if errors & 16 != 0 { + error!("write overflow"); } - error!("write underflow, channel={}, timestamp={}, counter={}, slack={}", - channel, timestamp_event, timestamp_counter, timestamp_event-timestamp_counter); - } - if errors & 16 != 0 { - error!("write overflow"); } unsafe { csr::drtiosat::protocol_error_write(errors); @@ -612,21 +698,23 @@ pub extern fn main() -> i32 { si5324::siphaser::calibrate_skew().expect("failed to calibrate skew"); } - // DMA manager created here, so when link is dropped, all DMA traces - // are cleared out for a clean slate on subsequent connections, - // without a manual intervention. + // various managers created here, so when link is dropped, DMA traces, + // analyzer logs, kernels are cleared and/or stopped for a clean slate + // on subsequent connections, without a manual intervention. let mut dma_manager = DmaManager::new(); - - // Reset the analyzer as well. let mut analyzer = Analyzer::new(); + let mut kernelmgr = KernelManager::new(); + cricon_select(RtioMaster::Drtio); drtioaux::reset(0); drtiosat_reset(false); drtiosat_reset_phy(false); while drtiosat_link_rx_up() { drtiosat_process_errors(); - process_aux_packets(&mut dma_manager, &mut analyzer, &mut repeaters, &mut routing_table, &mut rank); + process_aux_packets(&mut dma_manager, &mut analyzer, + &mut kernelmgr, &mut repeaters, + &mut routing_table, &mut rank); for rep in repeaters.iter_mut() { rep.service(&routing_table, rank); } @@ -649,6 +737,7 @@ pub extern fn main() -> i32 { error!("aux packet error: {}", e); } } + kernelmgr.process_kern_requests(rank); } drtiosat_reset_phy(true); diff --git a/artiq/firmware/satman/satman.ld b/artiq/firmware/satman/satman.ld index f58ef38d8..c188dc3ec 100644 --- a/artiq/firmware/satman/satman.ld +++ b/artiq/firmware/satman/satman.ld @@ -22,6 +22,19 @@ SECTIONS __eh_frame_end = .; } > main_ram + .eh_frame_hdr : + { + KEEP(*(.eh_frame_hdr)) + } > main_ram + + __eh_frame_hdr_start = SIZEOF(.eh_frame_hdr) > 0 ? ADDR(.eh_frame_hdr) : 0; + __eh_frame_hdr_end = SIZEOF(.eh_frame_hdr) > 0 ? . : 0; + + .gcc_except_table : + { + *(.gcc_except_table) + } > main_ram + /* https://sourceware.org/bugzilla/show_bug.cgi?id=20475 */ .got : { @@ -68,11 +81,11 @@ SECTIONS _fstack = . - 16; } > main_ram - /* 64MB heap for alloc use */ + /* remainder of 64MB for heap for alloc use */ .heap (NOLOAD) : ALIGN(16) { _fheap = .; - . = . + 0x4000000; + . = 0x44000000; // not to overwrite RPC queue _eheap = .; } > main_ram }