From 963af1f04fee1f9372424a5b58ce31ceebfbadfd Mon Sep 17 00:00:00 2001 From: mwojcik Date: Thu, 23 Mar 2023 15:41:08 +0800 Subject: [PATCH] runtime DMA: bring architecture close to mainline --- src/runtime/src/comms.rs | 13 ++++--- src/runtime/src/kernel/dma.rs | 28 ++------------- src/runtime/src/kernel/mod.rs | 2 +- src/runtime/src/main.rs | 1 + src/runtime/src/rtio_dma.rs | 38 +++++++++++++++++++++ src/runtime/src/rtio_mgt.rs | 64 +++++++++++++++++++++++++++++++++++ 6 files changed, 112 insertions(+), 34 deletions(-) create mode 100644 src/runtime/src/rtio_dma.rs diff --git a/src/runtime/src/comms.rs b/src/runtime/src/comms.rs index 1465d77..5b53f88 100644 --- a/src/runtime/src/comms.rs +++ b/src/runtime/src/comms.rs @@ -27,7 +27,8 @@ use crate::pl; use crate::{analyzer, kernel, mgmt, moninj, proto_async::*, rpc, - rtio_mgt::{self, resolve_channel_name}}; + rtio_mgt::{self, resolve_channel_name}, + rtio_dma}; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum Error { @@ -79,7 +80,7 @@ enum Reply { } static CACHE_STORE: Mutex>> = Mutex::new(BTreeMap::new()); -static DMA_RECORD_STORE: Mutex, i64)>> = Mutex::new(BTreeMap::new()); + async fn write_header(stream: &TcpStream, reply: Reply) -> Result<()> { stream @@ -319,16 +320,14 @@ async fn handle_run_kernel( .await; } kernel::Message::DmaPutRequest(recorder) => { - DMA_RECORD_STORE - .lock() - .insert(recorder.name, (recorder.buffer, recorder.duration)); + rtio_dma::put_record(recorder); } kernel::Message::DmaEraseRequest(name) => { // prevent possible OOM when we have large DMA record replacement. - DMA_RECORD_STORE.lock().remove(&name); + rtio_dma::erase(name); } kernel::Message::DmaGetRequest(name) => { - let result = DMA_RECORD_STORE.lock().get(&name).map(|v| v.clone()); + let result = rtio_dma::retrieve(name); control .borrow_mut() .tx diff --git a/src/runtime/src/kernel/dma.rs b/src/runtime/src/kernel/dma.rs index d4c473a..ed3d492 100644 --- a/src/runtime/src/kernel/dma.rs +++ b/src/runtime/src/kernel/dma.rs @@ -1,14 +1,11 @@ -use alloc::{boxed::Box, string::String, vec::Vec}; +use alloc::{string::String, vec::Vec}; use core::mem; use cslice::CSlice; -use libcortex_a9::cache::dcci_slice; use super::{Message, KERNEL_CHANNEL_0TO1, KERNEL_CHANNEL_1TO0, KERNEL_IMAGE}; use crate::{artiq_raise, pl::csr, rtio}; -const ALIGNMENT: usize = 16 * 8; - #[repr(C)] pub struct DmaTrace { duration: i64, @@ -151,20 +148,7 @@ pub extern "C" fn dma_retrieve(name: CSlice) -> DmaTrace { } match unsafe { KERNEL_CHANNEL_0TO1.as_mut().unwrap() }.recv() { Message::DmaGetReply(None) => (), - Message::DmaGetReply(Some((mut v, duration))) => { - v.reserve(ALIGNMENT - 1); - let original_length = v.len(); - let padding = ALIGNMENT - v.as_ptr() as usize % ALIGNMENT; - let padding = if padding == ALIGNMENT { 0 } else { padding }; - for _ in 0..padding { - v.push(0); - } - // trailing zero to indicate end of buffer - v.push(0); - v.copy_within(0..original_length, padding); - dcci_slice(&v); - let v = Box::new(v); - let address = Box::into_raw(v) as *mut Vec as i32; + Message::DmaGetReply(Some((address, duration))) => { return DmaTrace { address, duration }; } _ => panic!("Expected DmaGetReply after DmaGetRequest!"), @@ -175,11 +159,6 @@ pub extern "C" fn dma_retrieve(name: CSlice) -> DmaTrace { pub extern "C" fn dma_playback(timestamp: i64, ptr: i32) { unsafe { - let v = Box::from_raw(ptr as *mut Vec); - let padding = ALIGNMENT - v.as_ptr() as usize % ALIGNMENT; - let padding = if padding == ALIGNMENT { 0 } else { padding }; - let ptr = v.as_ptr().add(padding) as i32; - csr::rtio_dma::base_address_write(ptr as u32); csr::rtio_dma::time_offset_write(timestamp as u64); @@ -188,9 +167,6 @@ pub extern "C" fn dma_playback(timestamp: i64, ptr: i32) { while csr::rtio_dma::enable_read() != 0 {} csr::cri_con::selected_write(0); - // leave the handle as we may try to do playback for another time. - mem::forget(v); - let error = csr::rtio_dma::error_read(); if error != 0 { let timestamp = csr::rtio_dma::error_timestamp_read(); diff --git a/src/runtime/src/kernel/mod.rs b/src/runtime/src/kernel/mod.rs index 9f88297..4e00888 100644 --- a/src/runtime/src/kernel/mod.rs +++ b/src/runtime/src/kernel/mod.rs @@ -52,7 +52,7 @@ pub enum Message { DmaPutRequest(DmaRecorder), DmaEraseRequest(String), DmaGetRequest(String), - DmaGetReply(Option<(Vec, i64)>), + DmaGetReply(Option<(i32, i64)>), #[cfg(has_drtio)] UpDestinationsRequest(i32), diff --git a/src/runtime/src/main.rs b/src/runtime/src/main.rs index 5eefd0b..6bcab9b 100644 --- a/src/runtime/src/main.rs +++ b/src/runtime/src/main.rs @@ -47,6 +47,7 @@ mod rtio; mod rtio; mod rtio_clocking; mod rtio_mgt; +mod rtio_dma; static mut SEEN_ASYNC_ERRORS: u8 = 0; diff --git a/src/runtime/src/rtio_dma.rs b/src/runtime/src/rtio_dma.rs new file mode 100644 index 0000000..8555a7e --- /dev/null +++ b/src/runtime/src/rtio_dma.rs @@ -0,0 +1,38 @@ + +use alloc::{collections::BTreeMap, string::String, vec::Vec}; +use libcortex_a9::{mutex::Mutex, cache::dcci_slice}; +use crate::kernel::DmaRecorder; + +const ALIGNMENT: usize = 16 * 8; + +static DMA_RECORD_STORE: Mutex, i64)>> = Mutex::new(BTreeMap::new()); + + +pub fn put_record(mut recorder: DmaRecorder) { + recorder.buffer.reserve(ALIGNMENT - 1); + let original_length = recorder.buffer.len(); + let padding = ALIGNMENT - recorder.buffer.as_ptr() as usize % ALIGNMENT; + let padding = if padding == ALIGNMENT { 0 } else { padding }; + for _ in 0..padding { + recorder.buffer.push(0); + } + // trailing zero to indicate end of buffer + recorder.buffer.push(0); + recorder.buffer.copy_within(0..original_length, padding); + dcci_slice(&recorder.buffer); + + let ptr = recorder.buffer[padding..].as_ptr() as u32; + + DMA_RECORD_STORE + .lock() + .insert(recorder.name, (ptr, recorder.buffer, recorder.duration)); +} + +pub fn erase(name: String) { + DMA_RECORD_STORE.lock().remove(&name); +} + +pub fn retrieve(name: String) -> Option<(i32, i64)> { + let (ptr, _v, duration) = DMA_RECORD_STORE.lock().get(&name)?.clone(); + Some((ptr as i32, duration)) +} \ No newline at end of file diff --git a/src/runtime/src/rtio_mgt.rs b/src/runtime/src/rtio_mgt.rs index 9a75d20..315ab73 100644 --- a/src/runtime/src/rtio_mgt.rs +++ b/src/runtime/src/rtio_mgt.rs @@ -389,6 +389,70 @@ pub mod drtio { } } } + + // pub fn ddma_upload_trace(aux_mutex: Rc>, + // routing_table: &drtio_routing::RoutingTable, + // id: u32, destination: u8, trace: &Vec) -> Result<(), &'static str> { + // let linkno = routing_table.0[destination as usize][0] - 1; + // let mut i = 0; + // while i < trace.len() { + // let mut trace_slice: [u8; DMA_TRACE_MAX_SIZE] = [0; DMA_TRACE_MAX_SIZE]; + // let len: usize = if i + DMA_TRACE_MAX_SIZE < trace.len() { DMA_TRACE_MAX_SIZE } else { trace.len() - i } as usize; + // let last = i + len == trace.len(); + // trace_slice[..len].clone_from_slice(&trace[i..i+len]); + // i += len; + // let reply = aux_transact(io, aux_mutex, linkno, + // &drtioaux::Packet::DmaAddTraceRequest { + // id: id, destination: destination, last: last, length: len as u16, trace: trace_slice}); + // match reply { + // Ok(drtioaux::Packet::DmaAddTraceReply { succeeded: true }) => (), + // Ok(drtioaux::Packet::DmaAddTraceReply { succeeded: false }) => { + // return Err("error adding trace on satellite"); }, + // Ok(_) => { return Err("adding DMA trace failed, unexpected aux packet"); }, + // Err(_) => { return Err("adding DMA trace failed, aux error"); } + // } + // } + // Ok(()) + // } + + + // pub fn ddma_send_erase(io: &Io, aux_mutex: &Mutex, + // routing_table: &drtio_routing::RoutingTable, + // id: u32, destination: u8) -> Result<(), &'static str> { + // let linkno = routing_table.0[destination as usize][0] - 1; + // let reply = aux_transact(io, aux_mutex, linkno, + // &drtioaux::Packet::DmaRemoveTraceRequest { id: id, destination: destination }); + // match reply { + // Ok(drtioaux::Packet::DmaRemoveTraceReply { succeeded: true }) => Ok(()), + // Ok(drtioaux::Packet::DmaRemoveTraceReply { succeeded: false }) => Err("satellite DMA erase error"), + // Ok(_) => Err("adding trace failed, unexpected aux packet"), + // Err(_) => Err("erasing trace failed, aux error") + // } + // } + + // pub fn ddma_send_playback(io: &Io, aux_mutex: &Mutex, + // routing_table: &drtio_routing::RoutingTable, + // ddma_mutex: &Mutex, id: u32, destination: u8, timestamp: u64) -> Result<(), &'static str> { + // let linkno = routing_table.0[destination as usize][0] - 1; + // let _lock = aux_mutex.lock(io).unwrap(); + // drtioaux::send(linkno, &drtioaux::Packet::DmaPlaybackRequest{ + // id: id, destination: destination, timestamp: timestamp }).unwrap(); + // loop { + // let reply = recv_aux_timeout(io, linkno, 200); + // match reply { + // Ok(drtioaux::Packet::DmaPlaybackReply { succeeded: true }) => { return Ok(()) }, + // Ok(drtioaux::Packet::DmaPlaybackReply { succeeded: false }) => { + // return Err("error on DMA playback request") }, + // // in case we received status from another destination + // // but we want to get DmaPlaybackReply anyway, thus the loop + // Ok(drtioaux::Packet::DmaPlaybackStatus { id, destination, error, channel, timestamp }) => { + // remote_dma::playback_done(io, ddma_mutex, id, destination, error, channel, timestamp); + // }, + // Ok(_) => { return Err("received unexpected aux packet while DMA playback") }, + // Err(_) => { return Err("aux error on DMA playback") } + // } + // } + // } } fn read_device_map(cfg: &Config) -> BTreeMap {