runtime DMA: bring architecture close to mainline
parent
908dfc780e
commit
963af1f04f
|
@ -27,7 +27,8 @@ use crate::pl;
|
||||||
use crate::{analyzer, kernel, mgmt, moninj,
|
use crate::{analyzer, kernel, mgmt, moninj,
|
||||||
proto_async::*,
|
proto_async::*,
|
||||||
rpc,
|
rpc,
|
||||||
rtio_mgt::{self, resolve_channel_name}};
|
rtio_mgt::{self, resolve_channel_name},
|
||||||
|
rtio_dma};
|
||||||
|
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
pub enum Error {
|
pub enum Error {
|
||||||
|
@ -79,7 +80,7 @@ enum Reply {
|
||||||
}
|
}
|
||||||
|
|
||||||
static CACHE_STORE: Mutex<BTreeMap<String, Vec<i32>>> = Mutex::new(BTreeMap::new());
|
static CACHE_STORE: Mutex<BTreeMap<String, Vec<i32>>> = Mutex::new(BTreeMap::new());
|
||||||
static DMA_RECORD_STORE: Mutex<BTreeMap<String, (Vec<u8>, i64)>> = Mutex::new(BTreeMap::new());
|
|
||||||
|
|
||||||
async fn write_header(stream: &TcpStream, reply: Reply) -> Result<()> {
|
async fn write_header(stream: &TcpStream, reply: Reply) -> Result<()> {
|
||||||
stream
|
stream
|
||||||
|
@ -319,16 +320,14 @@ async fn handle_run_kernel(
|
||||||
.await;
|
.await;
|
||||||
}
|
}
|
||||||
kernel::Message::DmaPutRequest(recorder) => {
|
kernel::Message::DmaPutRequest(recorder) => {
|
||||||
DMA_RECORD_STORE
|
rtio_dma::put_record(recorder);
|
||||||
.lock()
|
|
||||||
.insert(recorder.name, (recorder.buffer, recorder.duration));
|
|
||||||
}
|
}
|
||||||
kernel::Message::DmaEraseRequest(name) => {
|
kernel::Message::DmaEraseRequest(name) => {
|
||||||
// prevent possible OOM when we have large DMA record replacement.
|
// prevent possible OOM when we have large DMA record replacement.
|
||||||
DMA_RECORD_STORE.lock().remove(&name);
|
rtio_dma::erase(name);
|
||||||
}
|
}
|
||||||
kernel::Message::DmaGetRequest(name) => {
|
kernel::Message::DmaGetRequest(name) => {
|
||||||
let result = DMA_RECORD_STORE.lock().get(&name).map(|v| v.clone());
|
let result = rtio_dma::retrieve(name);
|
||||||
control
|
control
|
||||||
.borrow_mut()
|
.borrow_mut()
|
||||||
.tx
|
.tx
|
||||||
|
|
|
@ -1,14 +1,11 @@
|
||||||
use alloc::{boxed::Box, string::String, vec::Vec};
|
use alloc::{string::String, vec::Vec};
|
||||||
use core::mem;
|
use core::mem;
|
||||||
|
|
||||||
use cslice::CSlice;
|
use cslice::CSlice;
|
||||||
use libcortex_a9::cache::dcci_slice;
|
|
||||||
|
|
||||||
use super::{Message, KERNEL_CHANNEL_0TO1, KERNEL_CHANNEL_1TO0, KERNEL_IMAGE};
|
use super::{Message, KERNEL_CHANNEL_0TO1, KERNEL_CHANNEL_1TO0, KERNEL_IMAGE};
|
||||||
use crate::{artiq_raise, pl::csr, rtio};
|
use crate::{artiq_raise, pl::csr, rtio};
|
||||||
|
|
||||||
const ALIGNMENT: usize = 16 * 8;
|
|
||||||
|
|
||||||
#[repr(C)]
|
#[repr(C)]
|
||||||
pub struct DmaTrace {
|
pub struct DmaTrace {
|
||||||
duration: i64,
|
duration: i64,
|
||||||
|
@ -151,20 +148,7 @@ pub extern "C" fn dma_retrieve(name: CSlice<u8>) -> DmaTrace {
|
||||||
}
|
}
|
||||||
match unsafe { KERNEL_CHANNEL_0TO1.as_mut().unwrap() }.recv() {
|
match unsafe { KERNEL_CHANNEL_0TO1.as_mut().unwrap() }.recv() {
|
||||||
Message::DmaGetReply(None) => (),
|
Message::DmaGetReply(None) => (),
|
||||||
Message::DmaGetReply(Some((mut v, duration))) => {
|
Message::DmaGetReply(Some((address, duration))) => {
|
||||||
v.reserve(ALIGNMENT - 1);
|
|
||||||
let original_length = v.len();
|
|
||||||
let padding = ALIGNMENT - v.as_ptr() as usize % ALIGNMENT;
|
|
||||||
let padding = if padding == ALIGNMENT { 0 } else { padding };
|
|
||||||
for _ in 0..padding {
|
|
||||||
v.push(0);
|
|
||||||
}
|
|
||||||
// trailing zero to indicate end of buffer
|
|
||||||
v.push(0);
|
|
||||||
v.copy_within(0..original_length, padding);
|
|
||||||
dcci_slice(&v);
|
|
||||||
let v = Box::new(v);
|
|
||||||
let address = Box::into_raw(v) as *mut Vec<u8> as i32;
|
|
||||||
return DmaTrace { address, duration };
|
return DmaTrace { address, duration };
|
||||||
}
|
}
|
||||||
_ => panic!("Expected DmaGetReply after DmaGetRequest!"),
|
_ => panic!("Expected DmaGetReply after DmaGetRequest!"),
|
||||||
|
@ -175,11 +159,6 @@ pub extern "C" fn dma_retrieve(name: CSlice<u8>) -> DmaTrace {
|
||||||
|
|
||||||
pub extern "C" fn dma_playback(timestamp: i64, ptr: i32) {
|
pub extern "C" fn dma_playback(timestamp: i64, ptr: i32) {
|
||||||
unsafe {
|
unsafe {
|
||||||
let v = Box::from_raw(ptr as *mut Vec<u8>);
|
|
||||||
let padding = ALIGNMENT - v.as_ptr() as usize % ALIGNMENT;
|
|
||||||
let padding = if padding == ALIGNMENT { 0 } else { padding };
|
|
||||||
let ptr = v.as_ptr().add(padding) as i32;
|
|
||||||
|
|
||||||
csr::rtio_dma::base_address_write(ptr as u32);
|
csr::rtio_dma::base_address_write(ptr as u32);
|
||||||
csr::rtio_dma::time_offset_write(timestamp as u64);
|
csr::rtio_dma::time_offset_write(timestamp as u64);
|
||||||
|
|
||||||
|
@ -188,9 +167,6 @@ pub extern "C" fn dma_playback(timestamp: i64, ptr: i32) {
|
||||||
while csr::rtio_dma::enable_read() != 0 {}
|
while csr::rtio_dma::enable_read() != 0 {}
|
||||||
csr::cri_con::selected_write(0);
|
csr::cri_con::selected_write(0);
|
||||||
|
|
||||||
// leave the handle as we may try to do playback for another time.
|
|
||||||
mem::forget(v);
|
|
||||||
|
|
||||||
let error = csr::rtio_dma::error_read();
|
let error = csr::rtio_dma::error_read();
|
||||||
if error != 0 {
|
if error != 0 {
|
||||||
let timestamp = csr::rtio_dma::error_timestamp_read();
|
let timestamp = csr::rtio_dma::error_timestamp_read();
|
||||||
|
|
|
@ -52,7 +52,7 @@ pub enum Message {
|
||||||
DmaPutRequest(DmaRecorder),
|
DmaPutRequest(DmaRecorder),
|
||||||
DmaEraseRequest(String),
|
DmaEraseRequest(String),
|
||||||
DmaGetRequest(String),
|
DmaGetRequest(String),
|
||||||
DmaGetReply(Option<(Vec<u8>, i64)>),
|
DmaGetReply(Option<(i32, i64)>),
|
||||||
|
|
||||||
#[cfg(has_drtio)]
|
#[cfg(has_drtio)]
|
||||||
UpDestinationsRequest(i32),
|
UpDestinationsRequest(i32),
|
||||||
|
|
|
@ -47,6 +47,7 @@ mod rtio;
|
||||||
mod rtio;
|
mod rtio;
|
||||||
mod rtio_clocking;
|
mod rtio_clocking;
|
||||||
mod rtio_mgt;
|
mod rtio_mgt;
|
||||||
|
mod rtio_dma;
|
||||||
|
|
||||||
static mut SEEN_ASYNC_ERRORS: u8 = 0;
|
static mut SEEN_ASYNC_ERRORS: u8 = 0;
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,38 @@
|
||||||
|
|
||||||
|
use alloc::{collections::BTreeMap, string::String, vec::Vec};
|
||||||
|
use libcortex_a9::{mutex::Mutex, cache::dcci_slice};
|
||||||
|
use crate::kernel::DmaRecorder;
|
||||||
|
|
||||||
|
const ALIGNMENT: usize = 16 * 8;
|
||||||
|
|
||||||
|
static DMA_RECORD_STORE: Mutex<BTreeMap<String, (u32, Vec<u8>, i64)>> = Mutex::new(BTreeMap::new());
|
||||||
|
|
||||||
|
|
||||||
|
pub fn put_record(mut recorder: DmaRecorder) {
|
||||||
|
recorder.buffer.reserve(ALIGNMENT - 1);
|
||||||
|
let original_length = recorder.buffer.len();
|
||||||
|
let padding = ALIGNMENT - recorder.buffer.as_ptr() as usize % ALIGNMENT;
|
||||||
|
let padding = if padding == ALIGNMENT { 0 } else { padding };
|
||||||
|
for _ in 0..padding {
|
||||||
|
recorder.buffer.push(0);
|
||||||
|
}
|
||||||
|
// trailing zero to indicate end of buffer
|
||||||
|
recorder.buffer.push(0);
|
||||||
|
recorder.buffer.copy_within(0..original_length, padding);
|
||||||
|
dcci_slice(&recorder.buffer);
|
||||||
|
|
||||||
|
let ptr = recorder.buffer[padding..].as_ptr() as u32;
|
||||||
|
|
||||||
|
DMA_RECORD_STORE
|
||||||
|
.lock()
|
||||||
|
.insert(recorder.name, (ptr, recorder.buffer, recorder.duration));
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn erase(name: String) {
|
||||||
|
DMA_RECORD_STORE.lock().remove(&name);
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn retrieve(name: String) -> Option<(i32, i64)> {
|
||||||
|
let (ptr, _v, duration) = DMA_RECORD_STORE.lock().get(&name)?.clone();
|
||||||
|
Some((ptr as i32, duration))
|
||||||
|
}
|
|
@ -389,6 +389,70 @@ pub mod drtio {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// pub fn ddma_upload_trace(aux_mutex: Rc<Mutex<bool>>,
|
||||||
|
// routing_table: &drtio_routing::RoutingTable,
|
||||||
|
// id: u32, destination: u8, trace: &Vec<u8>) -> Result<(), &'static str> {
|
||||||
|
// let linkno = routing_table.0[destination as usize][0] - 1;
|
||||||
|
// let mut i = 0;
|
||||||
|
// while i < trace.len() {
|
||||||
|
// let mut trace_slice: [u8; DMA_TRACE_MAX_SIZE] = [0; DMA_TRACE_MAX_SIZE];
|
||||||
|
// let len: usize = if i + DMA_TRACE_MAX_SIZE < trace.len() { DMA_TRACE_MAX_SIZE } else { trace.len() - i } as usize;
|
||||||
|
// let last = i + len == trace.len();
|
||||||
|
// trace_slice[..len].clone_from_slice(&trace[i..i+len]);
|
||||||
|
// i += len;
|
||||||
|
// let reply = aux_transact(io, aux_mutex, linkno,
|
||||||
|
// &drtioaux::Packet::DmaAddTraceRequest {
|
||||||
|
// id: id, destination: destination, last: last, length: len as u16, trace: trace_slice});
|
||||||
|
// match reply {
|
||||||
|
// Ok(drtioaux::Packet::DmaAddTraceReply { succeeded: true }) => (),
|
||||||
|
// Ok(drtioaux::Packet::DmaAddTraceReply { succeeded: false }) => {
|
||||||
|
// return Err("error adding trace on satellite"); },
|
||||||
|
// Ok(_) => { return Err("adding DMA trace failed, unexpected aux packet"); },
|
||||||
|
// Err(_) => { return Err("adding DMA trace failed, aux error"); }
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
// Ok(())
|
||||||
|
// }
|
||||||
|
|
||||||
|
|
||||||
|
// pub fn ddma_send_erase(io: &Io, aux_mutex: &Mutex,
|
||||||
|
// routing_table: &drtio_routing::RoutingTable,
|
||||||
|
// id: u32, destination: u8) -> Result<(), &'static str> {
|
||||||
|
// let linkno = routing_table.0[destination as usize][0] - 1;
|
||||||
|
// let reply = aux_transact(io, aux_mutex, linkno,
|
||||||
|
// &drtioaux::Packet::DmaRemoveTraceRequest { id: id, destination: destination });
|
||||||
|
// match reply {
|
||||||
|
// Ok(drtioaux::Packet::DmaRemoveTraceReply { succeeded: true }) => Ok(()),
|
||||||
|
// Ok(drtioaux::Packet::DmaRemoveTraceReply { succeeded: false }) => Err("satellite DMA erase error"),
|
||||||
|
// Ok(_) => Err("adding trace failed, unexpected aux packet"),
|
||||||
|
// Err(_) => Err("erasing trace failed, aux error")
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
|
||||||
|
// pub fn ddma_send_playback(io: &Io, aux_mutex: &Mutex,
|
||||||
|
// routing_table: &drtio_routing::RoutingTable,
|
||||||
|
// ddma_mutex: &Mutex, id: u32, destination: u8, timestamp: u64) -> Result<(), &'static str> {
|
||||||
|
// let linkno = routing_table.0[destination as usize][0] - 1;
|
||||||
|
// let _lock = aux_mutex.lock(io).unwrap();
|
||||||
|
// drtioaux::send(linkno, &drtioaux::Packet::DmaPlaybackRequest{
|
||||||
|
// id: id, destination: destination, timestamp: timestamp }).unwrap();
|
||||||
|
// loop {
|
||||||
|
// let reply = recv_aux_timeout(io, linkno, 200);
|
||||||
|
// match reply {
|
||||||
|
// Ok(drtioaux::Packet::DmaPlaybackReply { succeeded: true }) => { return Ok(()) },
|
||||||
|
// Ok(drtioaux::Packet::DmaPlaybackReply { succeeded: false }) => {
|
||||||
|
// return Err("error on DMA playback request") },
|
||||||
|
// // in case we received status from another destination
|
||||||
|
// // but we want to get DmaPlaybackReply anyway, thus the loop
|
||||||
|
// Ok(drtioaux::Packet::DmaPlaybackStatus { id, destination, error, channel, timestamp }) => {
|
||||||
|
// remote_dma::playback_done(io, ddma_mutex, id, destination, error, channel, timestamp);
|
||||||
|
// },
|
||||||
|
// Ok(_) => { return Err("received unexpected aux packet while DMA playback") },
|
||||||
|
// Err(_) => { return Err("aux error on DMA playback") }
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
// }
|
||||||
}
|
}
|
||||||
|
|
||||||
fn read_device_map(cfg: &Config) -> BTreeMap<u32, String> {
|
fn read_device_map(cfg: &Config) -> BTreeMap<u32, String> {
|
||||||
|
|
Loading…
Reference in New Issue