2020-07-21 16:57:14 +08:00
|
|
|
use crate::{
|
|
|
|
pl::csr,
|
|
|
|
artiq_raise,
|
|
|
|
rtio,
|
|
|
|
};
|
2020-08-05 12:25:48 +08:00
|
|
|
use alloc::{vec::Vec, string::String, boxed::Box};
|
2020-07-21 16:57:14 +08:00
|
|
|
use cslice::CSlice;
|
2020-08-05 12:25:48 +08:00
|
|
|
use super::{KERNEL_IMAGE, KERNEL_CHANNEL_0TO1, KERNEL_CHANNEL_1TO0, Message};
|
2020-07-21 16:57:14 +08:00
|
|
|
use core::mem;
|
|
|
|
|
2020-08-05 12:25:48 +08:00
|
|
|
use libcortex_a9::cache::dcci_slice;
|
2020-07-21 16:57:14 +08:00
|
|
|
|
|
|
|
const ALIGNMENT: usize = 16 * 8;
|
|
|
|
|
|
|
|
#[repr(C)]
|
|
|
|
pub struct DmaTrace {
|
|
|
|
duration: i64,
|
|
|
|
address: i32,
|
|
|
|
}
|
|
|
|
|
2020-08-05 12:25:48 +08:00
|
|
|
#[derive(Clone, Debug)]
|
|
|
|
pub struct DmaRecorder {
|
|
|
|
pub name: String,
|
|
|
|
pub buffer: Vec<u8>,
|
|
|
|
pub duration: i64,
|
2020-07-21 16:57:14 +08:00
|
|
|
}
|
|
|
|
|
2020-08-05 12:25:48 +08:00
|
|
|
static mut RECORDER: Option<DmaRecorder> = None;
|
|
|
|
|
|
|
|
pub unsafe fn init_dma_recorder() {
|
|
|
|
// as static would remain after restart, we have to reset it,
|
|
|
|
// without running its destructor.
|
|
|
|
mem::forget(mem::replace(&mut RECORDER, None));
|
|
|
|
}
|
2020-07-21 16:57:14 +08:00
|
|
|
|
2020-08-05 12:25:48 +08:00
|
|
|
pub extern fn dma_record_start(name: CSlice<u8>) {
|
|
|
|
let name = String::from_utf8(name.as_ref().to_vec()).unwrap();
|
2020-08-24 16:18:31 +08:00
|
|
|
unsafe {
|
|
|
|
KERNEL_CHANNEL_1TO0.as_mut().unwrap().send(Message::DmaEraseRequest(name.clone()));
|
|
|
|
}
|
2020-07-21 16:57:14 +08:00
|
|
|
unsafe {
|
2020-08-05 12:25:48 +08:00
|
|
|
if RECORDER.is_some() {
|
2020-07-21 16:57:14 +08:00
|
|
|
artiq_raise!("DMAError", "DMA is already recording")
|
|
|
|
}
|
|
|
|
|
2020-07-24 12:24:01 +08:00
|
|
|
let library = KERNEL_IMAGE.as_ref().unwrap();
|
2020-07-21 16:57:14 +08:00
|
|
|
library.rebind(b"rtio_output",
|
|
|
|
dma_record_output as *const ()).unwrap();
|
|
|
|
library.rebind(b"rtio_output_wide",
|
|
|
|
dma_record_output_wide as *const ()).unwrap();
|
|
|
|
|
2020-08-05 12:25:48 +08:00
|
|
|
RECORDER = Some(DmaRecorder {
|
|
|
|
name,
|
|
|
|
buffer: Vec::new(),
|
|
|
|
duration: 0,
|
|
|
|
});
|
2020-07-21 16:57:14 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-08-06 11:11:32 +08:00
|
|
|
pub extern fn dma_record_stop(duration: i64) {
|
2020-07-21 16:57:14 +08:00
|
|
|
unsafe {
|
2020-08-05 12:25:48 +08:00
|
|
|
if RECORDER.is_none() {
|
2020-07-21 16:57:14 +08:00
|
|
|
artiq_raise!("DMAError", "DMA is not recording")
|
|
|
|
}
|
|
|
|
|
2020-07-24 12:24:01 +08:00
|
|
|
let library = KERNEL_IMAGE.as_ref().unwrap();
|
2020-07-21 16:57:14 +08:00
|
|
|
library.rebind(b"rtio_output",
|
|
|
|
rtio::output as *const ()).unwrap();
|
|
|
|
library.rebind(b"rtio_output_wide",
|
|
|
|
rtio::output_wide as *const ()).unwrap();
|
|
|
|
|
2020-08-06 11:11:32 +08:00
|
|
|
let mut recorder = RECORDER.take().unwrap();
|
|
|
|
recorder.duration = duration;
|
2020-08-24 16:18:31 +08:00
|
|
|
KERNEL_CHANNEL_1TO0.as_mut().unwrap().send(
|
2020-08-06 11:11:32 +08:00
|
|
|
Message::DmaPutRequest(recorder)
|
2020-08-05 12:25:48 +08:00
|
|
|
);
|
2020-07-21 16:57:14 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
#[inline(always)]
|
|
|
|
unsafe fn dma_record_output_prepare(timestamp: i64, target: i32,
|
2020-08-05 12:25:48 +08:00
|
|
|
words: usize) {
|
2020-07-21 16:57:14 +08:00
|
|
|
// See gateware/rtio/dma.py.
|
|
|
|
const HEADER_LENGTH: usize = /*length*/1 + /*channel*/3 + /*timestamp*/8 + /*address*/1;
|
|
|
|
let length = HEADER_LENGTH + /*data*/words * 4;
|
|
|
|
|
2020-08-05 12:25:48 +08:00
|
|
|
let buffer = &mut RECORDER.as_mut().unwrap().buffer;
|
|
|
|
buffer.reserve(length);
|
|
|
|
buffer.extend_from_slice(&[
|
2020-07-21 16:57:14 +08:00
|
|
|
(length >> 0) as u8,
|
|
|
|
(target >> 8) as u8,
|
|
|
|
(target >> 16) as u8,
|
|
|
|
(target >> 24) as u8,
|
|
|
|
(timestamp >> 0) as u8,
|
|
|
|
(timestamp >> 8) as u8,
|
|
|
|
(timestamp >> 16) as u8,
|
|
|
|
(timestamp >> 24) as u8,
|
|
|
|
(timestamp >> 32) as u8,
|
|
|
|
(timestamp >> 40) as u8,
|
|
|
|
(timestamp >> 48) as u8,
|
|
|
|
(timestamp >> 56) as u8,
|
|
|
|
(target >> 0) as u8,
|
|
|
|
]);
|
|
|
|
}
|
|
|
|
|
|
|
|
pub extern fn dma_record_output(target: i32, word: i32) {
|
|
|
|
unsafe {
|
2020-08-04 13:14:00 +08:00
|
|
|
let timestamp = rtio::now_mu();
|
2020-08-05 12:25:48 +08:00
|
|
|
dma_record_output_prepare(timestamp, target, 1);
|
|
|
|
RECORDER.as_mut().unwrap().buffer.extend_from_slice(&[
|
2020-07-21 16:57:14 +08:00
|
|
|
(word >> 0) as u8,
|
|
|
|
(word >> 8) as u8,
|
|
|
|
(word >> 16) as u8,
|
|
|
|
(word >> 24) as u8,
|
|
|
|
]);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
pub extern fn dma_record_output_wide(target: i32, words: CSlice<i32>) {
|
|
|
|
assert!(words.len() <= 16); // enforce the hardware limit
|
|
|
|
|
|
|
|
unsafe {
|
2020-08-04 13:14:00 +08:00
|
|
|
let timestamp = rtio::now_mu();
|
2020-08-05 12:25:48 +08:00
|
|
|
dma_record_output_prepare(timestamp, target, words.len());
|
|
|
|
let buffer = &mut RECORDER.as_mut().unwrap().buffer;
|
2020-07-21 16:57:14 +08:00
|
|
|
for word in words.as_ref().iter() {
|
2020-08-05 12:25:48 +08:00
|
|
|
buffer.extend_from_slice(&[
|
2020-07-21 16:57:14 +08:00
|
|
|
(word >> 0) as u8,
|
|
|
|
(word >> 8) as u8,
|
|
|
|
(word >> 16) as u8,
|
|
|
|
(word >> 24) as u8,
|
|
|
|
]);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
pub extern fn dma_erase(name: CSlice<u8>) {
|
2020-08-05 12:25:48 +08:00
|
|
|
let name = String::from_utf8(name.as_ref().to_vec()).unwrap();
|
2020-08-24 16:18:31 +08:00
|
|
|
unsafe {
|
|
|
|
KERNEL_CHANNEL_1TO0.as_mut().unwrap().send(Message::DmaEraseRequest(name));
|
|
|
|
}
|
2020-07-21 16:57:14 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
pub extern fn dma_retrieve(name: CSlice<u8>) -> DmaTrace {
|
2020-08-05 12:25:48 +08:00
|
|
|
let name = String::from_utf8(name.as_ref().to_vec()).unwrap();
|
2020-08-24 16:18:31 +08:00
|
|
|
unsafe {
|
|
|
|
KERNEL_CHANNEL_1TO0.as_mut().unwrap().send(Message::DmaGetRequest(name));
|
|
|
|
}
|
|
|
|
match unsafe {KERNEL_CHANNEL_0TO1.as_mut().unwrap()}.recv() {
|
2020-08-05 12:25:48 +08:00
|
|
|
Message::DmaGetReply(None) => (),
|
|
|
|
Message::DmaGetReply(Some((mut v, duration))) => {
|
|
|
|
v.reserve(ALIGNMENT - 1);
|
|
|
|
let original_length = v.len();
|
|
|
|
let padding = ALIGNMENT - v.as_ptr() as usize % ALIGNMENT;
|
|
|
|
let padding = if padding == ALIGNMENT { 0 } else { padding };
|
|
|
|
for _ in 0..padding {
|
|
|
|
v.push(0);
|
|
|
|
}
|
2020-08-06 13:59:28 +08:00
|
|
|
// trailing zero to indicate end of buffer
|
|
|
|
v.push(0);
|
2020-08-05 12:25:48 +08:00
|
|
|
v.copy_within(0..original_length, padding);
|
2020-09-09 21:16:46 +08:00
|
|
|
dcci_slice(&v);
|
2020-08-05 12:25:48 +08:00
|
|
|
let v = Box::new(v);
|
|
|
|
let address = Box::into_raw(v) as *mut Vec<u8> as i32;
|
|
|
|
return DmaTrace {
|
|
|
|
address,
|
|
|
|
duration,
|
|
|
|
};
|
|
|
|
},
|
|
|
|
_ => panic!("Expected DmaGetReply after DmaGetRequest!"),
|
|
|
|
}
|
|
|
|
// we have to defer raising error as we have to drop the message first...
|
|
|
|
artiq_raise!("DMAError", "DMA trace not found");
|
2020-07-21 16:57:14 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
pub extern fn dma_playback(timestamp: i64, ptr: i32) {
|
|
|
|
unsafe {
|
2020-08-05 12:25:48 +08:00
|
|
|
let v = Box::from_raw(ptr as *mut Vec<u8>);
|
|
|
|
let padding = ALIGNMENT - v.as_ptr() as usize % ALIGNMENT;
|
|
|
|
let padding = if padding == ALIGNMENT { 0 } else { padding };
|
|
|
|
let ptr = v.as_ptr().add(padding) as i32;
|
|
|
|
|
2020-07-21 16:57:14 +08:00
|
|
|
csr::rtio_dma::base_address_write(ptr as u32);
|
|
|
|
csr::rtio_dma::time_offset_write(timestamp as u64);
|
|
|
|
|
|
|
|
csr::cri_con::selected_write(1);
|
|
|
|
csr::rtio_dma::enable_write(1);
|
|
|
|
while csr::rtio_dma::enable_read() != 0 {}
|
|
|
|
csr::cri_con::selected_write(0);
|
|
|
|
|
2020-09-09 21:16:46 +08:00
|
|
|
// leave the handle as we may try to do playback for another time.
|
2020-08-06 10:12:22 +08:00
|
|
|
mem::forget(v);
|
2020-07-30 22:25:49 +08:00
|
|
|
|
2020-07-21 16:57:14 +08:00
|
|
|
let error = csr::rtio_dma::error_read();
|
|
|
|
if error != 0 {
|
|
|
|
let timestamp = csr::rtio_dma::error_timestamp_read();
|
|
|
|
let channel = csr::rtio_dma::error_channel_read();
|
|
|
|
csr::rtio_dma::error_write(1);
|
|
|
|
if error & 1 != 0 {
|
|
|
|
artiq_raise!("RTIOUnderflow",
|
|
|
|
"RTIO underflow at {0} mu, channel {1}",
|
|
|
|
timestamp as i64, channel as i64, 0);
|
|
|
|
}
|
|
|
|
if error & 2 != 0 {
|
|
|
|
artiq_raise!("RTIODestinationUnreachable",
|
|
|
|
"RTIO destination unreachable, output, at {0} mu, channel {1}",
|
|
|
|
timestamp as i64, channel as i64, 0);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|