use alloc::{string::String, vec::Vec}; use core::mem; use cslice::CSlice; use super::{Message, KERNEL_CHANNEL_0TO1, KERNEL_CHANNEL_1TO0, KERNEL_IMAGE}; use crate::{artiq_raise, pl::csr, rtio}; #[repr(C)] pub struct DmaTrace { duration: i64, address: i32, uses_ddma: bool, } #[derive(Clone, Debug)] pub struct DmaRecorder { pub name: String, pub buffer: Vec, pub duration: i64, pub enable_ddma: bool, } static mut RECORDER: Option = None; pub unsafe fn init_dma_recorder() { // as static would remain after restart, we have to reset it, // without running its destructor. mem::forget(mem::replace(&mut RECORDER, None)); } pub extern "C" fn dma_record_start(name: CSlice) { let name = String::from_utf8(name.as_ref().to_vec()).unwrap(); unsafe { KERNEL_CHANNEL_1TO0 .as_mut() .unwrap() .send(Message::DmaEraseRequest(name.clone())); } unsafe { if RECORDER.is_some() { artiq_raise!("DMAError", "DMA is already recording") } let library = KERNEL_IMAGE.as_ref().unwrap(); library.rebind(b"rtio_output", dma_record_output as *const ()).unwrap(); library .rebind(b"rtio_output_wide", dma_record_output_wide as *const ()) .unwrap(); RECORDER = Some(DmaRecorder { name, buffer: Vec::new(), duration: 0, enable_ddma: false, }); } } pub extern "C" fn dma_record_stop(duration: i64, enable_ddma: bool) { unsafe { if RECORDER.is_none() { artiq_raise!("DMAError", "DMA is not recording") } let library = KERNEL_IMAGE.as_ref().unwrap(); library.rebind(b"rtio_output", rtio::output as *const ()).unwrap(); library .rebind(b"rtio_output_wide", rtio::output_wide as *const ()) .unwrap(); let mut recorder = RECORDER.take().unwrap(); recorder.duration = duration; recorder.enable_ddma = enable_ddma; KERNEL_CHANNEL_1TO0 .as_mut() .unwrap() .send(Message::DmaPutRequest(recorder)); } } #[inline(always)] unsafe fn dma_record_output_prepare(timestamp: i64, target: i32, words: usize) { // See gateware/rtio/dma.py. const HEADER_LENGTH: usize = /*length*/1 + /*channel*/3 + /*timestamp*/8 + /*address*/1; let length = HEADER_LENGTH + /*data*/words * 4; let buffer = &mut RECORDER.as_mut().unwrap().buffer; buffer.reserve(length); buffer.extend_from_slice(&[ (length >> 0) as u8, (target >> 8) as u8, (target >> 16) as u8, (target >> 24) as u8, (timestamp >> 0) as u8, (timestamp >> 8) as u8, (timestamp >> 16) as u8, (timestamp >> 24) as u8, (timestamp >> 32) as u8, (timestamp >> 40) as u8, (timestamp >> 48) as u8, (timestamp >> 56) as u8, (target >> 0) as u8, ]); } pub extern "C" fn dma_record_output(target: i32, word: i32) { unsafe { let timestamp = rtio::now_mu(); dma_record_output_prepare(timestamp, target, 1); RECORDER.as_mut().unwrap().buffer.extend_from_slice(&[ (word >> 0) as u8, (word >> 8) as u8, (word >> 16) as u8, (word >> 24) as u8, ]); } } pub extern "C" fn dma_record_output_wide(target: i32, words: &CSlice) { assert!(words.len() <= 16); // enforce the hardware limit unsafe { let timestamp = rtio::now_mu(); dma_record_output_prepare(timestamp, target, words.len()); let buffer = &mut RECORDER.as_mut().unwrap().buffer; for word in words.as_ref().iter() { buffer.extend_from_slice(&[ (word >> 0) as u8, (word >> 8) as u8, (word >> 16) as u8, (word >> 24) as u8, ]); } } } pub extern "C" fn dma_erase(name: CSlice) { let name = String::from_utf8(name.as_ref().to_vec()).unwrap(); unsafe { KERNEL_CHANNEL_1TO0 .as_mut() .unwrap() .send(Message::DmaEraseRequest(name)); } } pub extern "C" fn dma_retrieve(name: CSlice) -> DmaTrace { let name = String::from_utf8(name.as_ref().to_vec()).unwrap(); unsafe { KERNEL_CHANNEL_1TO0.as_mut().unwrap().send(Message::DmaGetRequest(name)); } match unsafe { KERNEL_CHANNEL_0TO1.as_mut().unwrap() }.recv() { Message::DmaGetReply(None) => (), Message::DmaGetReply(Some((address, duration, uses_ddma))) => { return DmaTrace { address, duration, uses_ddma, }; } _ => panic!("Expected DmaGetReply after DmaGetRequest!"), } // we have to defer raising error as we have to drop the message first... artiq_raise!("DMAError", "DMA trace not found"); } pub extern "C" fn dma_playback(timestamp: i64, ptr: i32, _uses_ddma: bool) { unsafe { csr::rtio_dma::base_address_write(ptr as u32); csr::rtio_dma::time_offset_write(timestamp as u64); let old_cri_master = csr::cri_con::selected_read(); csr::cri_con::selected_write(1); csr::rtio_dma::enable_write(1); #[cfg(has_drtio)] if _uses_ddma { KERNEL_CHANNEL_1TO0 .as_mut() .unwrap() .send(Message::DmaStartRemoteRequest { id: ptr, timestamp: timestamp, }); } while csr::rtio_dma::enable_read() != 0 {} csr::cri_con::selected_write(old_cri_master); let error = csr::rtio_dma::error_read(); if error != 0 { let timestamp = csr::rtio_dma::error_timestamp_read(); let channel = csr::rtio_dma::error_channel_read(); csr::rtio_dma::error_write(1); if error & 1 != 0 { artiq_raise!( "RTIOUnderflow", "RTIO underflow at {1} mu, channel {rtio_channel_info:0}", channel as i64, timestamp as i64, 0 ); } if error & 2 != 0 { artiq_raise!( "RTIODestinationUnreachable", "RTIO destination unreachable, output, at {1} mu, channel {rtio_channel_info:0}", channel as i64, timestamp as i64, 0 ); } } #[cfg(has_drtio)] if _uses_ddma { KERNEL_CHANNEL_1TO0 .as_mut() .unwrap() .send(Message::DmaAwaitRemoteRequest(ptr)); match KERNEL_CHANNEL_0TO1.as_mut().unwrap().recv() { Message::DmaAwaitRemoteReply { timeout, error, channel, timestamp, } => { if timeout { artiq_raise!( "DMAError", "Error running DMA on satellite device, timed out waiting for results" ); } if error & 1 != 0 { artiq_raise!( "RTIOUnderflow", "RTIO underflow at {1} mu, channel {rtio_channel_info:0}", channel as i64, timestamp as i64, 0 ); } if error & 2 != 0 { artiq_raise!( "RTIODestinationUnreachable", "RTIO destination unreachable, output, at {1} mu, channel {rtio_channel_info:0}", channel as i64, timestamp as i64, 0 ); } } _ => panic!("Expected DmaAwaitRemoteReply after DmaAwaitRemoteRequest!"), } } } }