Implement DMA playback.

artiq/coredevice/dma.py

@@ -1,5 +1,5 @@
 from artiq.language.core import syscall, kernel
-from artiq.language.types import TStr, TNone
+from artiq.language.types import TInt64, TStr, TNone
 
 from numpy import int64
 
@@ -8,11 +8,18 @@ from numpy import int64
 def dma_record_start() -> TNone:
     raise NotImplementedError("syscall not simulated")
 
-
 @syscall
 def dma_record_stop(name: TStr) -> TNone:
     raise NotImplementedError("syscall not simulated")
 
+@syscall
+def dma_erase(name: TStr) -> TNone:
+    raise NotImplementedError("syscall not simulated")
+
+@syscall
+def dma_playback(timestamp: TInt64, name: TStr) -> TNone:
+    raise NotImplementedError("syscall not simulated")
+
 
 class DMARecordContextManager:
     def __init__(self):
@@ -50,6 +57,19 @@ class CoreDMA:
 
     @kernel
     def record(self, name):
-        """Returns a context manager that will record a DMA trace called ``name``."""
+        """Returns a context manager that will record a DMA trace called ``name``.
+        Any previously recorded trace with the same name is overwritten.
+        The trace will persist across kernel switches."""
         self.recorder.name = name
         return self.recorder
+
+    @kernel
+    def erase(self, name):
+        """Removes the DMA trace with the given name from storage."""
+        dma_erase(name)
+
+    @kernel
+    def replay(self, name):
+        """Replays a previously recorded DMA trace. This function blocks until
+        the entire trace is submitted to the RTIO FIFOs."""
+        dma_playback(now_mu(), name)

artiq/firmware/ksupport/api.rs

@@ -107,6 +107,8 @@ static mut API: &'static [(&'static str, *const ())] = &[
 
     api!(dma_record_start = ::dma_record_start),
     api!(dma_record_stop = ::dma_record_stop),
+    api!(dma_erase = ::dma_erase),
+    api!(dma_playback = ::dma_playback),
 
     api!(drtio_get_channel_state = ::rtio::drtio_dbg::get_channel_state),
     api!(drtio_reset_channel_state = ::rtio::drtio_dbg::reset_channel_state),

artiq/firmware/ksupport/lib.rs

@@ -17,6 +17,7 @@ extern crate amp;
 use core::{mem, ptr, slice, str};
 use std::io::Cursor;
 use cslice::{CSlice, AsCSlice};
+use board::csr;
 use dyld::Library;
 use proto::{kernel_proto, rpc_proto};
 use proto::kernel_proto::*;
@@ -257,6 +258,8 @@ extern fn dma_record_start() {
 }
 
 extern fn dma_record_stop(name: CSlice<u8>) {
+    let name = str::from_utf8(name.as_ref()).unwrap();
+
     unsafe {
         if !DMA_RECORDING {
             raise!("DMAError", "DMA is not recording")
@@ -269,7 +272,7 @@ extern fn dma_record_stop(name: CSlice<u8>) {
                        rtio::output_wide as *const () as u32).unwrap();
 
         DMA_RECORDING = false;
-        send(&DmaRecordStop(str::from_utf8(name.as_ref()).unwrap()));
+        send(&DmaRecordStop(name));
     }
 }
 
@@ -292,6 +295,64 @@ extern fn dma_record_output_wide(timestamp: i64, channel: i32, address: i32, dat
     })
 }
 
+extern fn dma_erase(name: CSlice<u8>) {
+    let name = str::from_utf8(name.as_ref()).unwrap();
+
+    send(&DmaEraseRequest(name));
+}
+
+extern fn dma_playback(timestamp: i64, name: CSlice<u8>) {
+    let name = str::from_utf8(name.as_ref()).unwrap();
+
+    send(&DmaPlaybackRequest(name));
+    let succeeded = recv!(&DmaPlaybackReply(data) => unsafe {
+        // Here, we take advantage of the fact that DmaPlaybackReply always refers
+        // to an entire heap allocation, which is 4-byte-aligned.
+        let data = match data { Some(bytes) => bytes, None => return false };
+        csr::rtio_dma::base_address_write(data.as_ptr() as u64);
+
+        csr::rtio_dma::time_offset_write(timestamp as u64);
+        csr::rtio_dma::enable_write(1);
+        while csr::rtio_dma::enable_read() != 0 {}
+
+        let status = csr::rtio_dma::error_status_read();
+        let timestamp = csr::rtio_dma::error_timestamp_read();
+        let channel = csr::rtio_dma::error_channel_read();
+        if status & rtio::RTIO_O_STATUS_UNDERFLOW != 0 {
+            csr::rtio_dma::error_underflow_reset_write(1);
+            raise!("RTIOUnderflow",
+                "RTIO underflow at {0} mu, channel {1}",
+                timestamp as i64, channel as i64, 0)
+        }
+        if status & rtio::RTIO_O_STATUS_SEQUENCE_ERROR != 0 {
+            csr::rtio_dma::error_sequence_error_reset_write(1);
+            raise!("RTIOSequenceError",
+                "RTIO sequence error at {0} mu, channel {1}",
+                timestamp as i64, channel as i64, 0)
+        }
+        if status & rtio::RTIO_O_STATUS_COLLISION != 0 {
+            csr::rtio_dma::error_collision_reset_write(1);
+            raise!("RTIOCollision",
+                "RTIO collision at {0} mu, channel {1}",
+                timestamp as i64, channel as i64, 0)
+        }
+        if status & rtio::RTIO_O_STATUS_BUSY != 0 {
+            csr::rtio_dma::error_busy_reset_write(1);
+            raise!("RTIOBusy",
+                "RTIO busy on channel {0}",
+                channel as i64, 0, 0)
+        }
+
+        true
+    });
+
+    if !succeeded {
+        println!("DMA trace called {:?} not found", name);
+        raise!("DMAError",
+            "DMA trace not found");
+    }
+}
+
 unsafe fn attribute_writeback(typeinfo: *const ()) {
     struct Attr {
         offset: usize,

artiq/firmware/ksupport/rtio.rs

@@ -4,13 +4,13 @@ use board::csr;
 use ::send;
 use kernel_proto::*;
 
-const RTIO_O_STATUS_FULL:           u32 = 1;
+pub const RTIO_O_STATUS_FULL:           u32 = 1;
-const RTIO_O_STATUS_UNDERFLOW:      u32 = 2;
+pub const RTIO_O_STATUS_UNDERFLOW:      u32 = 2;
-const RTIO_O_STATUS_SEQUENCE_ERROR: u32 = 4;
+pub const RTIO_O_STATUS_SEQUENCE_ERROR: u32 = 4;
-const RTIO_O_STATUS_COLLISION:      u32 = 8;
+pub const RTIO_O_STATUS_COLLISION:      u32 = 8;
-const RTIO_O_STATUS_BUSY:           u32 = 16;
+pub const RTIO_O_STATUS_BUSY:           u32 = 16;
-const RTIO_I_STATUS_EMPTY:          u32 = 1;
+pub const RTIO_I_STATUS_EMPTY:          u32 = 1;
-const RTIO_I_STATUS_OVERFLOW:       u32 = 2;
+pub const RTIO_I_STATUS_OVERFLOW:       u32 = 2;
 
 pub extern fn init() {
     send(&RtioInitRequest);

artiq/firmware/libproto/kernel_proto.rs

@@ -38,6 +38,11 @@ pub enum Message<'a> {
     },
     DmaRecordStop(&'a str),
 
+    DmaEraseRequest(&'a str),
+
+    DmaPlaybackRequest(&'a str),
+    DmaPlaybackReply(Option<&'a [u8]>),
+
     DrtioChannelStateRequest { channel: u32 },
     DrtioChannelStateReply { fifo_space: u16, last_timestamp: u64 },
     DrtioResetChannelStateRequest { channel: u32 },

artiq/firmware/runtime/rtio_dma.rs

@@ -1,7 +1,8 @@
+use std::mem;
 use std::vec::Vec;
 use std::string::String;
 use std::btree_map::BTreeMap;
-use std::mem;
+use std::io::Write;
 use proto::WriteExt;
 
 #[derive(Debug)]
@@ -29,16 +30,26 @@ impl Manager {
 
     pub fn record_append(&mut self, timestamp: u64, channel: u32,
                          address: u32, data: &[u32]) {
+        let writer = &mut self.recording;
         // See gateware/rtio/dma.py.
         let length = /*length*/1 + /*channel*/3 + /*timestamp*/8 + /*address*/2 +
                      /*data*/data.len() * 4;
-        let writer = &mut self.recording;
+        writer.write_all(&[
-        writer.write_u8(length as u8).unwrap();
+            (length    >>  0) as u8,
-        writer.write_u8((channel >> 24) as u8).unwrap();
+            (channel   >>  0) as u8,
-        writer.write_u8((channel >> 16) as u8).unwrap();
+            (channel   >>  8) as u8,
-        writer.write_u8((channel >>  8) as u8).unwrap();
+            (channel   >> 16) as u8,
-        writer.write_u64(timestamp).unwrap();
+            (timestamp >>  0) as u8,
-        writer.write_u16(address as u16).unwrap();
+            (timestamp >>  8) as u8,
+            (timestamp >> 16) as u8,
+            (timestamp >> 24) as u8,
+            (timestamp >> 32) as u8,
+            (timestamp >> 40) as u8,
+            (timestamp >> 48) as u8,
+            (timestamp >> 56) as u8,
+            (address   >>  0) as u8,
+            (address   >>  8) as u8,
+        ]).unwrap();
         for &word in data {
             writer.write_u32(word).unwrap();
         }
@@ -55,4 +66,12 @@ impl Manager {
             data: recorded
         });
     }
+
+    pub fn erase(&mut self, name: &str) {
+        self.entries.remove(name);
+    }
+
+    pub fn with_trace<F: FnOnce(Option<&[u8]>) -> R, R>(&self, name: &str, f: F) -> R {
+        f(self.entries.get(name).map(|entry| &entry.data[..]))
+    }
 }

artiq/firmware/runtime/session.rs

@@ -408,6 +408,15 @@ fn process_kern_message(io: &Io, mut stream: Option<&mut TcpStream>,
                 session.congress.dma_manager.record_stop(name);
                 kern_acknowledge()
             }
+            &kern::DmaEraseRequest(name) => {
+                session.congress.dma_manager.erase(name);
+                kern_acknowledge()
+            }
+            &kern::DmaPlaybackRequest(name) => {
+                session.congress.dma_manager.with_trace(name, |trace| {
+                    kern_send(io, &kern::DmaPlaybackReply(trace))
+                })
+            }
 
             &kern::DrtioChannelStateRequest { channel } => {
                 let (fifo_space, last_timestamp) = rtio_mgt::drtio_dbg::get_channel_state(channel);

artiq/test/coredevice/test_rtio.py

@@ -457,3 +457,51 @@ class RPCTest(ExperimentCase):
         self.assertGreater(rpc_time_mean, 100*ns)
         self.assertLess(rpc_time_mean, 2*ms)
         self.assertLess(self.dataset_mgr.get("rpc_time_stddev"), 1*ms)
+
+
+class _DMA(EnvExperiment):
+    def build(self, trace_name="foobar"):
+        self.setattr_device("core")
+        self.setattr_device("core_dma")
+        self.setattr_device("ttl0")
+        self.trace_name = trace_name
+
+    @kernel
+    def record(self):
+        with self.core_dma.record(self.trace_name):
+            delay(100*ns)
+            self.ttl0.on()
+            delay(100*ns)
+            self.ttl0.off()
+
+    @kernel
+    def replay(self):
+        self.core_dma.replay(self.trace_name)
+
+    @kernel
+    def erase(self):
+        self.core_dma.erase(self.trace_name)
+
+    @kernel
+    def nested(self):
+        with self.core_dma.record(self.trace_name):
+            with self.core_dma.record(self.trace_name):
+                pass
+
+
+class DMATest(ExperimentCase):
+    def test_dma_storage(self):
+        exp = self.create(_DMA)
+        exp.record()
+        exp.record() # overwrite
+        exp.replay()
+        exp.erase()
+        with self.assertRaises(exceptions.DMAError):
+            exp.replay()
+
+    def test_dma_nested(self):
+        exp = self.create(_DMA)
+        with self.assertRaises(exceptions.DMAError):
+            exp.nested()
+
+    # TODO: check replay against analyzer