satman: distributed DMA support

artiq/firmware/Cargo.lock

@@ -347,6 +347,7 @@ dependencies = [
 name = "satman"
 version = "0.0.0"
 dependencies = [
+ "alloc_list",
  "board_artiq",
  "board_misoc",
  "build_misoc",

artiq/firmware/libproto_artiq/drtioaux_proto.rs

@@ -14,6 +14,9 @@ impl<T> From<IoError<T>> for Error<T> {
     }
 }
 
+/* 512 (max size) - 4 (CRC) - 1 (packet ID) - 4 (trace ID) - 1 (last) - 2 (length) */
+const DMA_TRACE_MAX_SIZE: usize = 500;
+
 #[derive(PartialEq, Debug)]
 pub enum Packet {
     EchoRequest,
@@ -54,6 +57,15 @@ pub enum Packet {
     SpiReadRequest { destination: u8, busno: u8 },
     SpiReadReply { succeeded: bool, data: u32 },
     SpiBasicReply { succeeded: bool },
+
+    DmaAddTraceRequest { id: u32, last: bool, length: u16, trace: [u8; DMA_TRACE_MAX_SIZE] },
+    DmaAddTraceReply { succeeded: bool },
+    DmaRemoveTraceRequest { id: u32 },
+    DmaRemoveTraceReply { succeeded: bool },
+    DmaPlaybackRequest { id: u32, timestamp: u64 },
+    DmaPlaybackReply { succeeded: bool },
+    DmaPlaybackStatus { id: u32, error: u8, channel: u32, timestamp: u64 }
+
 }
 
 impl Packet {
@@ -185,6 +197,42 @@ impl Packet {
                 succeeded: reader.read_bool()?
             },
 
+            0xb0 => { 
+                let id = reader.read_u32()?;
+                let last = reader.read_bool()?;
+                let length = reader.read_u16()?;
+                let mut trace: [u8; DMA_TRACE_MAX_SIZE] = [0; DMA_TRACE_MAX_SIZE];
+                reader.read_exact(&mut trace[0..length as usize])?;
+                Packet::DmaAddTraceRequest {
+                    id: id,
+                    last: last,
+                    length: length as u16,
+                    trace: trace,
+                }
+            },
+            0xb1 => Packet::DmaAddTraceReply {
+                succeeded: reader.read_bool()?
+            },
+            0xb2 => Packet::DmaRemoveTraceRequest {
+                id: reader.read_u32()?
+            },
+            0xb3 => Packet::DmaRemoveTraceReply {
+                succeeded: reader.read_bool()?
+            },
+            0xb4 => Packet::DmaPlaybackRequest {
+                id: reader.read_u32()?,
+                timestamp: reader.read_u64()?
+            },
+            0xb5 => Packet::DmaPlaybackReply {
+                succeeded: reader.read_bool()?
+            },
+            0xb6 => Packet::DmaPlaybackStatus {
+                id: reader.read_u32()?,
+                error: reader.read_u8()?,
+                channel: reader.read_u32()?,
+                timestamp: reader.read_u64()?
+            },
+
             ty => return Err(Error::UnknownPacket(ty))
         })
     }
@@ -343,6 +391,44 @@ impl Packet {
                 writer.write_u8(0x95)?;
                 writer.write_bool(succeeded)?;
             },
+
+            Packet::DmaAddTraceRequest { id, last, trace, length } => {
+                writer.write_u8(0xb0)?;
+                writer.write_u32(id)?;
+                writer.write_bool(last)?;
+                // trace may be broken down to fit within drtio aux memory limit
+                // will be reconstructed by satellite
+                writer.write_u16(length)?;
+                writer.write_all(&trace[0..length as usize])?;
+            },
+            Packet::DmaAddTraceReply { succeeded } => {
+                writer.write_u8(0xb1)?;
+                writer.write_bool(succeeded)?;
+            },
+            Packet::DmaRemoveTraceRequest { id } => {
+                writer.write_u8(0xb2)?;
+                writer.write_u32(id)?;
+            },
+            Packet::DmaRemoveTraceReply { succeeded } => {
+                writer.write_u8(0xb3)?;
+                writer.write_bool(succeeded)?;
+            },
+            Packet::DmaPlaybackRequest { id, timestamp } => {
+                writer.write_u8(0xb4)?;
+                writer.write_u32(id)?;
+                writer.write_u64(timestamp)?;
+            },
+            Packet::DmaPlaybackReply { succeeded } => {
+                writer.write_u8(0xb5)?;
+                writer.write_bool(succeeded)?;
+            },
+            Packet::DmaPlaybackStatus { id, error, channel, timestamp } => {
+                writer.write_u8(0xb6)?;
+                writer.write_u32(id)?;
+                writer.write_u8(error)?;
+                writer.write_u32(channel)?;
+                writer.write_u64(timestamp)?;
+            }
         }
         Ok(())
     }

artiq/firmware/satman/Cargo.toml

@@ -16,4 +16,5 @@ build_misoc = { path = "../libbuild_misoc" }
 log = { version = "0.4", default-features = false }
 board_misoc = { path = "../libboard_misoc", features = ["uart_console", "log"] }
 board_artiq = { path = "../libboard_artiq" }
+alloc_list = { path = "../liballoc_list" }
 riscv = { version = "0.6.0", features = ["inline-asm"] }

artiq/firmware/satman/dma.rs

@@ -0,0 +1,151 @@
+use board_misoc::csr;
+use alloc::{vec::Vec, collections::btree_map::BTreeMap};
+
+const ALIGNMENT: usize = 64;
+
+#[derive(Debug, PartialEq)]
+enum ManagerState {
+    Idle,
+    Playback
+}
+
+pub struct RtioStatus {
+    pub id: u32, 
+    pub error: u8, 
+    pub channel: u32, 
+    pub timestamp: u64
+}
+
+pub enum Error {
+    IdNotFound,
+    PlaybackInProgress,
+    EntryNotComplete
+}
+
+#[derive(Debug)]
+struct Entry {
+    trace: Vec<u8>,
+    padding_len: usize,
+    complete: bool
+}
+
+#[derive(Debug)]
+pub struct Manager {
+    entries: BTreeMap<u32, Entry>,
+    state: ManagerState,
+    currentid: u32
+}
+
+impl Manager {
+    pub fn new() -> Manager {
+        // in case Manager is created during a DMA in progress
+        // wait for it to end
+        unsafe {
+            while csr::rtio_dma::enable_read() != 0 {} 
+        }
+        Manager {
+            entries: BTreeMap::new(),
+            currentid: 0,
+            state: ManagerState::Idle,
+        }
+    }
+
+    pub fn add(&mut self, id: u32, last: bool, trace: &[u8], trace_len: usize) -> Result<(), Error> {
+        let entry = match self.entries.get_mut(&id) {
+            Some(entry) => entry,
+            None => {
+                self.entries.insert(id, Entry {
+                    trace: Vec::new(),
+                    padding_len: 0,
+                    complete: false });
+                self.entries.get_mut(&id).unwrap()
+            },
+        };
+        entry.trace.extend(&trace[0..trace_len]);
+
+        if last {
+            entry.trace.push(0);
+            let data_len = entry.trace.len();
+    
+            // Realign.
+            entry.trace.reserve(ALIGNMENT - 1);
+            let padding = ALIGNMENT - entry.trace.as_ptr() as usize % ALIGNMENT;
+            let padding = if padding == ALIGNMENT { 0 } else { padding };
+            for _ in 0..padding {
+                // Vec guarantees that this will not reallocate
+                entry.trace.push(0)
+            }
+            for i in 1..data_len + 1 {
+                entry.trace[data_len + padding - i] = entry.trace[data_len - i]
+            }
+            entry.complete = true;
+            entry.padding_len = padding;
+        }
+        Ok(())
+    }
+
+    pub fn erase(&mut self, id: u32) -> Result<(), Error> {
+        match self.entries.remove(&id) {
+            Some(_) => Ok(()),
+            None => Err(Error::IdNotFound)
+        }
+    }
+
+    pub fn playback(&mut self, id: u32, timestamp: u64) -> Result<(), Error> {
+        if self.state != ManagerState::Idle {
+            return Err(Error::PlaybackInProgress);
+        }
+
+        let entry = match self.entries.get(&id){
+            Some(entry) => entry,
+            None => { return Err(Error::IdNotFound); }
+        };
+        if !entry.complete {
+            return Err(Error::EntryNotComplete);
+        }
+        let ptr = entry.trace[entry.padding_len..].as_ptr();
+        assert!(ptr as u32 % 64 == 0);
+
+        self.state = ManagerState::Playback;
+        self.currentid = id;
+
+        unsafe {
+            csr::rtio_dma::base_address_write(ptr as u64);
+            csr::rtio_dma::time_offset_write(timestamp as u64);
+    
+            csr::cri_con::selected_write(1);
+            csr::rtio_dma::enable_write(1);
+            // playback has begun here, for status call check_state
+        }
+        Ok(())
+    }
+
+    pub fn check_state(&mut self) -> Option<RtioStatus> {
+        if self.state != ManagerState::Playback {
+            // nothing to report
+            return None;
+        }
+        let dma_enable = unsafe { csr::rtio_dma::enable_read() };
+        if dma_enable != 0 {
+            return None;
+        }
+        else {
+            self.state = ManagerState::Idle;
+            unsafe { 
+                csr::cri_con::selected_write(0);
+                let error =  csr::rtio_dma::error_read();
+                let channel = csr::rtio_dma::error_channel_read();
+                let timestamp = csr::rtio_dma::error_timestamp_read();
+                if error != 0 {
+                    csr::rtio_dma::error_write(1);
+                }
+                return Some(RtioStatus { 
+                    id: self.currentid, 
+                    error: error,
+                    channel: channel, 
+                    timestamp: timestamp });
+            }
+        }
+    }
+
+}

artiq/firmware/satman/main.rs

@@ -1,4 +1,4 @@
-#![feature(never_type, panic_info_message, llvm_asm)]
+#![feature(never_type, panic_info_message, llvm_asm, default_alloc_error_handler)]
 #![no_std]
 
 #[macro_use]
@@ -7,6 +7,7 @@ extern crate log;
 extern crate board_misoc;
 extern crate board_artiq;
 extern crate riscv;
+extern crate alloc;
 
 use core::convert::TryFrom;
 use board_misoc::{csr, ident, clock, uart_logger, i2c, pmp};
@@ -15,8 +16,13 @@ use board_artiq::si5324;
 use board_artiq::{spi, drtioaux};
 use board_artiq::drtio_routing;
 use riscv::register::{mcause, mepc, mtval};
+use dma::Manager as DmaManager;
+
+#[global_allocator]
+static mut ALLOC: alloc_list::ListAlloc = alloc_list::EMPTY;
 
 mod repeater;
+mod dma;
 
 fn drtiosat_reset(reset: bool) {
     unsafe {
@@ -67,7 +73,7 @@ macro_rules! forward {
     ($routing_table:expr, $destination:expr, $rank:expr, $repeaters:expr, $packet:expr) => {}
 }
 
-fn process_aux_packet(_repeaters: &mut [repeater::Repeater],
+fn process_aux_packet(_manager: &mut DmaManager, _repeaters: &mut [repeater::Repeater],
         _routing_table: &mut drtio_routing::RoutingTable, _rank: &mut u8,
         packet: drtioaux::Packet) -> Result<(), drtioaux::Error<!>> {
     // In the code below, *_chan_sel_write takes an u8 if there are fewer than 256 channels,
@@ -294,6 +300,24 @@ fn process_aux_packet(_repeaters: &mut [repeater::Repeater],
                     &drtioaux::Packet::SpiReadReply { succeeded: false, data: 0 })
             }
         }
+        #[cfg(has_rtio_dma)]
+        drtioaux::Packet::DmaAddTraceRequest { id, last, length, trace } => {
+            let succeeded = _manager.add(id, last, &trace, length as usize).is_ok();
+            drtioaux::send(0,
+                &drtioaux::Packet::DmaAddTraceReply { succeeded: succeeded })
+        }
+        #[cfg(has_rtio_dma)]
+        drtioaux::Packet::DmaRemoveTraceRequest { id } => {
+            let succeeded = _manager.erase(id).is_ok();
+            drtioaux::send(0,
+                &drtioaux::Packet::DmaRemoveTraceReply { succeeded: succeeded })
+        }
+        #[cfg(has_rtio_dma)]
+        drtioaux::Packet::DmaPlaybackRequest { id, timestamp } => {
+            let succeeded = _manager.playback(id, timestamp).is_ok();
+            drtioaux::send(0,
+                &drtioaux::Packet::DmaPlaybackReply { succeeded: succeeded })
+        }
 
         _ => {
             warn!("received unexpected aux packet");
@@ -302,12 +326,12 @@ fn process_aux_packet(_repeaters: &mut [repeater::Repeater],
     }
 }
 
-fn process_aux_packets(repeaters: &mut [repeater::Repeater],
+fn process_aux_packets(dma_manager: &mut DmaManager, repeaters: &mut [repeater::Repeater],
         routing_table: &mut drtio_routing::RoutingTable, rank: &mut u8) {
     let result =
         drtioaux::recv(0).and_then(|packet| {
             if let Some(packet) = packet {
-                process_aux_packet(repeaters, routing_table, rank, packet)
+                process_aux_packet(dma_manager, repeaters, routing_table, rank, packet)
             } else {
                 Ok(())
             }
@@ -432,10 +456,13 @@ fn sysclk_setup() {
 #[no_mangle]
 pub extern fn main() -> i32 {
     extern {
+        static mut _fheap: u8;
+        static mut _eheap: u8;
         static mut _sstack_guard: u8;
     }
 
     unsafe {
+        ALLOC.add_range(&mut _fheap, &mut _eheap);
         pmp::init_stack_guard(&_sstack_guard as *const u8 as usize);
     }
 
@@ -511,13 +538,18 @@ pub extern fn main() -> i32 {
             si5324::siphaser::calibrate_skew().expect("failed to calibrate skew");
         }
 
+        // DMA manager created here, so when link is dropped, all DMA traces
+        // are cleared out for a clean slate on subsequent connections,
+        // without a manual intervention.
+        let mut dma_manager = DmaManager::new();
+
         drtioaux::reset(0);
         drtiosat_reset(false);
         drtiosat_reset_phy(false);
 
         while drtiosat_link_rx_up() {
             drtiosat_process_errors();
-            process_aux_packets(&mut repeaters, &mut routing_table, &mut rank);
+            process_aux_packets(&mut dma_manager, &mut repeaters, &mut routing_table, &mut rank);
             for rep in repeaters.iter_mut() {
                 rep.service(&routing_table, rank);
             }
@@ -538,6 +570,12 @@ pub extern fn main() -> i32 {
                     error!("aux packet error: {}", e);
                 }
             }
+            if let Some(status) = dma_manager.check_state() {
+                if let Err(e) = drtioaux::send(0, &drtioaux::Packet::DmaPlaybackStatus { 
+                    id: status.id, error: status.error, channel: status.channel, timestamp: status.timestamp }) {
+                    error!("error sending DMA playback status: {}", e);
+                }
+            }
         }
 
         drtiosat_reset_phy(true);

artiq/firmware/satman/satman.ld

@@ -66,4 +66,12 @@ SECTIONS
         . += 0x10000;
         _fstack = . - 16;
     } > main_ram
+
+    /* 64MB heap for alloc use */
+    .heap (NOLOAD) : ALIGN(16)
+    {
+        _fheap = .;
+        . = . + 0x4000000;
+        _eheap = .;
+    } > main_ram
 }