drtio: implement destination state checks on operations

artiq/coredevice/exceptions.py

@@ -91,7 +91,7 @@ class RTIOOverflow(Exception):
     artiq_builtin = True
 
 
-class RTIOLinkError(Exception):
+class RTIODestinationUnreachable(Exception):
     """Raised with a RTIO operation could not be completed due to a DRTIO link
     being down.
     """

artiq/firmware/ksupport/lib.rs

@@ -428,8 +428,8 @@ extern fn dma_playback(timestamp: i64, ptr: i32) {
                     timestamp as i64, channel as i64, 0);
             }
             if error & 2 != 0 {
-                raise!("RTIOLinkError",
+                raise!("RTIODestinationUnreachable",
-                    "RTIO output link error at {0} mu, channel {1}",
+                    "RTIO destination unreachable, output, at {0} mu, channel {1}",
                     timestamp as i64, channel as i64, 0);
             }
         }

artiq/firmware/ksupport/rtio.rs

@@ -7,13 +7,13 @@ mod imp {
     use ::send;
     use kernel_proto::*;
 
-    pub const RTIO_O_STATUS_WAIT:           u8 = 1;
+    pub const RTIO_O_STATUS_WAIT:                      u8 = 1;
-    pub const RTIO_O_STATUS_UNDERFLOW:      u8 = 2;
+    pub const RTIO_O_STATUS_UNDERFLOW:                 u8 = 2;
-    pub const RTIO_O_STATUS_LINK_ERROR:     u8 = 4;
+    pub const RTIO_O_STATUS_DESTINATION_UNREACHABLE:   u8 = 4;
-    pub const RTIO_I_STATUS_WAIT_EVENT:     u8 = 1;
+    pub const RTIO_I_STATUS_WAIT_EVENT:                u8 = 1;
-    pub const RTIO_I_STATUS_OVERFLOW:       u8 = 2;
+    pub const RTIO_I_STATUS_OVERFLOW:                  u8 = 2;
-    pub const RTIO_I_STATUS_WAIT_STATUS:    u8 = 4;
+    pub const RTIO_I_STATUS_WAIT_STATUS:               u8 = 4;
-    pub const RTIO_I_STATUS_LINK_ERROR:     u8 = 8;
+    pub const RTIO_I_STATUS_DESTINATION_UNREACHABLE:   u8 = 8;
 
     pub extern fn init() {
         send(&RtioInitRequest);
@@ -49,9 +49,9 @@ mod imp {
                 "RTIO underflow at {0} mu, channel {1}, slack {2} mu",
                 timestamp, channel as i64, timestamp - get_counter());
         }
-        if status & RTIO_O_STATUS_LINK_ERROR != 0 {
+        if status & RTIO_O_STATUS_DESTINATION_UNREACHABLE != 0 {
-            raise!("RTIOLinkError",
+            raise!("RTIODestinationUnreachable",
-                "RTIO output link error at {0} mu, channel {1}",
+                "RTIO destination unreachable, output, at {0} mu, channel {1}",
                 timestamp, channel as i64, 0);
         }
     }
@@ -108,9 +108,9 @@ mod imp {
             if status & RTIO_I_STATUS_WAIT_EVENT != 0 {
                 return !0
             }
-            if status & RTIO_I_STATUS_LINK_ERROR != 0 {
+            if status & RTIO_I_STATUS_DESTINATION_UNREACHABLE != 0 {
-                raise!("RTIOLinkError",
+                raise!("RTIODestinationUnreachable",
-                    "RTIO input link error on channel {0}",
+                    "RTIO destination unreachable, input, on channel {0}",
                     channel as i64, 0, 0);
             }
 
@@ -135,9 +135,9 @@ mod imp {
                     "RTIO input overflow on channel {0}",
                     channel as i64, 0, 0);
             }
-            if status & RTIO_I_STATUS_LINK_ERROR != 0 {
+            if status & RTIO_I_STATUS_DESTINATION_UNREACHABLE != 0 {
-                raise!("RTIOLinkError",
+                raise!("RTIODestinationUnreachable",
-                    "RTIO input link error on channel {0}",
+                    "RTIO destination unreachable, input, on channel {0}",
                     channel as i64, 0, 0);
             }
 

artiq/firmware/libboard_artiq/drtio_routing.rs

@@ -72,13 +72,25 @@ pub fn config_routing_table(default_n_links: usize) -> RoutingTable {
 }
 
 #[cfg(has_drtio_routing)]
-pub fn program_interconnect(rt: &RoutingTable, rank: u8)
+pub fn interconnect_enable(routing_table: &RoutingTable, rank: u8, destination: u8) {
-{
+    let hop = routing_table.0[destination as usize][rank as usize];
-    for i in 0..DEST_COUNT {
+    unsafe {
-        let hop = rt.0[i][rank as usize];
+        csr::routing_table::destination_write(destination);
-        unsafe {
+        csr::routing_table::hop_write(hop);
-            csr::routing_table::destination_write(i as _);
+    }
-            csr::routing_table::hop_write(hop);
+}
-        }
+
+#[cfg(has_drtio_routing)]
+pub fn interconnect_disable(destination: u8) {
+    unsafe {
+        csr::routing_table::destination_write(destination);
+        csr::routing_table::hop_write(INVALID_HOP);
+    }
+}
+
+#[cfg(has_drtio_routing)]
+pub fn interconnect_enable_all(routing_table: &RoutingTable, rank: u8) {
+    for i in 0..DEST_COUNT {
+        interconnect_enable(routing_table, rank, i as u8);
     }
 }

artiq/firmware/runtime/rtio_mgt.rs

@@ -216,6 +216,7 @@ pub mod drtio {
                 if !up_destinations[destination] {
                     info!("[DEST#{}] destination is up", destination);
                     up_destinations[destination] = true;
+                    drtio_routing::interconnect_enable(routing_table, 0, destination as u8);
                 }
             } else if hop as usize <= csr::DRTIO.len() {
                 let linkno = hop - 1;
@@ -228,6 +229,7 @@ pub mod drtio {
                             Ok(drtioaux::Packet::DestinationDownReply) => {
                                 info!("[DEST#{}] destination is down", destination);
                                 up_destinations[destination] = false;
+                                drtio_routing::interconnect_disable(destination as u8);
                             },
                             Ok(drtioaux::Packet::DestinationOkReply) => (),
                             Ok(drtioaux::Packet::DestinationSequenceErrorReply { channel }) =>
@@ -242,6 +244,7 @@ pub mod drtio {
                     } else {
                         info!("[DEST#{}] destination is down", destination);
                         up_destinations[destination] = false;
+                        drtio_routing::interconnect_disable(destination as u8);
                     }
                 } else {
                     if link_up(linkno) {
@@ -253,6 +256,7 @@ pub mod drtio {
                             Ok(drtioaux::Packet::DestinationOkReply) => {
                                 info!("[DEST#{}] destination is up", destination);
                                 up_destinations[destination] = true;
+                                drtio_routing::interconnect_enable(routing_table, 0, destination as u8);
                                 init_buffer_space(destination as u8, linkno);
                             },
                             Ok(packet) => error!("[DEST#{}] received unexpected aux packet: {:?}", destination, packet),
@@ -394,12 +398,6 @@ pub fn startup(io: &Io, routing_table: &Urc<RefCell<drtio_routing::RoutingTable>
         }
     }
 
-    #[cfg(has_drtio_routing)]
-    {
-        let routing_table = routing_table.clone();
-        drtio_routing::program_interconnect(&routing_table.borrow(), 0);
-    }
-
     drtio::startup(io, &routing_table);
     init_core(true);
     io.spawn(4096, async_error_thread);

artiq/firmware/satman/main.rs

@@ -171,7 +171,7 @@ fn process_aux_packet(_repeaters: &mut [repeater::Repeater],
         #[cfg(has_drtio_routing)]
         drtioaux::Packet::RoutingSetRank { rank } => {
             *_rank = rank;
-            drtio_routing::program_interconnect(_routing_table, rank);
+            drtio_routing::interconnect_enable_all(_routing_table, rank);
 
             let rep_rank = rank + 1;
             for rep in _repeaters.iter() {

artiq/gateware/drtio/rt_controller_master.py

@@ -102,8 +102,7 @@ class RTController(Module):
         o_status_wait = Signal()
         o_status_underflow = Signal()
         self.comb += [
-            self.cri.o_status.eq(Cat(
+            self.cri.o_status.eq(Cat(o_status_wait, o_status_underflow)),
-                o_status_wait, o_status_underflow, ~self.csrs.link_up.storage)),
             self.csrs.o_wait.status.eq(o_status_wait)
         ]
         o_underflow_set = Signal()
@@ -143,8 +142,7 @@ class RTController(Module):
         i_status_overflow = Signal()
         i_status_wait_status = Signal()
         self.comb += self.cri.i_status.eq(Cat(
-            i_status_wait_event, i_status_overflow, i_status_wait_status,
+            i_status_wait_event, i_status_overflow, i_status_wait_status))
-            ~self.csrs.link_up.storage))
 
         load_read_reply = Signal()
         self.sync.sys_with_rst += [

artiq/gateware/rtio/cri.py

@@ -34,7 +34,7 @@ layout = [
     ("o_data", 512, DIR_M_TO_S),
     ("o_address", 16, DIR_M_TO_S),
     # o_status bits:
-    # <0:wait> <1:underflow> <2:link error>
+    # <0:wait> <1:underflow> <2:destination unreachable>
     ("o_status", 3, DIR_S_TO_M),
 
     # pessimistic estimate of the number of outputs events that can be
@@ -47,7 +47,7 @@ layout = [
     ("i_timestamp", 64, DIR_S_TO_M),
     # i_status bits:
     # <0:wait for event (command timeout)> <1:overflow> <2:wait for status>
-    # <3:link error>
+    # <3:destination unreachable>
     # <0> and <1> are mutually exclusive. <1> has higher priority.
     ("i_status", 4, DIR_S_TO_M),
 ]
@@ -122,6 +122,17 @@ class CRIDecoder(Module, AutoCSR):
         # # #
 
         # routing
+        if enable_routing:
+            destination_unreachable = Interface()
+            self.comb += [
+                destination_unreachable.o_status.eq(4),
+                destination_unreachable.i_status.eq(8)
+            ]
+            slaves = slaves[:]
+            slaves.append(destination_unreachable)
+            target_len = 2**(len(slaves) - 1).bit_length()
+            slaves += [destination_unreachable]*(target_len - len(slaves))
+
         slave_bits = bits_for(len(slaves)-1)
         selected = Signal(slave_bits)
 

artiq/gateware/test/drtio/test_full_stack.py

@@ -125,7 +125,7 @@ class OutputsTestbench:
             if status & 0x2:
                 raise RTIOUnderflow
             if status & 0x4:
-                raise RTIOLinkError
+                raise RTIODestinationUnreachable
             yield
             wlen += 1
         return wlen
@@ -264,7 +264,7 @@ class TestFullStack(unittest.TestCase):
             if status & 0x2:
                 return "overflow"
             if status & 0x8:
-                return "link error"
+                return "destination unreachable"
             return ((yield from kcsrs.i_data.read()),
                     (yield from kcsrs.i_timestamp.read()))
 

artiq/gateware/test/rtio/test_dma.py

@@ -5,7 +5,7 @@ import itertools
 from migen import *
 from misoc.interconnect import wishbone
 
-from artiq.coredevice.exceptions import RTIOUnderflow, RTIOLinkError
+from artiq.coredevice.exceptions import RTIOUnderflow, RTIODestinationUnreachable
 from artiq.gateware import rtio
 from artiq.gateware.rtio import dma, cri
 from artiq.gateware.rtio.phy import ttl_simple
@@ -61,7 +61,7 @@ def do_dma(dut, address):
     if error & 1:
         raise RTIOUnderflow
     if error & 2:
-        raise RTIOLinkError
+        raise RTIODestinationUnreachable
 
 
 test_writes1 = [