firmware: send DRTIO routing table to satellite

artiq/firmware/libboard_artiq/drtio_routing.rs

@@ -5,11 +5,11 @@ pub const DEST_COUNT: usize = 256;
 pub const MAX_HOPS: usize = 32;
 pub const INVALID_HOP: u8 = 0xff;
 
-pub struct RoutingTable([[u8; MAX_HOPS]; DEST_COUNT]);
+pub struct RoutingTable(pub [[u8; MAX_HOPS]; DEST_COUNT]);
 
 impl RoutingTable {
     // default routing table is for star topology with no hops
-    fn default_master(default_n_links: usize) -> RoutingTable {
+    pub fn default_master(default_n_links: usize) -> RoutingTable {
         let mut ret = RoutingTable([[INVALID_HOP; MAX_HOPS]; DEST_COUNT]);
         for i in 0..default_n_links {
             ret.0[i][0] = i as u8;
@@ -20,9 +20,9 @@ impl RoutingTable {
         ret
     }
 
-    // satellites receive the routing table from the master
+    // use this by default on satellite, as they receive
-    // by default, block everything
+    // the routing table from the master
-    fn default_satellite() -> RoutingTable {
+    pub fn default_empty() -> RoutingTable {
         RoutingTable([[INVALID_HOP; MAX_HOPS]; DEST_COUNT])
     }
 }
@@ -68,6 +68,7 @@ pub fn config_routing_table(default_n_links: usize) -> RoutingTable {
     ret
 }
 
+#[cfg(has_drtio_routing)]
 pub fn program_interconnect(rt: &RoutingTable, rank: u8)
 {
     for i in 0..DEST_COUNT {

artiq/firmware/libboard_artiq/lib.rs

@@ -48,5 +48,4 @@ pub mod grabber;
 
 #[cfg(has_drtio)]
 pub mod drtioaux;
-#[cfg(has_drtio_routing)]
 pub mod drtio_routing;

artiq/firmware/libproto_artiq/drtioaux_proto.rs

@@ -14,7 +14,7 @@ impl<T> From<IoError<T>> for Error<T> {
     }
 }
 
-#[derive(Debug)]
+#[derive(PartialEq, Debug)]
 pub enum Packet {
     EchoRequest,
     EchoReply,

artiq/firmware/runtime/main.rs

@@ -25,6 +25,7 @@ extern crate board_artiq;
 extern crate logger_artiq;
 extern crate proto_artiq;
 
+use core::cell::RefCell;
 use core::convert::TryFrom;
 use smoltcp::wire::{EthernetAddress, IpAddress, IpCidr};
 
@@ -282,14 +283,16 @@ fn startup_ethernet() {
                        .finalize();
 
     #[cfg(has_drtio_routing)]
-    let drtio_routing_table = drtio_routing::config_routing_table(csr::DRTIO.len());
+    let drtio_routing_table = urc::Urc::new(RefCell::new(
-    #[cfg(has_drtio_routing)]
+        drtio_routing::config_routing_table(csr::DRTIO.len())));
-    drtio_routing::program_interconnect(&drtio_routing_table, 0);
 
     let mut scheduler = sched::Scheduler::new();
     let io = scheduler.io();
 
-    rtio_mgt::startup(&io);
+    #[cfg(has_drtio_routing)]
+    rtio_mgt::startup(&io, &drtio_routing_table);
+    #[cfg(not(has_drtio_routing))]
+    rtio_mgt::startup(&io, &drtio_routing::RoutingTable::default_empty());
 
     io.spawn(4096, mgmt::thread);
     io.spawn(16384, session::thread);

artiq/firmware/runtime/rtio_mgt.rs

@@ -1,6 +1,9 @@
-use board_misoc::csr;
+use core::cell::RefCell;
+use urc::Urc;
+use board_misoc::{csr, clock};
 #[cfg(has_rtio_clock_switch)]
 use board_misoc::config;
+use board_artiq::drtio_routing;
 use sched::Io;
 
 #[cfg(has_rtio_crg)]
@@ -42,11 +45,15 @@ pub mod drtio {
     use super::*;
     use drtioaux;
 
-    pub fn startup(io: &Io) {
+    pub fn startup(io: &Io, routing_table: &Urc<RefCell<drtio_routing::RoutingTable>>) {
         unsafe {
             csr::drtio_transceiver::stable_clkin_write(1);
         }
-        io.spawn(4096, link_thread);
+        let routing_table = routing_table.clone();
+        io.spawn(4096, move |io| {
+            let routing_table = routing_table.borrow();
+            link_thread(io, &routing_table)
+        });
     }
 
     fn link_rx_up(linkno: u8) -> bool {
@@ -106,28 +113,51 @@ pub mod drtio {
         }
     }
 
-    fn sync_tsc(linkno: u8, io: &Io) -> Result<(), &'static str> {
+    fn recv_aux_timeout(io: &Io, linkno: u8, timeout: u32) -> Result<drtioaux::Packet, &'static str> {
+        let max_time = clock::get_ms() + timeout as u64;
+        loop {
+            if !link_rx_up(linkno) {
+                return Err("link went down");
+            }
+            if clock::get_ms() > max_time {
+                return Err("timeout");
+            }
+            match drtioaux::recv_link(linkno) {
+                Ok(Some(packet)) => return Ok(packet),
+                Ok(None) => (),
+                Err(_) => return Err("aux packet error")
+            }
+            io.relinquish().unwrap();
+        }
+    }
+
+    fn sync_tsc(io: &Io, linkno: u8) -> Result<(), &'static str> {
         unsafe {
             (csr::DRTIO[linkno as usize].set_time_write)(1);
             while (csr::DRTIO[linkno as usize].set_time_read)() == 1 {}
         }
+        // TSCAck is the only aux packet that is sent spontaneously
+        // by the satellite, in response to a TSC set on the RT link.
+        let reply = recv_aux_timeout(io, linkno, 10000)?;
+        if reply == drtioaux::Packet::TSCAck {
+            return Ok(());
+        } else {
+            return Err("unexpected reply");
+        }
+    }
 
-        loop {
+    fn load_routing_table(io: &Io, linkno: u8, routing_table: &drtio_routing::RoutingTable) -> Result<(), &'static str> {
-            let mut count = 0;
+        for i in 0..drtio_routing::DEST_COUNT {
-            if !link_rx_up(linkno) {
+            drtioaux::send_link(linkno, &drtioaux::Packet::RoutingSetPath {
-                return Err("link went down");
+                destination: i as u8,
-            }
+                hops: routing_table.0[i]
-            count += 1;
+            }).unwrap();
-            if count > 200 {
+            let reply = recv_aux_timeout(io, linkno, 200)?;
-                return Err("timeout");
+            if reply != drtioaux::Packet::RoutingAck {
-            }
+                return Err("unexpected reply");
-            io.sleep(100).unwrap();
-            // TSCAck is the only aux packet that is sent spontaneously
-            // by the satellite, in response to a TSC set on the RT link.
-            if let Ok(Some(drtioaux::Packet::TSCAck)) = drtioaux::recv_link(linkno) {
-                return Ok(())
             }
         }
+        Ok(())
     }
 
     fn process_local_errors(linkno: u8) {
@@ -166,7 +196,7 @@ pub mod drtio {
         }
     }
 
-    pub fn link_thread(io: Io) {
+    pub fn link_thread(io: Io, routing_table: &drtio_routing::RoutingTable) {
         loop {
             for linkno in 0..csr::DRTIO.len() {
                 let linkno = linkno as u8;
@@ -188,11 +218,13 @@ pub mod drtio {
                             info!("[LINK#{}] remote replied after {} packets", linkno, ping_count);
                             set_link_up(linkno, true);
                             init_buffer_space(linkno);
-                            if sync_tsc(linkno, &io).is_err() {
+                            if let Err(e) = sync_tsc(&io, linkno) {
-                                error!("[LINK#{}] remote failed to ack TSC", linkno);
+                                error!("[LINK#{}] failed to sync TSC ({})", linkno, e);
-                            } else {
-                                info!("[LINK#{}] link initialization completed", linkno);
                             }
+                            if let Err(e) = load_routing_table(&io, linkno, routing_table) {
+                                error!("[LINK#{}] failed to load routing table ({})", linkno, e);
+                            }
+                            info!("[LINK#{}] link initialization completed", linkno);
                         } else {
                             error!("[LINK#{}] ping failed", linkno);
                         }
@@ -223,7 +255,7 @@ pub mod drtio {
 pub mod drtio {
     use super::*;
 
-    pub fn startup(_io: &Io) {}
+    pub fn startup(_io: &Io, _routing_table: &Urc<RefCell<drtio_routing::RoutingTable>>) {}
     pub fn init() {}
     pub fn link_up(_linkno: u8) -> bool { false }
 }
@@ -250,7 +282,7 @@ fn async_error_thread(io: Io) {
     }
 }
 
-pub fn startup(io: &Io) {
+pub fn startup(io: &Io, routing_table: &Urc<RefCell<drtio_routing::RoutingTable>>) {
     #[cfg(has_rtio_crg)]
     {
         #[cfg(has_rtio_clock_switch)]
@@ -290,7 +322,13 @@ pub fn startup(io: &Io) {
         }
     }
 
-    drtio::startup(io);
+    #[cfg(has_drtio_routing)]
+    {
+        let routing_table = routing_table.clone();
+        drtio_routing::program_interconnect(&routing_table.borrow(), 0);
+    }
+
+    drtio::startup(io, &routing_table);
     init_core(true);
     io.spawn(4096, async_error_thread);
 }

artiq/firmware/satman/main.rs

@@ -96,6 +96,11 @@ fn process_aux_packet(packet: drtioaux::Packet) -> Result<(), drtioaux::Error<!>
             }
         }
 
+        drtioaux::Packet::RoutingSetPath { destination, hops } => {
+            info!("routing: {} -> {:?}", destination, hops);
+            drtioaux::send_link(0, &drtioaux::Packet::RoutingAck)
+        }
+
         drtioaux::Packet::MonitorRequest { channel, probe } => {
             let value;
             #[cfg(has_rtio_moninj)]
@@ -343,8 +348,8 @@ pub extern fn main() -> i32 {
                     }
                 }
                 for rep in repeaters.iter() {
-                    if rep.sync_tsc().is_err() {
+                    if let Err(e) = rep.sync_tsc() {
-                        error!("remote failed to ack TSC");
+                        error!("failed to sync TSC ({})", e);
                     }
                 }
                 if let Err(e) = drtioaux::send_link(0, &drtioaux::Packet::TSCAck) {

artiq/firmware/satman/repeater.rs

@@ -63,8 +63,8 @@ impl Repeater {
                 if rep_link_rx_up(self.repno) {
                     if let Ok(Some(drtioaux::Packet::EchoReply)) = drtioaux::recv_link(self.auxno) {
                         info!("[REP#{}] remote replied after {} packets", self.repno, ping_count);
-                        if self.sync_tsc().is_err() {
+                        if let Err(e) = self.sync_tsc() {
-                            error!("[REP#{}] remote failed to ack TSC", self.repno);
+                            error!("[REP#{}] failed to sync TSC ({})", self.repno, e);
                             self.state = RepeaterState::Failed;
                             return;
                         }