Merge branch 'switching125' into new

artiq/coredevice/core.py

@@ -44,11 +44,11 @@ def rtio_init() -> TNone:
     raise NotImplementedError("syscall not simulated")
 
 @syscall(flags={"nounwind", "nowrite"})
-def rtio_get_counter() -> TInt64:
+def rtio_get_destination_status(linkno: TInt32) -> TBool:
     raise NotImplementedError("syscall not simulated")
 
 @syscall(flags={"nounwind", "nowrite"})
-def drtio_get_link_status(linkno: TInt32) -> TBool:
+def rtio_get_counter() -> TInt64:
     raise NotImplementedError("syscall not simulated")
 
 
@@ -174,12 +174,11 @@ class Core:
             pass
 
     @kernel
-    def get_drtio_link_status(self, linkno):
-        """Return whether the specified DRTIO link is up.
-
+    def get_rtio_destination_status(self, destination):
+        """Returns whether the specified RTIO destination is up.
         This is particularly useful in startup kernels to delay
-        startup until certain DRTIO links are up."""
-        return drtio_get_link_status(linkno)
+        startup until certain DRTIO destinations are up."""
+        return rtio_get_destination_status(destination)
 
     @kernel
     def reset(self):

artiq/coredevice/drtio_dbg.py

@@ -1,17 +0,0 @@
-"""
-DRTIO debugging functions.
-
-Those syscalls are intended for ARTIQ developers only.
-"""
-
-from artiq.language.core import syscall
-from artiq.language.types import TTuple, TInt32, TInt64, TNone
-
-
-@syscall(flags={"nounwind", "nowrite"})
-def drtio_get_packet_counts(linkno: TInt32) -> TTuple([TInt32, TInt32]):
-    raise NotImplementedError("syscall not simulated")
-
-@syscall(flags={"nounwind", "nowrite"})
-def drtio_get_fifo_space_req_count(linkno: TInt32) -> TInt32:
-    raise NotImplementedError("syscall not simulated")

artiq/coredevice/exceptions.py

@@ -91,7 +91,7 @@ class RTIOOverflow(Exception):
     artiq_builtin = True
 
 
-class RTIOLinkError(Exception):
+class RTIODestinationUnreachable(Exception):
     """Raised with a RTIO operation could not be completed due to a DRTIO link
     being down.
     """

artiq/examples/kasli_drtioswitching/device_db.py

@@ -0,0 +1,34 @@
+core_addr = "kasli-1.lab.m-labs.hk"
+
+device_db = {
+    "core": {
+        "type": "local",
+        "module": "artiq.coredevice.core",
+        "class": "Core",
+        "arguments": {"host": core_addr, "ref_period": 1/(8*150e6)}
+    },
+    "core_log": {
+        "type": "controller",
+        "host": "::1",
+        "port": 1068,
+        "command": "aqctl_corelog -p {port} --bind {bind} " + core_addr
+    },
+    "core_cache": {
+        "type": "local",
+        "module": "artiq.coredevice.cache",
+        "class": "CoreCache"
+    },
+    "core_dma": {
+        "type": "local",
+        "module": "artiq.coredevice.dma",
+        "class": "CoreDMA"
+    },
+}
+
+for i in range(3):
+    device_db["led" + str(i)] = {
+        "type": "local",
+        "module": "artiq.coredevice.ttl",
+        "class": "TTLOut",
+        "arguments": {"channel": i << 16},
+    }

artiq/examples/kasli_drtioswitching/repository/blink.py

@@ -0,0 +1,16 @@
+from artiq.experiment import *
+
+
+class Blink(EnvExperiment):
+    def build(self):
+        self.setattr_device("core")
+        self.leds = [self.get_device("led0"), self.get_device("led2")]
+
+    @kernel
+    def run(self):
+        self.core.reset()
+
+        while True:
+            for led in self.leds:
+                led.pulse(200*ms)
+                delay(200*ms)

artiq/examples/kasli_sawgmaster/repository/sines_2sayma.py

@@ -10,8 +10,8 @@ class Sines2Sayma(EnvExperiment):
     def run(self):
         while True:
             print("waiting for DRTIO ready...")
-            while not (self.core.get_drtio_link_status(0) and
-                       self.core.get_drtio_link_status(1)):
+            while not (self.core.get_rtio_destination_status(0) and
+                       self.core.get_rtio_destination_status(1)):
                 pass
             print("OK")
 
@@ -27,5 +27,5 @@ class Sines2Sayma(EnvExperiment):
                 # Do not use a sub-multiple of oscilloscope sample rates.
                 sawg.frequency0.set(9*MHz)
 
-            while self.core.get_drtio_link_status(0) and self.core.get_drtio_link_status(1):
+            while self.core.get_rtio_destination_status(0) and self.core.get_rtio_destination_status(1):
                 pass

artiq/examples/kasli_sawgmaster/repository/sines_urukul_sayma.py

@@ -23,7 +23,7 @@ class SinesUrukulSayma(EnvExperiment):
 
         while True:
             print("waiting for DRTIO ready...")
-            while not self.core.get_drtio_link_status(0):
+            while not self.core.get_rtio_destination_status(0):
                 pass
             print("OK")
 
@@ -38,5 +38,5 @@ class SinesUrukulSayma(EnvExperiment):
                 sawg.amplitude1.set(.4)
                 sawg.frequency0.set(9*MHz)
 
-            while self.core.get_drtio_link_status(0):
+            while self.core.get_rtio_destination_status(0):
                 pass

artiq/examples/sayma_masterdac/repository/sines_drtio.py

@@ -10,7 +10,7 @@ class SAWGTestDRTIO(EnvExperiment):
     @kernel
     def run(self):
         core_log("waiting for DRTIO ready...")
-        while not self.core.get_drtio_link_status(0):
+        while not self.core.get_rtio_destination_status(0):
             pass
         core_log("OK")
 

artiq/firmware/ksupport/api.rs

@@ -95,6 +95,7 @@ static mut API: &'static [(&'static str, *const ())] = &[
 
     /* direct syscalls */
     api!(rtio_init = ::rtio::init),
+    api!(rtio_get_destination_status = ::rtio::get_destination_status),
     api!(rtio_get_counter = ::rtio::get_counter),
     api!(rtio_log),
     api!(rtio_output = ::rtio::output),
@@ -108,10 +109,6 @@ static mut API: &'static [(&'static str, *const ())] = &[
     api!(dma_retrieve = ::dma_retrieve),
     api!(dma_playback = ::dma_playback),
 
-    api!(drtio_get_link_status = ::rtio::drtio::get_link_status),
-    api!(drtio_get_packet_counts = ::rtio::drtio::get_packet_counts),
-    api!(drtio_get_buffer_space_req_count = ::rtio::drtio::get_buffer_space_req_count),
-
     api!(i2c_start = ::nrt_bus::i2c::start),
     api!(i2c_restart = ::nrt_bus::i2c::restart),
     api!(i2c_stop = ::nrt_bus::i2c::stop),

artiq/firmware/ksupport/lib.rs

@@ -428,8 +428,8 @@ extern fn dma_playback(timestamp: i64, ptr: i32) {
                     timestamp as i64, channel as i64, 0);
             }
             if error & 2 != 0 {
-                raise!("RTIOLinkError",
-                    "RTIO output link error at {0} mu, channel {1}",
+                raise!("RTIODestinationUnreachable",
+                    "RTIO destination unreachable, output, at {0} mu, channel {1}",
                     timestamp as i64, channel as i64, 0);
             }
         }

artiq/firmware/ksupport/rtio.rs

@@ -5,20 +5,30 @@ mod imp {
 
     use board_misoc::csr;
     use ::send;
+    use ::recv;
     use kernel_proto::*;
 
-    pub const RTIO_O_STATUS_WAIT:           u8 = 1;
-    pub const RTIO_O_STATUS_UNDERFLOW:      u8 = 2;
-    pub const RTIO_O_STATUS_LINK_ERROR:     u8 = 4;
-    pub const RTIO_I_STATUS_WAIT_EVENT:     u8 = 1;
-    pub const RTIO_I_STATUS_OVERFLOW:       u8 = 2;
-    pub const RTIO_I_STATUS_WAIT_STATUS:    u8 = 4;
-    pub const RTIO_I_STATUS_LINK_ERROR:     u8 = 8;
+    pub const RTIO_O_STATUS_WAIT:                      u8 = 1;
+    pub const RTIO_O_STATUS_UNDERFLOW:                 u8 = 2;
+    pub const RTIO_O_STATUS_DESTINATION_UNREACHABLE:   u8 = 4;
+    pub const RTIO_I_STATUS_WAIT_EVENT:                u8 = 1;
+    pub const RTIO_I_STATUS_OVERFLOW:                  u8 = 2;
+    pub const RTIO_I_STATUS_WAIT_STATUS:               u8 = 4;
+    pub const RTIO_I_STATUS_DESTINATION_UNREACHABLE:   u8 = 8;
 
     pub extern fn init() {
         send(&RtioInitRequest);
     }
 
+    pub extern fn get_destination_status(destination: i32) -> bool {
+        if 0 <= destination && destination <= 255 {
+            send(&RtioDestinationStatusRequest { destination: destination as u8 });
+            recv!(&RtioDestinationStatusReply { up } => up)
+        } else {
+            false
+        }
+    }
+
     pub extern fn get_counter() -> i64 {
         unsafe {
             csr::rtio::counter_update_write(1);
@@ -49,9 +59,9 @@ mod imp {
                 "RTIO underflow at {0} mu, channel {1}, slack {2} mu",
                 timestamp, channel as i64, timestamp - get_counter());
         }
-        if status & RTIO_O_STATUS_LINK_ERROR != 0 {
-            raise!("RTIOLinkError",
-                "RTIO output link error at {0} mu, channel {1}",
+        if status & RTIO_O_STATUS_DESTINATION_UNREACHABLE != 0 {
+            raise!("RTIODestinationUnreachable",
+                "RTIO destination unreachable, output, at {0} mu, channel {1}",
                 timestamp, channel as i64, 0);
         }
     }
@@ -108,9 +118,9 @@ mod imp {
             if status & RTIO_I_STATUS_WAIT_EVENT != 0 {
                 return !0
             }
-            if status & RTIO_I_STATUS_LINK_ERROR != 0 {
-                raise!("RTIOLinkError",
-                    "RTIO input link error on channel {0}",
+            if status & RTIO_I_STATUS_DESTINATION_UNREACHABLE != 0 {
+                raise!("RTIODestinationUnreachable",
+                    "RTIO destination unreachable, input, on channel {0}",
                     channel as i64, 0, 0);
             }
 
@@ -135,9 +145,9 @@ mod imp {
                     "RTIO input overflow on channel {0}",
                     channel as i64, 0, 0);
             }
-            if status & RTIO_I_STATUS_LINK_ERROR != 0 {
-                raise!("RTIOLinkError",
-                    "RTIO input link error on channel {0}",
+            if status & RTIO_I_STATUS_DESTINATION_UNREACHABLE != 0 {
+                raise!("RTIODestinationUnreachable",
+                    "RTIO destination unreachable, input, on channel {0}",
                     channel as i64, 0, 0);
             }
 
@@ -209,29 +219,3 @@ mod imp {
 }
 
 pub use self::imp::*;
-
-pub mod drtio {
-    use ::send;
-    use ::recv;
-    use kernel_proto::*;
-
-    pub extern fn get_link_status(linkno: i32) -> bool {
-        send(&DrtioLinkStatusRequest { linkno: linkno as u8 });
-        recv!(&DrtioLinkStatusReply { up } => up)
-    }
-
-    #[repr(C)]
-    pub struct PacketCounts(i32, i32);
-
-    pub extern fn get_packet_counts(linkno: i32) -> PacketCounts {
-        send(&DrtioPacketCountRequest { linkno: linkno as u8 });
-        recv!(&DrtioPacketCountReply { tx_cnt, rx_cnt }
-              => PacketCounts(tx_cnt as i32, rx_cnt as i32))
-    }
-
-    pub extern fn get_buffer_space_req_count(linkno: i32) -> i32 {
-        send(&DrtioBufferSpaceReqCountRequest { linkno: linkno as u8 });
-        recv!(&DrtioBufferSpaceReqCountReply { cnt }
-              => cnt as i32)
-    }
-}

artiq/firmware/libboard_artiq/drtio_routing.rs

@@ -0,0 +1,99 @@
+use board_misoc::config;
+#[cfg(has_drtio_routing)]
+use board_misoc::csr;
+use core::fmt;
+
+#[cfg(has_drtio_routing)]
+pub const DEST_COUNT: usize = 256;
+#[cfg(not(has_drtio_routing))]
+pub const DEST_COUNT: usize = 0;
+pub const MAX_HOPS: usize = 32;
+pub const INVALID_HOP: u8 = 0xff;
+
+pub struct RoutingTable(pub [[u8; MAX_HOPS]; DEST_COUNT]);
+
+impl RoutingTable {
+    // default routing table is for star topology with no hops
+    pub fn default_master(default_n_links: usize) -> RoutingTable {
+        let mut ret = RoutingTable([[INVALID_HOP; MAX_HOPS]; DEST_COUNT]);
+        for i in 0..default_n_links {
+            ret.0[i][0] = i as u8;
+        }
+        for i in 1..default_n_links {
+            ret.0[i][1] = 0x00;
+        }
+        ret
+    }
+
+    // use this by default on satellite, as they receive
+    // the routing table from the master
+    pub fn default_empty() -> RoutingTable {
+        RoutingTable([[INVALID_HOP; MAX_HOPS]; DEST_COUNT])
+    }
+}
+
+impl fmt::Display for RoutingTable {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "RoutingTable {{")?;
+        for i in 0..DEST_COUNT {
+            if self.0[i][0] != INVALID_HOP {
+                write!(f, " {}:", i)?;
+                for j in 0..MAX_HOPS {
+                    if self.0[i][j] == INVALID_HOP {
+                        break;
+                    }
+                    write!(f, " {}", self.0[i][j])?;
+                }
+                write!(f, ";")?;
+            }
+        }
+        write!(f, " }}")?;
+        Ok(())
+    }
+}
+
+pub fn config_routing_table(default_n_links: usize) -> RoutingTable {
+    let mut ret = RoutingTable::default_master(default_n_links);
+    let ok = config::read("routing_table", |result| {
+        if let Ok(data) = result {
+            if data.len() == DEST_COUNT*MAX_HOPS {
+                for i in 0..DEST_COUNT {
+                    for j in 0..MAX_HOPS {
+                        ret.0[i][j] = data[i*MAX_HOPS+j];
+                    }
+                }
+                return true;
+            }
+        }
+        false
+    });
+    if !ok {
+        warn!("could not read routing table from configuration, using default");
+    }
+    info!("routing table: {}", ret);
+    ret
+}
+
+#[cfg(has_drtio_routing)]
+pub fn interconnect_enable(routing_table: &RoutingTable, rank: u8, destination: u8) {
+    let hop = routing_table.0[destination as usize][rank as usize];
+    unsafe {
+        csr::routing_table::destination_write(destination);
+        csr::routing_table::hop_write(hop);
+    }
+}
+
+#[cfg(has_drtio_routing)]
+pub fn interconnect_disable(destination: u8) {
+    unsafe {
+        csr::routing_table::destination_write(destination);
+        csr::routing_table::hop_write(INVALID_HOP);
+    }
+}
+
+#[cfg(has_drtio_routing)]
+pub fn interconnect_enable_all(routing_table: &RoutingTable, rank: u8) {
+    for i in 0..DEST_COUNT {
+        interconnect_enable(routing_table, rank, i as u8);
+    }
+}

artiq/firmware/libboard_artiq/drtioaux.rs

@@ -2,7 +2,7 @@ use core::slice;
 use crc;
 
 use io::{ProtoRead, ProtoWrite, Cursor, Error as IoError};
-use board_misoc::{csr::DRTIO, mem::DRTIO_AUX, clock};
+use board_misoc::{csr::DRTIOAUX, mem::DRTIOAUX_MEM, clock};
 use proto_artiq::drtioaux_proto::Error as ProtocolError;
 
 pub use proto_artiq::drtioaux_proto::Packet;
@@ -10,14 +10,21 @@ pub use proto_artiq::drtioaux_proto::Packet;
 // this is parametric over T because there's no impl Fail for !.
 #[derive(Fail, Debug)]
 pub enum Error<T> {
-    #[fail(display = "packet CRC failed")]
-    CorruptedPacket,
-    #[fail(display = "timed out waiting for data")]
-    TimedOut,
-    #[fail(display = "invalid node number")]
-    NoRoute,
     #[fail(display = "gateware reported error")]
     GatewareError,
+    #[fail(display = "packet CRC failed")]
+    CorruptedPacket,
+
+    #[fail(display = "link is down")]
+    LinkDown,
+    #[fail(display = "timed out waiting for data")]
+    TimedOut,
+    #[fail(display = "unexpected reply")]
+    UnexpectedReply,
+
+    #[fail(display = "routing error")]
+    RoutingError,
+
     #[fail(display = "protocol error: {}", _0)]
     Protocol(#[cause] ProtocolError<T>)
 }
@@ -40,17 +47,17 @@ pub fn reset(linkno: u8) {
         // clear buffer first to limit race window with buffer overflow
         // error. We assume the CPU is fast enough so that no two packets
         // will be received between the buffer and the error flag are cleared.
-        (DRTIO[linkno].aux_rx_present_write)(1);
-        (DRTIO[linkno].aux_rx_error_write)(1);
+        (DRTIOAUX[linkno].aux_rx_present_write)(1);
+        (DRTIOAUX[linkno].aux_rx_error_write)(1);
     }
 }
 
 fn has_rx_error(linkno: u8) -> bool {
     let linkno = linkno as usize;
     unsafe {
-        let error = (DRTIO[linkno].aux_rx_error_read)() != 0;
+        let error = (DRTIOAUX[linkno].aux_rx_error_read)() != 0;
         if error {
-            (DRTIO[linkno].aux_rx_error_write)(1)
+            (DRTIOAUX[linkno].aux_rx_error_write)(1)
         }
         error
     }
@@ -61,11 +68,11 @@ fn receive<F, T>(linkno: u8, f: F) -> Result<Option<T>, Error<!>>
 {
     let linkidx = linkno as usize;
     unsafe {
-        if (DRTIO[linkidx].aux_rx_present_read)() == 1 {
-            let ptr = DRTIO_AUX[linkidx].base + DRTIO_AUX[linkidx].size / 2;
-            let len = (DRTIO[linkidx].aux_rx_length_read)();
+        if (DRTIOAUX[linkidx].aux_rx_present_read)() == 1 {
+            let ptr = DRTIOAUX_MEM[linkidx].base + DRTIOAUX_MEM[linkidx].size / 2;
+            let len = (DRTIOAUX[linkidx].aux_rx_length_read)();
             let result = f(slice::from_raw_parts(ptr as *mut u8, len as usize));
-            (DRTIO[linkidx].aux_rx_present_write)(1);
+            (DRTIOAUX[linkidx].aux_rx_present_write)(1);
             Ok(Some(result?))
         } else {
             Ok(None)
@@ -73,7 +80,7 @@ fn receive<F, T>(linkno: u8, f: F) -> Result<Option<T>, Error<!>>
     }
 }
 
-pub fn recv_link(linkno: u8) -> Result<Option<Packet>, Error<!>> {
+pub fn recv(linkno: u8) -> Result<Option<Packet>, Error<!>> {
     if has_rx_error(linkno) {
         return Err(Error::GatewareError)
     }
@@ -97,11 +104,11 @@ pub fn recv_link(linkno: u8) -> Result<Option<Packet>, Error<!>> {
     })
 }
 
-pub fn recv_timeout_link(linkno: u8, timeout_ms: Option<u64>) -> Result<Packet, Error<!>> {
+pub fn recv_timeout(linkno: u8, timeout_ms: Option<u64>) -> Result<Packet, Error<!>> {
     let timeout_ms = timeout_ms.unwrap_or(10);
     let limit = clock::get_ms() + timeout_ms;
     while clock::get_ms() < limit {
-        match recv_link(linkno)? {
+        match recv(linkno)? {
             None => (),
             Some(packet) => return Ok(packet),
         }
@@ -114,17 +121,17 @@ fn transmit<F>(linkno: u8, f: F) -> Result<(), Error<!>>
 {
     let linkno = linkno as usize;
     unsafe {
-        while (DRTIO[linkno].aux_tx_read)() != 0 {}
-        let ptr = DRTIO_AUX[linkno].base;
-        let len = DRTIO_AUX[linkno].size / 2;
+        while (DRTIOAUX[linkno].aux_tx_read)() != 0 {}
+        let ptr = DRTIOAUX_MEM[linkno].base;
+        let len = DRTIOAUX_MEM[linkno].size / 2;
         let len = f(slice::from_raw_parts_mut(ptr as *mut u8, len))?;
-        (DRTIO[linkno].aux_tx_length_write)(len as u16);
-        (DRTIO[linkno].aux_tx_write)(1);
+        (DRTIOAUX[linkno].aux_tx_length_write)(len as u16);
+        (DRTIOAUX[linkno].aux_tx_write)(1);
         Ok(())
     }
 }
 
-pub fn send_link(linkno: u8, packet: &Packet) -> Result<(), Error<!>> {
+pub fn send(linkno: u8, packet: &Packet) -> Result<(), Error<!>> {
     transmit(linkno, |buffer| {
         let mut writer = Cursor::new(buffer);
 
@@ -143,26 +150,3 @@ pub fn send_link(linkno: u8, packet: &Packet) -> Result<(), Error<!>> {
         Ok(writer.position())
     })
 }
-
-// TODO: routing
-fn get_linkno(nodeno: u8) -> Result<u8, Error<!>> {
-    if nodeno == 0 || nodeno as usize > DRTIO.len() {
-        return Err(Error::NoRoute)
-    }
-    Ok(nodeno - 1)
-}
-
-pub fn recv(nodeno: u8) -> Result<Option<Packet>, Error<!>> {
-    let linkno = get_linkno(nodeno)?;
-    recv_link(linkno)
-}
-
-pub fn recv_timeout(nodeno: u8, timeout_ms: Option<u64>) -> Result<Packet, Error<!>> {
-    let linkno = get_linkno(nodeno)?;
-    recv_timeout_link(linkno, timeout_ms)
-}
-
-pub fn send(nodeno: u8, packet: &Packet) -> Result<(), Error<!>> {
-    let linkno = get_linkno(nodeno)?;
-    send_link(linkno, packet)
-}

artiq/firmware/libboard_artiq/lib.rs

@@ -48,3 +48,4 @@ pub mod grabber;
 
 #[cfg(has_drtio)]
 pub mod drtioaux;
+pub mod drtio_routing;

artiq/firmware/libproto_artiq/drtioaux_proto.rs

@@ -14,38 +14,43 @@ impl<T> From<IoError<T>> for Error<T> {
     }
 }
 
-#[derive(Debug)]
+#[derive(PartialEq, Debug)]
 pub enum Packet {
     EchoRequest,
     EchoReply,
-    ResetRequest { phy: bool },
+    ResetRequest,
     ResetAck,
     TSCAck,
 
-    RtioErrorRequest,
-    RtioNoErrorReply,
-    RtioErrorSequenceErrorReply { channel: u16 },
-    RtioErrorCollisionReply { channel: u16 },
-    RtioErrorBusyReply { channel: u16 },
+    DestinationStatusRequest { destination: u8 },
+    DestinationDownReply,
+    DestinationOkReply,
+    DestinationSequenceErrorReply { channel: u16 },
+    DestinationCollisionReply { channel: u16 },
+    DestinationBusyReply { channel: u16 },
 
-    MonitorRequest { channel: u16, probe: u8 },
+    RoutingSetPath { destination: u8, hops: [u8; 32] },
+    RoutingSetRank { rank: u8 },
+    RoutingAck,
+
+    MonitorRequest { destination: u8, channel: u16, probe: u8 },
     MonitorReply { value: u32 },
-    InjectionRequest { channel: u16, overrd: u8, value: u8 },
-    InjectionStatusRequest { channel: u16, overrd: u8 },
+    InjectionRequest { destination: u8, channel: u16, overrd: u8, value: u8 },
+    InjectionStatusRequest { destination: u8, channel: u16, overrd: u8 },
     InjectionStatusReply { value: u8 },
 
-    I2cStartRequest { busno: u8 },
-    I2cRestartRequest { busno: u8 },
-    I2cStopRequest { busno: u8 },
-    I2cWriteRequest { busno: u8, data: u8 },
+    I2cStartRequest { destination: u8, busno: u8 },
+    I2cRestartRequest { destination: u8, busno: u8 },
+    I2cStopRequest { destination: u8, busno: u8 },
+    I2cWriteRequest { destination: u8, busno: u8, data: u8 },
     I2cWriteReply { succeeded: bool, ack: bool },
-    I2cReadRequest { busno: u8, ack: bool },
+    I2cReadRequest { destination: u8, busno: u8, ack: bool },
     I2cReadReply { succeeded: bool, data: u8 },
     I2cBasicReply { succeeded: bool },
 
-    SpiSetConfigRequest { busno: u8, flags: u8, length: u8, div: u8, cs: u8 },
-    SpiWriteRequest { busno: u8, data: u32 },
-    SpiReadRequest { busno: u8 },
+    SpiSetConfigRequest { destination: u8, busno: u8, flags: u8, length: u8, div: u8, cs: u8 },
+    SpiWriteRequest { destination: u8, busno: u8, data: u32 },
+    SpiReadRequest { destination: u8, busno: u8 },
     SpiReadReply { succeeded: bool, data: u32 },
     SpiBasicReply { succeeded: bool },
 }
@@ -57,25 +62,41 @@ impl Packet {
         Ok(match reader.read_u8()? {
             0x00 => Packet::EchoRequest,
             0x01 => Packet::EchoReply,
-            0x02 => Packet::ResetRequest {
-                phy: reader.read_bool()?
-            },
+            0x02 => Packet::ResetRequest,
             0x03 => Packet::ResetAck,
             0x04 => Packet::TSCAck,
 
-            0x20 => Packet::RtioErrorRequest,
-            0x21 => Packet::RtioNoErrorReply,
-            0x22 => Packet::RtioErrorSequenceErrorReply {
+            0x20 => Packet::DestinationStatusRequest {
+                destination: reader.read_u8()?
+            },
+            0x21 => Packet::DestinationDownReply,
+            0x22 => Packet::DestinationOkReply,
+            0x23 => Packet::DestinationSequenceErrorReply {
                 channel: reader.read_u16()?
             },
-            0x23 => Packet::RtioErrorCollisionReply {
+            0x24 => Packet::DestinationCollisionReply {
                 channel: reader.read_u16()?
             },
-            0x24 => Packet::RtioErrorBusyReply {
+            0x25 => Packet::DestinationBusyReply {
                 channel: reader.read_u16()?
             },
 
+            0x30 => {
+                let destination = reader.read_u8()?;
+                let mut hops = [0; 32];
+                reader.read_exact(&mut hops)?;
+                Packet::RoutingSetPath {
+                    destination: destination,
+                    hops: hops
+                }
+            },
+            0x31 => Packet::RoutingSetRank {
+                rank: reader.read_u8()?
+            },
+            0x32 => Packet::RoutingAck,
+
             0x40 => Packet::MonitorRequest {
+                destination: reader.read_u8()?,
                 channel: reader.read_u16()?,
                 probe: reader.read_u8()?
             },
@@ -83,11 +104,13 @@ impl Packet {
                 value: reader.read_u32()?
             },
             0x50 => Packet::InjectionRequest {
+                destination: reader.read_u8()?,
                 channel: reader.read_u16()?,
                 overrd: reader.read_u8()?,
                 value: reader.read_u8()?
             },
             0x51 => Packet::InjectionStatusRequest {
+                destination: reader.read_u8()?,
                 channel: reader.read_u16()?,
                 overrd: reader.read_u8()?
             },
@@ -96,15 +119,19 @@ impl Packet {
             },
 
             0x80 => Packet::I2cStartRequest {
+                destination: reader.read_u8()?,
                 busno: reader.read_u8()?
             },
             0x81 => Packet::I2cRestartRequest {
+                destination: reader.read_u8()?,
                 busno: reader.read_u8()?
             },
             0x82 => Packet::I2cStopRequest {
+                destination: reader.read_u8()?,
                 busno: reader.read_u8()?
             },
             0x83 => Packet::I2cWriteRequest {
+                destination: reader.read_u8()?,
                 busno: reader.read_u8()?,
                 data: reader.read_u8()?
             },
@@ -113,6 +140,7 @@ impl Packet {
                 ack: reader.read_bool()?
             },
             0x85 => Packet::I2cReadRequest {
+                destination: reader.read_u8()?,
                 busno: reader.read_u8()?,
                 ack: reader.read_bool()?
             },
@@ -125,6 +153,7 @@ impl Packet {
             },
 
             0x90 => Packet::SpiSetConfigRequest {
+                destination: reader.read_u8()?,
                 busno: reader.read_u8()?,
                 flags: reader.read_u8()?,
                 length: reader.read_u8()?,
@@ -133,10 +162,12 @@ impl Packet {
             },
             /* 0x91: was Packet::SpiSetXferRequest */
             0x92 => Packet::SpiWriteRequest {
+                destination: reader.read_u8()?,
                 busno: reader.read_u8()?,
                 data: reader.read_u32()?
             },
             0x93 => Packet::SpiReadRequest {
+                destination: reader.read_u8()?,
                 busno: reader.read_u8()?
             },
             0x94 => Packet::SpiReadReply {
@@ -159,34 +190,49 @@ impl Packet {
                 writer.write_u8(0x00)?,
             Packet::EchoReply =>
                 writer.write_u8(0x01)?,
-            Packet::ResetRequest { phy } => {
-                writer.write_u8(0x02)?;
-                writer.write_bool(phy)?;
-            },
+            Packet::ResetRequest =>
+                writer.write_u8(0x02)?,
             Packet::ResetAck =>
                 writer.write_u8(0x03)?,
             Packet::TSCAck =>
                 writer.write_u8(0x04)?,
 
-            Packet::RtioErrorRequest =>
-                writer.write_u8(0x20)?,
-            Packet::RtioNoErrorReply =>
-                writer.write_u8(0x21)?,
-            Packet::RtioErrorSequenceErrorReply { channel } => {
-                writer.write_u8(0x22)?;
-                writer.write_u16(channel)?;
+            Packet::DestinationStatusRequest { destination } => {
+                writer.write_u8(0x20)?;
+                writer.write_u8(destination)?;
             },
-            Packet::RtioErrorCollisionReply { channel } => {
+            Packet::DestinationDownReply =>
+                writer.write_u8(0x21)?,
+            Packet::DestinationOkReply =>
+                writer.write_u8(0x22)?,
+            Packet::DestinationSequenceErrorReply { channel } => {
                 writer.write_u8(0x23)?;
                 writer.write_u16(channel)?;
             },
-            Packet::RtioErrorBusyReply { channel } => {
+            Packet::DestinationCollisionReply { channel } => {
                 writer.write_u8(0x24)?;
                 writer.write_u16(channel)?;
             },
+            Packet::DestinationBusyReply { channel } => {
+                writer.write_u8(0x25)?;
+                writer.write_u16(channel)?;
+            },
 
-            Packet::MonitorRequest { channel, probe } => {
+            Packet::RoutingSetPath { destination, hops } => {
+                writer.write_u8(0x30)?;
+                writer.write_u8(destination)?;
+                writer.write_all(&hops)?;
+            },
+            Packet::RoutingSetRank { rank } => {
+                writer.write_u8(0x31)?;
+                writer.write_u8(rank)?;
+            },
+            Packet::RoutingAck =>
+                writer.write_u8(0x32)?,
+
+            Packet::MonitorRequest { destination, channel, probe } => {
                 writer.write_u8(0x40)?;
+                writer.write_u8(destination)?;
                 writer.write_u16(channel)?;
                 writer.write_u8(probe)?;
             },
@@ -194,14 +240,16 @@ impl Packet {
                 writer.write_u8(0x41)?;
                 writer.write_u32(value)?;
             },
-            Packet::InjectionRequest { channel, overrd, value } => {
+            Packet::InjectionRequest { destination, channel, overrd, value } => {
                 writer.write_u8(0x50)?;
+                writer.write_u8(destination)?;
                 writer.write_u16(channel)?;
                 writer.write_u8(overrd)?;
                 writer.write_u8(value)?;
             },
-            Packet::InjectionStatusRequest { channel, overrd } => {
+            Packet::InjectionStatusRequest { destination, channel, overrd } => {
                 writer.write_u8(0x51)?;
+                writer.write_u8(destination)?;
                 writer.write_u16(channel)?;
                 writer.write_u8(overrd)?;
             },
@@ -210,20 +258,24 @@ impl Packet {
                 writer.write_u8(value)?;
             },
 
-            Packet::I2cStartRequest { busno } => {
+            Packet::I2cStartRequest { destination, busno } => {
                 writer.write_u8(0x80)?;
+                writer.write_u8(destination)?;
                 writer.write_u8(busno)?;
             },
-            Packet::I2cRestartRequest { busno } => {
+            Packet::I2cRestartRequest { destination, busno } => {
                 writer.write_u8(0x81)?;
+                writer.write_u8(destination)?;
                 writer.write_u8(busno)?;
             },
-            Packet::I2cStopRequest { busno } => {
+            Packet::I2cStopRequest { destination, busno } => {
                 writer.write_u8(0x82)?;
+                writer.write_u8(destination)?;
                 writer.write_u8(busno)?;
             },
-            Packet::I2cWriteRequest { busno, data } => {
+            Packet::I2cWriteRequest { destination, busno, data } => {
                 writer.write_u8(0x83)?;
+                writer.write_u8(destination)?;
                 writer.write_u8(busno)?;
                 writer.write_u8(data)?;
             },
@@ -232,8 +284,9 @@ impl Packet {
                 writer.write_bool(succeeded)?;
                 writer.write_bool(ack)?;
             },
-            Packet::I2cReadRequest { busno, ack } => {
+            Packet::I2cReadRequest { destination, busno, ack } => {
                 writer.write_u8(0x85)?;
+                writer.write_u8(destination)?;
                 writer.write_u8(busno)?;
                 writer.write_bool(ack)?;
             },
@@ -247,21 +300,24 @@ impl Packet {
                 writer.write_bool(succeeded)?;
             },
 
-            Packet::SpiSetConfigRequest { busno, flags, length, div, cs } => {
+            Packet::SpiSetConfigRequest { destination, busno, flags, length, div, cs } => {
                 writer.write_u8(0x90)?;
+                writer.write_u8(destination)?;
                 writer.write_u8(busno)?;
                 writer.write_u8(flags)?;
                 writer.write_u8(length)?;
                 writer.write_u8(div)?;
                 writer.write_u8(cs)?;
             },
-            Packet::SpiWriteRequest { busno, data } => {
+            Packet::SpiWriteRequest { destination, busno, data } => {
                 writer.write_u8(0x92)?;
+                writer.write_u8(destination)?;
                 writer.write_u8(busno)?;
                 writer.write_u32(data)?;
             },
-            Packet::SpiReadRequest { busno } => {
+            Packet::SpiReadRequest { destination, busno } => {
                 writer.write_u8(0x93)?;
+                writer.write_u8(destination)?;
                 writer.write_u8(busno)?;
             },
             Packet::SpiReadReply { succeeded, data } => {

artiq/firmware/libproto_artiq/kernel_proto.rs

@@ -28,6 +28,9 @@ pub enum Message<'a> {
 
     RtioInitRequest,
 
+    RtioDestinationStatusRequest { destination: u8 },
+    RtioDestinationStatusReply { up: bool },
+
     DmaRecordStart(&'a str),
     DmaRecordAppend(&'a [u8]),
     DmaRecordStop {
@@ -46,14 +49,6 @@ pub enum Message<'a> {
         duration: u64
     },
 
-    DrtioLinkStatusRequest { linkno: u8 },
-    DrtioLinkStatusReply { up: bool },
-
-    DrtioPacketCountRequest { linkno: u8 },
-    DrtioPacketCountReply { tx_cnt: u32, rx_cnt: u32 },
-    DrtioBufferSpaceReqCountRequest { linkno: u8 },
-    DrtioBufferSpaceReqCountReply { cnt: u32 },
-
     RunFinished,
     RunException {
         exception: Exception<'a>,

artiq/firmware/runtime/kern_hwreq.rs

@@ -1,20 +1,30 @@
+use core::cell::RefCell;
 use kernel_proto as kern;
-use sched::{Io, Error as SchedError};
+use sched::{Io, Mutex, Error as SchedError};
 use session::{kern_acknowledge, kern_send, Error};
-#[cfg(has_rtio_core)]
 use rtio_mgt;
+use urc::Urc;
+use board_artiq::drtio_routing;
+use board_artiq::i2c as local_i2c;
+use board_artiq::spi as local_spi;
 
 #[cfg(has_drtio)]
-mod drtio_i2c {
+mod remote_i2c {
     use drtioaux;
+    use rtio_mgt::drtio;
+    use sched::{Io, Mutex};
 
-    fn basic_reply(nodeno: u8) -> Result<(), ()> {
-        match drtioaux::recv_timeout(nodeno, None) {
+    pub fn start(io: &Io, aux_mutex: &Mutex, linkno: u8, destination: u8, busno: u8) -> Result<(), ()> {
+        let reply = drtio::aux_transact(io, aux_mutex, linkno, &drtioaux::Packet::I2cStartRequest {
+            destination: destination,
+            busno: busno
+        });
+        match reply {
             Ok(drtioaux::Packet::I2cBasicReply { succeeded }) => {
                 if succeeded { Ok(()) } else { Err(()) }
             }
-            Ok(_) => {
-                error!("received unexpected aux packet");
+            Ok(packet) => {
+                error!("received unexpected aux packet: {:?}", packet);
                 Err(())
             }
             Err(e) => {
@@ -24,39 +34,53 @@ mod drtio_i2c {
         }
     }
 
-    pub fn start(nodeno: u8, busno: u8) -> Result<(), ()> {
-        let request = drtioaux::Packet::I2cStartRequest { busno: busno };
-        if drtioaux::send(nodeno, &request).is_err() {
-            return Err(())
+    pub fn restart(io: &Io, aux_mutex: &Mutex, linkno: u8, destination: u8, busno: u8) -> Result<(), ()> {
+        let reply = drtio::aux_transact(io, aux_mutex, linkno, &drtioaux::Packet::I2cRestartRequest {
+            destination: destination,
+            busno: busno
+        });
+        match reply {
+            Ok(drtioaux::Packet::I2cBasicReply { succeeded }) => {
+                if succeeded { Ok(()) } else { Err(()) }
+            }
+            Ok(packet) => {
+                error!("received unexpected aux packet: {:?}", packet);
+                Err(())
+            }
+            Err(e) => {
+                error!("aux packet error ({})", e);
+                Err(())
+            }
         }
-        basic_reply(nodeno)
     }
 
-    pub fn restart(nodeno: u8, busno: u8) -> Result<(), ()> {
-        let request = drtioaux::Packet::I2cRestartRequest { busno: busno };
-        if drtioaux::send(nodeno, &request).is_err() {
-            return Err(())
+    pub fn stop(io: &Io, aux_mutex: &Mutex, linkno: u8, destination: u8, busno: u8) -> Result<(), ()> {
+        let reply = drtio::aux_transact(io, aux_mutex, linkno, &drtioaux::Packet::I2cStopRequest  {
+            destination: destination,
+            busno: busno
+        });
+        match reply {
+            Ok(drtioaux::Packet::I2cBasicReply { succeeded }) => {
+                if succeeded { Ok(()) } else { Err(()) }
+            }
+            Ok(packet) => {
+                error!("received unexpected aux packet: {:?}", packet);
+                Err(())
+            }
+            Err(e) => {
+                error!("aux packet error ({})", e);
+                Err(())
+            }
         }
-        basic_reply(nodeno)
     }
 
-    pub fn stop(nodeno: u8, busno: u8) -> Result<(), ()> {
-        let request = drtioaux::Packet::I2cStopRequest { busno: busno };
-        if drtioaux::send(nodeno, &request).is_err() {
-            return Err(())
-        }
-        basic_reply(nodeno)
-    }
-
-    pub fn write(nodeno: u8, busno: u8, data: u8) -> Result<bool, ()> {
-        let request = drtioaux::Packet::I2cWriteRequest {
+    pub fn write(io: &Io, aux_mutex: &Mutex, linkno: u8, destination: u8, busno: u8, data: u8) -> Result<bool, ()> {
+        let reply = drtio::aux_transact(io, aux_mutex, linkno, &drtioaux::Packet::I2cWriteRequest {
+            destination: destination,
             busno: busno,
             data: data
-        };
-        if drtioaux::send(nodeno, &request).is_err() {
-            return Err(())
-        }
-        match drtioaux::recv_timeout(nodeno, None) {
+        });
+        match reply {
             Ok(drtioaux::Packet::I2cWriteReply { succeeded, ack }) => {
                 if succeeded { Ok(ack) } else { Err(()) }
             }
@@ -71,15 +95,13 @@ mod drtio_i2c {
         }
     }
 
-    pub fn read(nodeno: u8, busno: u8, ack: bool) -> Result<u8, ()> {
-        let request = drtioaux::Packet::I2cReadRequest {
+    pub fn read(io: &Io, aux_mutex: &Mutex, linkno: u8, destination: u8, busno: u8, ack: bool) -> Result<u8, ()> {
+        let reply = drtio::aux_transact(io, aux_mutex, linkno, &drtioaux::Packet::I2cReadRequest {
+            destination: destination,
             busno: busno,
             ack: ack
-        };
-        if drtioaux::send(nodeno, &request).is_err() {
-            return Err(())
-        }
-        match drtioaux::recv_timeout(nodeno, None) {
+        });
+        match reply {
             Ok(drtioaux::Packet::I2cReadReply { succeeded, data }) => {
                 if succeeded { Ok(data) } else { Err(()) }
             }
@@ -95,140 +117,68 @@ mod drtio_i2c {
     }
 }
 
-#[cfg(not(has_drtio))]
-mod drtio_i2c {
-    pub fn start(_nodeno: u8, _busno: u8) -> Result<(), ()> {
-        Err(())
-    }
-
-    pub fn restart(_nodeno: u8, _busno: u8) -> Result<(), ()> {
-        Err(())
-    }
-
-    pub fn stop(_nodeno: u8, _busno: u8) -> Result<(), ()> {
-        Err(())
-    }
-
-    pub fn write(_nodeno: u8, _busno: u8, _data: u8) -> Result<bool, ()> {
-        Err(())
-    }
-
-    pub fn read(_nodeno: u8, _busno: u8, _ack: bool) -> Result<u8, ()> {
-        Err(())
-    }
-}
-
-mod i2c {
-    use board_artiq::i2c as local_i2c;
-    use super::drtio_i2c;
-
-    pub fn start(busno: u32) -> Result<(), ()> {
-        let nodeno = (busno >> 16) as u8;
-        let node_busno = busno as u8;
-        if nodeno == 0 {
-            local_i2c::start(node_busno)
-        } else {
-            drtio_i2c::start(nodeno, node_busno)
-        }
-    }
-
-    pub fn restart(busno: u32) -> Result<(), ()> {
-        let nodeno = (busno >> 16) as u8;
-        let node_busno = busno as u8;
-        if nodeno == 0 {
-            local_i2c::restart(node_busno)
-        } else {
-            drtio_i2c::restart(nodeno, node_busno)
-        }
-    }
-
-    pub fn stop(busno: u32) -> Result<(), ()> {
-        let nodeno = (busno >> 16) as u8;
-        let node_busno = busno as u8;
-        if nodeno == 0 {
-            local_i2c::stop(node_busno)
-        } else {
-            drtio_i2c::stop(nodeno, node_busno)
-        }
-    }
-
-    pub fn write(busno: u32, data: u8) -> Result<bool, ()> {
-        let nodeno = (busno >> 16 )as u8;
-        let node_busno = busno as u8;
-        if nodeno == 0 {
-            local_i2c::write(node_busno, data)
-        } else {
-            drtio_i2c::write(nodeno, node_busno, data)
-        }
-    }
-
-    pub fn read(busno: u32, ack: bool) -> Result<u8, ()> {
-        let nodeno = (busno >> 16) as u8;
-        let node_busno = busno as u8;
-        if nodeno == 0 {
-            local_i2c::read(node_busno, ack)
-        } else {
-            drtio_i2c::read(nodeno, node_busno, ack)
-        }
-    }
-}
-
 #[cfg(has_drtio)]
-mod drtio_spi {
+mod remote_spi {
     use drtioaux;
+    use rtio_mgt::drtio;
+    use sched::{Io, Mutex};
 
-    fn basic_reply(nodeno: u8) -> Result<(), ()> {
-        match drtioaux::recv_timeout(nodeno, None) {
-            Ok(drtioaux::Packet::SpiBasicReply { succeeded }) => {
-                if succeeded { Ok(()) } else { Err(()) }
-            }
-            Ok(_) => {
-                error!("received unexpected aux packet");
-                Err(())
-            }
-            Err(e) => {
-                error!("aux packet error ({})", e);
-                Err(())
-            }
-        }
-    }
-
-    pub fn set_config(nodeno: u8, busno: u8, flags: u8, length: u8, div: u8, cs: u8) -> Result<(), ()> {
-        let request = drtioaux::Packet::SpiSetConfigRequest {
+    pub fn set_config(io: &Io, aux_mutex: &Mutex, linkno: u8, destination: u8, busno: u8, flags: u8, length: u8, div: u8, cs: u8) -> Result<(), ()> {
+        let reply = drtio::aux_transact(io, aux_mutex, linkno, &drtioaux::Packet::SpiSetConfigRequest {
+            destination: destination,
             busno: busno,
             flags: flags,
             length: length,
             div: div,
             cs: cs
-        };
-        if drtioaux::send(nodeno, &request).is_err() {
-            return Err(())
+        });
+        match reply {
+            Ok(drtioaux::Packet::SpiBasicReply { succeeded }) => {
+                if succeeded { Ok(()) } else { Err(()) }
+            }
+            Ok(packet) => {
+                error!("received unexpected aux packet: {:?}", packet);
+                Err(())
+            }
+            Err(e) => {
+                error!("aux packet error ({})", e);
+                Err(())
+            }
         }
-        basic_reply(nodeno)
     }
 
-    pub fn write(nodeno: u8, busno: u8, data: u32) -> Result<(), ()> {
-        let request = drtioaux::Packet::SpiWriteRequest {
+    pub fn write(io: &Io, aux_mutex: &Mutex, linkno: u8, destination: u8, busno: u8, data: u32) -> Result<(), ()> {
+        let reply = drtio::aux_transact(io, aux_mutex, linkno, &drtioaux::Packet::SpiWriteRequest {
+            destination: destination,
             busno: busno,
             data: data
-        };
-        if drtioaux::send(nodeno, &request).is_err() {
-            return Err(())
+        });
+        match reply {
+            Ok(drtioaux::Packet::SpiBasicReply { succeeded }) => {
+                if succeeded { Ok(()) } else { Err(()) }
+            }
+            Ok(packet) => {
+                error!("received unexpected aux packet: {:?}", packet);
+                Err(())
+            }
+            Err(e) => {
+                error!("aux packet error ({})", e);
+                Err(())
+            }
         }
-        basic_reply(nodeno)
     }
 
-    pub fn read(nodeno: u8, busno: u8) -> Result<u32, ()> {
-        let request = drtioaux::Packet::SpiReadRequest { busno: busno };
-        if drtioaux::send(nodeno, &request).is_err() {
-            return Err(())
-        }
-        match drtioaux::recv_timeout(nodeno, None) {
+    pub fn read(io: &Io, aux_mutex: &Mutex, linkno: u8, destination: u8, busno: u8) -> Result<u32, ()> {
+        let reply = drtio::aux_transact(io, aux_mutex, linkno, &drtioaux::Packet::SpiReadRequest {
+            destination: destination,
+            busno: busno
+        });
+        match reply {
             Ok(drtioaux::Packet::SpiReadReply { succeeded, data }) => {
                 if succeeded { Ok(data) } else { Err(()) }
             }
-            Ok(_) => {
-                error!("received unexpected aux packet");
+            Ok(packet) => {
+                error!("received unexpected aux packet: {:?}", packet);
                 Err(())
             }
             Err(e) => {
@@ -239,118 +189,89 @@ mod drtio_spi {
     }
 }
 
+
+#[cfg(has_drtio)]
+macro_rules! dispatch {
+    ($io:ident, $aux_mutex:ident, $mod_local:ident, $mod_remote:ident, $routing_table:ident, $busno:expr, $func:ident $(, $param:expr)*) => {{
+        let destination = ($busno >> 16) as u8;
+        let busno = $busno as u8;
+        let hop = $routing_table.0[destination as usize][0];
+        if hop == 0 {
+            $mod_local::$func(busno, $($param, )*)
+        } else {
+            let linkno = hop - 1;
+            $mod_remote::$func($io, $aux_mutex, linkno, destination, busno, $($param, )*)
+        }
+    }}
+}
+
 #[cfg(not(has_drtio))]
-mod drtio_spi {
-    pub fn set_config(_nodeno: u8, _busno: u8, _flags: u8,
-                      _length: u8, _div: u8, _cs: u8) -> Result<(), ()> {
-        Err(())
-    }
-
-    pub fn write(_nodeno: u8, _busno: u8, _data: u32) -> Result<(), ()> {
-        Err(())
-    }
-
-    pub fn read(_nodeno: u8, _busno: u8) -> Result<u32, ()> {
-        Err(())
-    }
+macro_rules! dispatch {
+    ($io:ident, $aux_mutex:ident,$mod_local:ident, $mod_remote:ident, $routing_table:ident, $busno:expr, $func:ident $(, $param:expr)*) => {{
+        let busno = $busno as u8;
+        $mod_local::$func(busno, $($param, )*)
+    }}
 }
 
-mod spi {
-    use board_artiq::spi as local_spi;
-    use super::drtio_spi;
-
-    pub fn set_config(busno: u32, flags: u8, length: u8, div: u8, cs: u8) -> Result<(), ()> {
-        let nodeno = (busno >> 16) as u8;
-        let node_busno = busno as u8;
-        if nodeno == 0 {
-            local_spi::set_config(node_busno, flags, length, div, cs)
-        } else {
-            drtio_spi::set_config(nodeno, node_busno, flags, length, div, cs)
-        }
-    }
-
-    pub fn write(busno: u32, data: u32) -> Result<(), ()> {
-        let nodeno = (busno >> 16) as u8;
-        let node_busno = busno as u8;
-        if nodeno == 0 {
-            local_spi::write(node_busno, data)
-        } else {
-            drtio_spi::write(nodeno, node_busno, data)
-        }
-    }
-
-    pub fn read(busno: u32) -> Result<u32, ()> {
-        let nodeno = (busno >> 16) as u8;
-        let node_busno = busno as u8;
-        if nodeno == 0 {
-            local_spi::read(node_busno)
-        } else {
-            drtio_spi::read(nodeno, node_busno)
-        }
-    }
-}
-
-pub fn process_kern_hwreq(io: &Io, request: &kern::Message) -> Result<bool, Error<SchedError>> {
+pub fn process_kern_hwreq(io: &Io, aux_mutex: &Mutex,
+        _routing_table: &drtio_routing::RoutingTable,
+        _up_destinations: &Urc<RefCell<[bool; drtio_routing::DEST_COUNT]>>,
+        request: &kern::Message) -> Result<bool, Error<SchedError>> {
     match request {
-        #[cfg(has_rtio_core)]
         &kern::RtioInitRequest => {
             info!("resetting RTIO");
-            rtio_mgt::init_core(false);
+            rtio_mgt::reset(io, aux_mutex);
             kern_acknowledge()
         }
 
-        #[cfg(has_rtio_core)]
-        &kern::DrtioLinkStatusRequest { linkno } => {
-            let up = rtio_mgt::drtio::link_up(linkno);
-            kern_send(io, &kern::DrtioLinkStatusReply { up: up })
-        }
-
-        #[cfg(has_rtio_core)]
-        &kern::DrtioPacketCountRequest { linkno } => {
-            let (tx_cnt, rx_cnt) = rtio_mgt::drtio_dbg::get_packet_counts(linkno);
-            kern_send(io, &kern::DrtioPacketCountReply { tx_cnt: tx_cnt, rx_cnt: rx_cnt })
-        }
-        #[cfg(has_rtio_core)]
-        &kern::DrtioBufferSpaceReqCountRequest { linkno } => {
-            let cnt = rtio_mgt::drtio_dbg::get_buffer_space_req_count(linkno);
-            kern_send(io, &kern::DrtioBufferSpaceReqCountReply { cnt: cnt })
+        &kern::RtioDestinationStatusRequest { destination: _destination } => {
+            #[cfg(has_drtio)]
+            let up = {
+                let up_destinations = _up_destinations.borrow();
+                up_destinations[_destination as usize]
+            };
+            #[cfg(not(has_drtio))]
+            let up = true;
+            kern_send(io, &kern::RtioDestinationStatusReply { up: up })
         }
 
         &kern::I2cStartRequest { busno } => {
-            let succeeded = i2c::start(busno).is_ok();
+            let succeeded = dispatch!(io, aux_mutex, local_i2c, remote_i2c, _routing_table, busno, start).is_ok();
             kern_send(io, &kern::I2cBasicReply { succeeded: succeeded })
         }
         &kern::I2cRestartRequest { busno } => {
-            let succeeded = i2c::restart(busno).is_ok();
+            let succeeded = dispatch!(io, aux_mutex, local_i2c, remote_i2c, _routing_table, busno, restart).is_ok();
             kern_send(io, &kern::I2cBasicReply { succeeded: succeeded })
         }
         &kern::I2cStopRequest { busno } => {
-            let succeeded = i2c::stop(busno).is_ok();
+            let succeeded = dispatch!(io, aux_mutex, local_i2c, remote_i2c, _routing_table, busno, stop).is_ok();
             kern_send(io, &kern::I2cBasicReply { succeeded: succeeded })
         }
         &kern::I2cWriteRequest { busno, data } => {
-            match i2c::write(busno, data) {
+            match dispatch!(io, aux_mutex, local_i2c, remote_i2c, _routing_table, busno, write, data) {
                 Ok(ack) => kern_send(io, &kern::I2cWriteReply { succeeded: true, ack: ack }),
                 Err(_) => kern_send(io, &kern::I2cWriteReply { succeeded: false, ack: false })
             }
         }
         &kern::I2cReadRequest { busno, ack } => {
-            match i2c::read(busno, ack) {
+            match dispatch!(io, aux_mutex, local_i2c, remote_i2c, _routing_table, busno, read, ack) {
                 Ok(data) => kern_send(io, &kern::I2cReadReply { succeeded: true, data: data }),
                 Err(_) => kern_send(io, &kern::I2cReadReply { succeeded: false, data: 0xff })
             }
         }
 
         &kern::SpiSetConfigRequest { busno, flags, length, div, cs } => {
-            let succeeded = spi::set_config(busno, flags, length, div, cs).is_ok();
+            let succeeded = dispatch!(io, aux_mutex, local_spi, remote_spi, _routing_table, busno,
+                set_config, flags, length, div, cs).is_ok();
             kern_send(io, &kern::SpiBasicReply { succeeded: succeeded })
         },
         &kern::SpiWriteRequest { busno, data } => {
-            let succeeded = spi::write(busno, data).is_ok();
+            let succeeded = dispatch!(io, aux_mutex, local_spi, remote_spi, _routing_table, busno,
+                write, data).is_ok();
             kern_send(io, &kern::SpiBasicReply { succeeded: succeeded })
         }
         &kern::SpiReadRequest { busno } => {
-            match spi::read(busno) {
+            match dispatch!(io, aux_mutex, local_spi, remote_spi, _routing_table, busno, read) {
                 Ok(data) => kern_send(io, &kern::SpiReadReply { succeeded: true, data: data }),
                 Err(_) => kern_send(io, &kern::SpiReadReply { succeeded: false, data: 0 })
             }

artiq/firmware/runtime/main.rs

@@ -1,5 +1,6 @@
 #![feature(lang_items, alloc, try_from, nonzero, asm,
-           panic_implementation, panic_info_message)]
+           panic_implementation, panic_info_message,
+           const_slice_len)]
 #![no_std]
 
 extern crate eh;
@@ -25,6 +26,7 @@ extern crate board_artiq;
 extern crate logger_artiq;
 extern crate proto_artiq;
 
+use core::cell::RefCell;
 use core::convert::TryFrom;
 use smoltcp::wire::{EthernetAddress, IpAddress, IpCidr};
 
@@ -33,12 +35,12 @@ use board_misoc::{csr, irq, ident, clock, boot, config};
 use board_misoc::ethmac;
 #[cfg(has_drtio)]
 use board_artiq::drtioaux;
+use board_artiq::drtio_routing;
 use board_artiq::{mailbox, rpc_queue};
-use proto_artiq::{mgmt_proto, moninj_proto, rpc_proto, session_proto,kernel_proto};
+use proto_artiq::{mgmt_proto, moninj_proto, rpc_proto, session_proto, kernel_proto};
 #[cfg(has_rtio_analyzer)]
 use proto_artiq::analyzer_proto;
 
-#[cfg(has_rtio_core)]
 mod rtio_mgt;
 
 mod urc;
@@ -279,16 +281,37 @@ fn startup_ethernet() {
                        .ip_addrs([IpCidr::new(protocol_addr, 0)])
                        .finalize();
 
+    #[cfg(has_drtio)]
+    let drtio_routing_table = urc::Urc::new(RefCell::new(
+        drtio_routing::config_routing_table(csr::DRTIO.len())));
+    #[cfg(not(has_drtio))]
+    let drtio_routing_table = urc::Urc::new(RefCell::new(
+        drtio_routing::RoutingTable::default_empty()));
+    let up_destinations = urc::Urc::new(RefCell::new(
+        [false; drtio_routing::DEST_COUNT]));
+    let aux_mutex = sched::Mutex::new();
+
     let mut scheduler = sched::Scheduler::new();
     let io = scheduler.io();
-    #[cfg(has_rtio_core)]
-    rtio_mgt::startup(&io);
+
+    rtio_mgt::startup(&io, &aux_mutex, &drtio_routing_table, &up_destinations);
+
     io.spawn(4096, mgmt::thread);
-    io.spawn(16384, session::thread);
+    {
+        let aux_mutex = aux_mutex.clone();
+        let drtio_routing_table = drtio_routing_table.clone();
+        let up_destinations = up_destinations.clone();
+        io.spawn(16384, move |io| { session::thread(io, &aux_mutex, &drtio_routing_table, &up_destinations) });
+    }
     #[cfg(any(has_rtio_moninj, has_drtio))]
-    io.spawn(4096, moninj::thread);
+    {
+        let aux_mutex = aux_mutex.clone();
+        let drtio_routing_table = drtio_routing_table.clone();
+        io.spawn(4096, move |io| { moninj::thread(io, &aux_mutex, &drtio_routing_table) });
+    }
     #[cfg(has_rtio_analyzer)]
     io.spawn(4096, analyzer::thread);
+
     #[cfg(has_grabber)]
     io.spawn(4096, grabber_thread);
 

artiq/firmware/runtime/moninj.rs

@@ -1,151 +1,122 @@
 use alloc::btree_map::BTreeMap;
+use core::cell::RefCell;
 
 use io::Error as IoError;
 use moninj_proto::*;
-use sched::{Io, TcpListener, TcpStream, Error as SchedError};
-use board_misoc::{clock, csr};
-#[cfg(has_drtio)]
-use drtioaux;
+use sched::{Io, Mutex, TcpListener, TcpStream, Error as SchedError};
+use urc::Urc;
+use board_misoc::clock;
+use board_artiq::drtio_routing;
 
 #[cfg(has_rtio_moninj)]
-fn read_probe_local(channel: u16, probe: u8) -> u32 {
-    unsafe {
-        csr::rtio_moninj::mon_chan_sel_write(channel as _);
-        csr::rtio_moninj::mon_probe_sel_write(probe);
-        csr::rtio_moninj::mon_value_update_write(1);
-        csr::rtio_moninj::mon_value_read() as u32
+mod local_moninj {
+    use board_misoc::csr;
+
+    pub fn read_probe(channel: u16, probe: u8) -> u32 {
+        unsafe {
+            csr::rtio_moninj::mon_chan_sel_write(channel as _);
+            csr::rtio_moninj::mon_probe_sel_write(probe);
+            csr::rtio_moninj::mon_value_update_write(1);
+            csr::rtio_moninj::mon_value_read() as u32
+        }
+    }
+
+    pub fn inject(channel: u16, overrd: u8, value: u8) {
+        unsafe {
+            csr::rtio_moninj::inj_chan_sel_write(channel as _);
+            csr::rtio_moninj::inj_override_sel_write(overrd);
+            csr::rtio_moninj::inj_value_write(value);
+        }
+    }
+
+    pub fn read_injection_status(channel: u16, overrd: u8) -> u8 {
+        unsafe {
+            csr::rtio_moninj::inj_chan_sel_write(channel as _);
+            csr::rtio_moninj::inj_override_sel_write(overrd);
+            csr::rtio_moninj::inj_value_read()
+        }
+    }
+}
+
+#[cfg(not(has_rtio_moninj))]
+mod local_moninj {
+    pub fn read_probe(_channel: u16, _probe: u8) -> u32 { 0 }
+
+    pub fn inject(_channel: u16, _overrd: u8, _value: u8) { }
+
+    pub fn read_injection_status(_channel: u16, _overrd: u8) -> u8 { 0 }
+}
+
+#[cfg(has_drtio)]
+mod remote_moninj {
+    use drtioaux;
+    use rtio_mgt::drtio;
+    use sched::{Io, Mutex};
+
+    pub fn read_probe(io: &Io, aux_mutex: &Mutex, linkno: u8, destination: u8, channel: u16, probe: u8) -> u32 {
+        let reply = drtio::aux_transact(io, aux_mutex, linkno, &drtioaux::Packet::MonitorRequest { 
+            destination: destination,
+            channel: channel,
+            probe: probe
+        });
+        match reply {
+            Ok(drtioaux::Packet::MonitorReply { value }) => return value,
+            Ok(packet) => error!("received unexpected aux packet: {:?}", packet),
+            Err(e) => error!("aux packet error ({})", e)
+        }
+        0
+    }
+
+    pub fn inject(io: &Io, aux_mutex: &Mutex, linkno: u8, destination: u8, channel: u16, overrd: u8, value: u8) {
+        let _lock = aux_mutex.lock(io).unwrap();
+        drtioaux::send(linkno, &drtioaux::Packet::InjectionRequest {
+            destination: destination,
+            channel: channel,
+            overrd: overrd,
+            value: value
+        }).unwrap();
+    }
+
+    pub fn read_injection_status(io: &Io, aux_mutex: &Mutex, linkno: u8, destination: u8, channel: u16, overrd: u8) -> u8 {
+        let reply = drtio::aux_transact(io, aux_mutex, linkno, &drtioaux::Packet::InjectionStatusRequest {
+            destination: destination,
+            channel: channel,
+            overrd: overrd
+        });
+        match reply {
+            Ok(drtioaux::Packet::InjectionStatusReply { value }) => return value,
+            Ok(packet) => error!("received unexpected aux packet: {:?}", packet),
+            Err(e) => error!("aux packet error ({})", e)
+        }
+        0
     }
 }
 
 #[cfg(has_drtio)]
-fn read_probe_drtio(nodeno: u8, channel: u16, probe: u8) -> u32 {
-    let request = drtioaux::Packet::MonitorRequest { channel: channel, probe: probe };
-    match drtioaux::send(nodeno, &request) {
-        Ok(_) => (),
-        Err(e) => {
-            error!("aux packet error ({})", e);
-            return 0;
+macro_rules! dispatch {
+    ($io:ident, $aux_mutex:ident, $routing_table:ident, $channel:expr, $func:ident $(, $param:expr)*) => {{
+        let destination = ($channel >> 16) as u8;
+        let channel = $channel as u16;
+        let hop = $routing_table.0[destination as usize][0];
+        if hop == 0 {
+            local_moninj::$func(channel, $($param, )*)
+        } else {
+            let linkno = hop - 1;
+            remote_moninj::$func($io, $aux_mutex, linkno, destination, channel, $($param, )*)
         }
-    }
-    match drtioaux::recv_timeout(nodeno, None) {
-        Ok(drtioaux::Packet::MonitorReply { value }) => return value,
-        Ok(_) => error!("received unexpected aux packet"),
-        Err(e) => error!("aux packet error ({})", e)
-    }
-    0
+    }}
 }
 
-fn read_probe(channel: u32, probe: u8) -> u32 {
-    let nodeno = (channel >> 16) as u8;
-    let node_channel = channel as u16;
-    #[cfg(has_rtio_moninj)]
-    {
-        if nodeno == 0 {
-            return read_probe_local(node_channel, probe)
-        }
-    }
-    #[cfg(has_drtio)]
-    {
-        if nodeno != 0 {
-            return read_probe_drtio(nodeno, node_channel, probe)
-        }
-    }
-    error!("read_probe: unrecognized channel number {}", channel);
-    0
+#[cfg(not(has_drtio))]
+macro_rules! dispatch {
+    ($io:ident, $aux_mutex:ident, $routing_table:ident, $channel:expr, $func:ident $(, $param:expr)*) => {{
+        let channel = $channel as u16;
+        local_moninj::$func(channel, $($param, )*)
+    }}
 }
 
-#[cfg(has_rtio_moninj)]
-fn inject_local(channel: u16, overrd: u8, value: u8) {
-    unsafe {
-        csr::rtio_moninj::inj_chan_sel_write(channel as _);
-        csr::rtio_moninj::inj_override_sel_write(overrd);
-        csr::rtio_moninj::inj_value_write(value);
-    }
-}
-
-#[cfg(has_drtio)]
-fn inject_drtio(nodeno: u8, channel: u16, overrd: u8, value: u8) {
-    let request = drtioaux::Packet::InjectionRequest {
-        channel: channel,
-        overrd: overrd,
-        value: value
-    };
-    match drtioaux::send(nodeno, &request) {
-        Ok(_) => (),
-        Err(e) => error!("aux packet error ({})", e)
-    }
-}
-
-fn inject(channel: u32, overrd: u8, value: u8) {
-    let nodeno = (channel >> 16) as u8;
-    let node_channel = channel as u16;
-    #[cfg(has_rtio_moninj)]
-    {
-        if nodeno == 0 {
-            inject_local(node_channel, overrd, value);
-            return
-        }
-    }
-    #[cfg(has_drtio)]
-    {
-        if nodeno != 0 {
-            inject_drtio(nodeno, node_channel, overrd, value);
-            return
-        }
-    }
-    error!("inject: unrecognized channel number {}", channel);
-}
-
-#[cfg(has_rtio_moninj)]
-fn read_injection_status_local(channel: u16, overrd: u8) -> u8 {
-    unsafe {
-        csr::rtio_moninj::inj_chan_sel_write(channel as _);
-        csr::rtio_moninj::inj_override_sel_write(overrd);
-        csr::rtio_moninj::inj_value_read()
-    }
-}
-
-#[cfg(has_drtio)]
-fn read_injection_status_drtio(nodeno: u8, channel: u16, overrd: u8) -> u8 {
-    let request = drtioaux::Packet::InjectionStatusRequest {
-        channel: channel,
-        overrd: overrd
-    };
-    match drtioaux::send(nodeno, &request) {
-        Ok(_) => (),
-        Err(e) => {
-            error!("aux packet error ({})", e);
-            return 0;
-        }
-    }
-    match drtioaux::recv_timeout(nodeno, None) {
-        Ok(drtioaux::Packet::InjectionStatusReply { value }) => return value,
-        Ok(_) => error!("received unexpected aux packet"),
-        Err(e) => error!("aux packet error ({})", e)
-    }
-    0
-}
-
-fn read_injection_status(channel: u32, probe: u8) -> u8 {
-    let nodeno = (channel >> 16) as u8;
-    let node_channel = channel as u16;
-    #[cfg(has_rtio_moninj)]
-    {
-        if nodeno == 0 {
-            return read_injection_status_local(node_channel, probe)
-        }
-    }
-    #[cfg(has_drtio)]
-    {
-        if nodeno != 0 {
-            return read_injection_status_drtio(nodeno, node_channel, probe)
-        }
-    }
-    error!("read_injection_status: unrecognized channel number {}", channel);
-    0
-}
-
-fn connection_worker(io: &Io, mut stream: &mut TcpStream) -> Result<(), Error<SchedError>> {
+fn connection_worker(io: &Io, _aux_mutex: &Mutex, _routing_table: &drtio_routing::RoutingTable,
+        mut stream: &mut TcpStream) -> Result<(), Error<SchedError>> {
     let mut probe_watch_list = BTreeMap::new();
     let mut inject_watch_list = BTreeMap::new();
     let mut next_check = 0;
@@ -173,9 +144,9 @@ fn connection_worker(io: &Io, mut stream: &mut TcpStream) -> Result<(), Error<Sc
                         let _ = inject_watch_list.remove(&(channel, overrd));
                     }
                 },
-                HostMessage::Inject { channel, overrd, value } => inject(channel, overrd, value),
+                HostMessage::Inject { channel, overrd, value } => dispatch!(io, _aux_mutex, _routing_table, channel, inject, overrd, value),
                 HostMessage::GetInjectionStatus { channel, overrd } => {
-                    let value = read_injection_status(channel, overrd);
+                    let value = dispatch!(io, _aux_mutex, _routing_table, channel, read_injection_status, overrd);
                     let reply = DeviceMessage::InjectionStatus {
                         channel: channel,
                         overrd: overrd,
@@ -192,7 +163,7 @@ fn connection_worker(io: &Io, mut stream: &mut TcpStream) -> Result<(), Error<Sc
 
         if clock::get_ms() > next_check {
             for (&(channel, probe), previous) in probe_watch_list.iter_mut() {
-                let current = read_probe(channel, probe);
+                let current = dispatch!(io, _aux_mutex, _routing_table, channel, read_probe, probe);
                 if previous.is_none() || previous.unwrap() != current {
                     let message = DeviceMessage::MonitorStatus {
                         channel: channel,
@@ -207,7 +178,7 @@ fn connection_worker(io: &Io, mut stream: &mut TcpStream) -> Result<(), Error<Sc
                 }
             }
             for (&(channel, overrd), previous) in inject_watch_list.iter_mut() {
-                let current = read_injection_status(channel, overrd);
+                let current = dispatch!(io, _aux_mutex, _routing_table, channel, read_injection_status, overrd);
                 if previous.is_none() || previous.unwrap() != current {
                     let message = DeviceMessage::InjectionStatus {
                         channel: channel,
@@ -228,15 +199,18 @@ fn connection_worker(io: &Io, mut stream: &mut TcpStream) -> Result<(), Error<Sc
     }
 }
 
-pub fn thread(io: Io) {
+pub fn thread(io: Io, aux_mutex: &Mutex, routing_table: &Urc<RefCell<drtio_routing::RoutingTable>>) {
     let listener = TcpListener::new(&io, 2047);
     listener.listen(1383).expect("moninj: cannot listen");
 
     loop {
+        let aux_mutex = aux_mutex.clone();
+        let routing_table = routing_table.clone();
         let stream = listener.accept().expect("moninj: cannot accept").into_handle();
         io.spawn(16384, move |io| {
+            let routing_table = routing_table.borrow();
             let mut stream = TcpStream::from_handle(&io, stream);
-            match connection_worker(&io, &mut stream) {
+            match connection_worker(&io, &aux_mutex, &routing_table, &mut stream) {
                 Ok(()) => {},
                 Err(err) => error!("moninj aborted: {}", err)
             }

artiq/firmware/runtime/rtio_mgt.rs

@@ -1,7 +1,13 @@
+use core::cell::RefCell;
+use urc::Urc;
 use board_misoc::csr;
+#[cfg(has_drtio)]
+use board_misoc::clock;
 #[cfg(has_rtio_clock_switch)]
 use board_misoc::config;
+use board_artiq::drtio_routing;
 use sched::Io;
+use sched::Mutex;
 
 #[cfg(has_rtio_crg)]
 pub mod crg {
@@ -42,11 +48,19 @@ pub mod drtio {
     use super::*;
     use drtioaux;
 
-    pub fn startup(io: &Io) {
+    pub fn startup(io: &Io, aux_mutex: &Mutex,
+            routing_table: &Urc<RefCell<drtio_routing::RoutingTable>>,
+            up_destinations: &Urc<RefCell<[bool; drtio_routing::DEST_COUNT]>>) {
         unsafe {
             csr::drtio_transceiver::stable_clkin_write(1);
         }
-        io.spawn(4096, link_thread);
+        let aux_mutex = aux_mutex.clone();
+        let routing_table = routing_table.clone();
+        let up_destinations = up_destinations.clone();
+        io.spawn(4096, move |io| {
+            let routing_table = routing_table.borrow();
+            link_thread(io, &aux_mutex, &routing_table, &up_destinations);
+        });
     }
 
     fn link_rx_up(linkno: u8) -> bool {
@@ -56,83 +70,111 @@ pub mod drtio {
         }
     }
 
-    pub fn link_up(linkno: u8) -> bool {
-        let linkno = linkno as usize;
-        /* This function may be called by kernels with arbitrary
-         * linkno values.
-         */
-        if linkno >= csr::DRTIO.len() {
-            return false;
-        }
-        unsafe {
-            (csr::DRTIO[linkno].link_up_read)() == 1
+    fn recv_aux_timeout(io: &Io, linkno: u8, timeout: u32) -> Result<drtioaux::Packet, &'static str> {
+        let max_time = clock::get_ms() + timeout as u64;
+        loop {
+            if !link_rx_up(linkno) {
+                return Err("link went down");
+            }
+            if clock::get_ms() > max_time {
+                return Err("timeout");
+            }
+            match drtioaux::recv(linkno) {
+                Ok(Some(packet)) => return Ok(packet),
+                Ok(None) => (),
+                Err(_) => return Err("aux packet error")
+            }
+            io.relinquish().unwrap();
         }
     }
 
-    fn set_link_up(linkno: u8, up: bool) {
-        let linkno = linkno as usize;
-        unsafe {
-            (csr::DRTIO[linkno].link_up_write)(if up { 1 }  else { 0 });
-        }
+    pub fn aux_transact(io: &Io, aux_mutex: &Mutex,
+            linkno: u8, request: &drtioaux::Packet) -> Result<drtioaux::Packet, &'static str> {
+        let _lock = aux_mutex.lock(io).unwrap();
+        drtioaux::send(linkno, request).unwrap();
+        recv_aux_timeout(io, linkno, 200)
     }
 
-    fn sync_tsc(linkno: u8) {
-        let linkno = linkno as usize;
-        unsafe {
-            (csr::DRTIO[linkno].set_time_write)(1);
-            while (csr::DRTIO[linkno].set_time_read)() == 1 {}
-        }
-    }
-
-    fn init_buffer_space(linkno: u8) {
-        let linkidx = linkno as usize;
-        unsafe {
-            (csr::DRTIO[linkidx].o_get_buffer_space_write)(1);
-            while (csr::DRTIO[linkidx].o_wait_read)() == 1 {}
-            info!("[LINK#{}] buffer space is {}",
-                linkno, (csr::DRTIO[linkidx].o_dbg_buffer_space_read)());
-        }
-    }
-
-    fn ping_remote(linkno: u8, io: &Io) -> u32 {
+    fn ping_remote(io: &Io, aux_mutex: &Mutex, linkno: u8) -> u32 {
         let mut count = 0;
         loop {
             if !link_rx_up(linkno) {
                 return 0
             }
             count += 1;
-            if count > 200 {
+            if count > 100 {
                 return 0;
             }
-            drtioaux::send_link(linkno, &drtioaux::Packet::EchoRequest).unwrap();
-            io.sleep(100).unwrap();
-            let pr = drtioaux::recv_link(linkno);
-            match pr {
-                Ok(Some(drtioaux::Packet::EchoReply)) => return count,
+            let reply = aux_transact(io, aux_mutex, linkno, &drtioaux::Packet::EchoRequest);
+            match reply {
+                Ok(drtioaux::Packet::EchoReply) => return count,
                 _ => {}
             }
+            io.relinquish().unwrap();
         }
     }
 
-    fn wait_tsc_ack(linkno: u8, io: &Io) -> bool {
-        loop {
-            let mut count = 0;
-            if !link_rx_up(linkno) {
-                return false;
-            }
-            count += 1;
-            if count > 200 {
-                return false;
-            }
-            io.sleep(100).unwrap();
-            // TSCAck is the only aux packet that is sent spontaneously
-            // by the satellite, in response to a TSC set on the RT link.
-            let pr = drtioaux::recv_link(linkno);
-            match pr {
-                Ok(Some(drtioaux::Packet::TSCAck)) => return true,
-                _ => {}
+    fn sync_tsc(io: &Io, aux_mutex: &Mutex, linkno: u8) -> Result<(), &'static str> {
+        let _lock = aux_mutex.lock(io).unwrap();
+
+        unsafe {
+            (csr::DRTIO[linkno as usize].set_time_write)(1);
+            while (csr::DRTIO[linkno as usize].set_time_read)() == 1 {}
+        }
+        // TSCAck is the only aux packet that is sent spontaneously
+        // by the satellite, in response to a TSC set on the RT link.
+        let reply = recv_aux_timeout(io, linkno, 10000)?;
+        if reply == drtioaux::Packet::TSCAck {
+            return Ok(());
+        } else {
+            return Err("unexpected reply");
+        }
+    }
+
+    fn load_routing_table(io: &Io, aux_mutex: &Mutex, linkno: u8, routing_table: &drtio_routing::RoutingTable)
+            -> Result<(), &'static str> {
+        for i in 0..drtio_routing::DEST_COUNT {
+            let reply = aux_transact(io, aux_mutex, linkno, &drtioaux::Packet::RoutingSetPath {
+                destination: i as u8,
+                hops: routing_table.0[i]
+            })?;
+            if reply != drtioaux::Packet::RoutingAck {
+                return Err("unexpected reply");
             }
         }
+        Ok(())
+    }
+
+    fn set_rank(io: &Io, aux_mutex: &Mutex, linkno: u8, rank: u8) -> Result<(), &'static str> {
+        let reply = aux_transact(io, aux_mutex, linkno, &drtioaux::Packet::RoutingSetRank {
+            rank: rank
+        })?;
+        if reply != drtioaux::Packet::RoutingAck {
+            return Err("unexpected reply");
+        }
+        Ok(())
+    }
+
+    fn init_buffer_space(destination: u8, linkno: u8) {
+        let linkno = linkno as usize;
+        unsafe {
+            (csr::DRTIO[linkno].destination_write)(destination);
+            (csr::DRTIO[linkno].force_destination_write)(1);
+            (csr::DRTIO[linkno].o_get_buffer_space_write)(1);
+            while (csr::DRTIO[linkno].o_wait_read)() == 1 {}
+            info!("[DEST#{}] buffer space is {}",
+                destination, (csr::DRTIO[linkno].o_dbg_buffer_space_read)());
+            (csr::DRTIO[linkno].force_destination_write)(0);
+        }
+    }
+
+    fn process_unsolicited_aux(io: &Io, aux_mutex: &Mutex, linkno: u8) {
+        let _lock = aux_mutex.lock(io).unwrap();
+        match drtioaux::recv(linkno) {
+            Ok(Some(packet)) => warn!("[LINK#{}] unsolicited aux packet: {:?}", linkno, packet),
+            Ok(None) => (),
+            Err(_) => warn!("[LINK#{}] aux packet error", linkno)
+        }
     }
 
     fn process_local_errors(linkno: u8) {
@@ -156,66 +198,144 @@ pub mod drtio {
         }
     }
 
-    fn process_aux_errors(linkno: u8) {
-        drtioaux::send_link(linkno, &drtioaux::Packet::RtioErrorRequest).unwrap();
-        match drtioaux::recv_timeout_link(linkno, None) {
-            Ok(drtioaux::Packet::RtioNoErrorReply) => (),
-            Ok(drtioaux::Packet::RtioErrorSequenceErrorReply { channel }) =>
-                error!("[LINK#{}] RTIO sequence error involving channel {}", linkno, channel),
-            Ok(drtioaux::Packet::RtioErrorCollisionReply { channel }) =>
-                error!("[LINK#{}] RTIO collision involving channel {}", linkno, channel),
-            Ok(drtioaux::Packet::RtioErrorBusyReply { channel }) =>
-                error!("[LINK#{}] RTIO busy error involving channel {}", linkno, channel),
-            Ok(_) => error!("[LINK#{}] received unexpected aux packet", linkno),
-            Err(e) => error!("[LINK#{}] aux packet error ({})", linkno, e)
+    fn destination_set_up(routing_table: &drtio_routing::RoutingTable,
+            up_destinations: &Urc<RefCell<[bool; drtio_routing::DEST_COUNT]>>,
+            destination: u8, up: bool) {
+        let mut up_destinations = up_destinations.borrow_mut();
+        up_destinations[destination as usize] = up;
+        if up {
+            drtio_routing::interconnect_enable(routing_table, 0, destination);
+            info!("[DEST#{}] destination is up", destination);
+        } else {
+            drtio_routing::interconnect_disable(destination);
+            info!("[DEST#{}] destination is down", destination);
         }
     }
 
-    pub fn link_thread(io: Io) {
+    fn destination_up(up_destinations: &Urc<RefCell<[bool; drtio_routing::DEST_COUNT]>>, destination: u8) -> bool {
+        let up_destinations = up_destinations.borrow();
+        up_destinations[destination as usize]
+    }
+
+    fn destination_survey(io: &Io, aux_mutex: &Mutex, routing_table: &drtio_routing::RoutingTable,
+            up_links: &[bool],
+            up_destinations: &Urc<RefCell<[bool; drtio_routing::DEST_COUNT]>>) {
+        for destination in 0..drtio_routing::DEST_COUNT {
+            let hop = routing_table.0[destination][0];
+            let destination = destination as u8;
+
+            if hop == 0 {
+                /* local RTIO */
+                if !destination_up(up_destinations, destination) {
+                    destination_set_up(routing_table, up_destinations, destination, true);
+                }
+            } else if hop as usize <= csr::DRTIO.len() {
+                let linkno = hop - 1;
+                if destination_up(up_destinations, destination) {
+                    if up_links[linkno as usize] {
+                        let reply = aux_transact(io, aux_mutex, linkno, &drtioaux::Packet::DestinationStatusRequest {
+                            destination: destination
+                        });
+                        match reply {
+                            Ok(drtioaux::Packet::DestinationDownReply) =>
+                                destination_set_up(routing_table, up_destinations, destination, false),
+                            Ok(drtioaux::Packet::DestinationOkReply) => (),
+                            Ok(drtioaux::Packet::DestinationSequenceErrorReply { channel }) =>
+                                error!("[DEST#{}] RTIO sequence error involving channel 0x{:04x}", destination, channel),
+                            Ok(drtioaux::Packet::DestinationCollisionReply { channel }) =>
+                                error!("[DEST#{}] RTIO collision involving channel 0x{:04x}", destination, channel),
+                            Ok(drtioaux::Packet::DestinationBusyReply { channel }) =>
+                                error!("[DEST#{}] RTIO busy error involving channel 0x{:04x}", destination, channel),
+                            Ok(packet) => error!("[DEST#{}] received unexpected aux packet: {:?}", destination, packet),
+                            Err(e) => error!("[DEST#{}] communication failed ({})", destination, e)
+                        }
+                    } else {
+                        destination_set_up(routing_table, up_destinations, destination, false);
+                    }
+                } else {
+                    if up_links[linkno as usize] {
+                        let reply = aux_transact(io, aux_mutex, linkno, &drtioaux::Packet::DestinationStatusRequest {
+                            destination: destination
+                        });
+                        match reply {
+                            Ok(drtioaux::Packet::DestinationDownReply) => (),
+                            Ok(drtioaux::Packet::DestinationOkReply) => {
+                                destination_set_up(routing_table, up_destinations, destination, true);
+                                init_buffer_space(destination as u8, linkno);
+                            },
+                            Ok(packet) => error!("[DEST#{}] received unexpected aux packet: {:?}", destination, packet),
+                            Err(e) => error!("[DEST#{}] communication failed ({})", destination, e)
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    pub fn link_thread(io: Io, aux_mutex: &Mutex,
+            routing_table: &drtio_routing::RoutingTable,
+            up_destinations: &Urc<RefCell<[bool; drtio_routing::DEST_COUNT]>>) {
+        let mut up_links = [false; csr::DRTIO.len()];
         loop {
             for linkno in 0..csr::DRTIO.len() {
                 let linkno = linkno as u8;
-                if link_up(linkno) {
+                if up_links[linkno as usize] {
                     /* link was previously up */
                     if link_rx_up(linkno) {
+                        process_unsolicited_aux(&io, aux_mutex, linkno);
                         process_local_errors(linkno);
-                        process_aux_errors(linkno);
                     } else {
                         info!("[LINK#{}] link is down", linkno);
-                        set_link_up(linkno, false);
+                        up_links[linkno as usize] = false;
                     }
                 } else {
                     /* link was previously down */
                     if link_rx_up(linkno) {
                         info!("[LINK#{}] link RX became up, pinging", linkno);
-                        let ping_count = ping_remote(linkno, &io);
+                        let ping_count = ping_remote(&io, aux_mutex, linkno);
                         if ping_count > 0 {
                             info!("[LINK#{}] remote replied after {} packets", linkno, ping_count);
-                            set_link_up(linkno, true);
-                            init_buffer_space(linkno);
-                            sync_tsc(linkno);
-                            if !wait_tsc_ack(linkno, &io) {
-                                info!("[LINK#{}] remote failed to ack TSC", linkno);
-                            } else {
-                                info!("[LINK#{}] link initialization completed", linkno);
+                            up_links[linkno as usize] = true;
+                            if let Err(e) = sync_tsc(&io, aux_mutex, linkno) {
+                                error!("[LINK#{}] failed to sync TSC ({})", linkno, e);
                             }
+                            if let Err(e) = load_routing_table(&io, aux_mutex, linkno, routing_table) {
+                                error!("[LINK#{}] failed to load routing table ({})", linkno, e);
+                            }
+                            if let Err(e) = set_rank(&io, aux_mutex, linkno, 1) {
+                                error!("[LINK#{}] failed to set rank ({})", linkno, e);
+                            }
+                            info!("[LINK#{}] link initialization completed", linkno);
                         } else {
-                            info!("[LINK#{}] ping failed", linkno);
+                            error!("[LINK#{}] ping failed", linkno);
                         }
                     }
                 }
             }
+            destination_survey(&io, aux_mutex, routing_table, &up_links, up_destinations);
             io.sleep(200).unwrap();
         }
     }
 
-    pub fn init() {
+    pub fn reset(io: &Io, aux_mutex: &Mutex) {
+        for linkno in 0..csr::DRTIO.len() {
+            unsafe {
+                (csr::DRTIO[linkno].reset_write)(1);
+            }
+        }
+        io.sleep(1).unwrap();
+        for linkno in 0..csr::DRTIO.len() {
+            unsafe {
+                (csr::DRTIO[linkno].reset_write)(0);
+            }
+        }
+
         for linkno in 0..csr::DRTIO.len() {
             let linkno = linkno as u8;
-            if link_up(linkno) {
-                drtioaux::send_link(linkno,
-                    &drtioaux::Packet::ResetRequest { phy: false }).unwrap();
-                match drtioaux::recv_timeout_link(linkno, None) {
+            if link_rx_up(linkno) {
+                let reply = aux_transact(io, aux_mutex, linkno,
+                    &drtioaux::Packet::ResetRequest);
+                match reply {
                     Ok(drtioaux::Packet::ResetAck) => (),
                     Ok(_) => error!("[LINK#{}] reset failed, received unexpected aux packet", linkno),
                     Err(e) => error!("[LINK#{}] reset failed, aux packet error ({})", linkno, e)
@@ -229,9 +349,10 @@ pub mod drtio {
 pub mod drtio {
     use super::*;
 
-    pub fn startup(_io: &Io) {}
-    pub fn init() {}
-    pub fn link_up(_linkno: u8) -> bool { false }
+    pub fn startup(_io: &Io, _aux_mutex: &Mutex,
+        _routing_table: &Urc<RefCell<drtio_routing::RoutingTable>>,
+        _up_destinations: &Urc<RefCell<[bool; drtio_routing::DEST_COUNT]>>) {}
+    pub fn reset(_io: &Io, _aux_mutex: &Mutex) {}
 }
 
 fn async_error_thread(io: Io) {
@@ -256,7 +377,13 @@ fn async_error_thread(io: Io) {
     }
 }
 
-pub fn startup(io: &Io) {
+pub fn startup(io: &Io, aux_mutex: &Mutex,
+        routing_table: &Urc<RefCell<drtio_routing::RoutingTable>>,
+        up_destinations: &Urc<RefCell<[bool; drtio_routing::DEST_COUNT]>>) {
+    // The RTIO CRG may depend on the DRTIO transceiver clock.
+    // Initialize DRTIO first to bring up transceiver clocking.
+    drtio::startup(io, aux_mutex, routing_table, up_destinations);
+
     #[cfg(has_rtio_crg)]
     {
         #[cfg(has_rtio_clock_switch)]
@@ -295,46 +422,16 @@ pub fn startup(io: &Io) {
             }
         }
     }
+    unsafe {
+        csr::rtio_core::reset_phy_write(1);
+    }
 
-    drtio::startup(io);
-    init_core(true);
     io.spawn(4096, async_error_thread);
 }
 
-pub fn init_core(phy: bool) {
+pub fn reset(io: &Io, aux_mutex: &Mutex) {
     unsafe {
         csr::rtio_core::reset_write(1);
-        if phy {
-            csr::rtio_core::reset_phy_write(1);
-        }
     }
-    drtio::init()
-}
-
-#[cfg(has_drtio)]
-pub mod drtio_dbg {
-    use board_misoc::csr;
-
-    pub fn get_packet_counts(linkno: u8) -> (u32, u32) {
-        let linkno = linkno as usize;
-        unsafe {
-            (csr::DRTIO[linkno].update_packet_cnt_write)(1);
-            ((csr::DRTIO[linkno].packet_cnt_tx_read)(),
-             (csr::DRTIO[linkno].packet_cnt_rx_read)())
-        }
-    }
-
-    pub fn get_buffer_space_req_count(linkno: u8) -> u32 {
-        let linkno = linkno as usize;
-        unsafe {
-            (csr::DRTIO[linkno].o_dbg_buffer_space_req_cnt_read)()
-        }
-    }
-}
-
-#[cfg(not(has_drtio))]
-pub mod drtio_dbg {
-    pub fn get_packet_counts(_linkno: u8) -> (u32, u32) { (0, 0) }
-
-    pub fn get_buffer_space_req_count(_linkno: u8) -> u32 { 0 }
+    drtio::reset(io, aux_mutex)
 }

artiq/firmware/runtime/sched.rs

@@ -265,6 +265,29 @@ impl<'a> Io<'a> {
     }
 }
 
+#[derive(Clone)]
+pub struct Mutex(Urc<Cell<bool>>);
+
+impl Mutex {
+    pub fn new() -> Mutex {
+        Mutex(Urc::new(Cell::new(false)))
+    }
+
+    pub fn lock<'a>(&'a self, io: &Io) -> Result<MutexGuard<'a>, Error> {
+        io.until(|| !self.0.get())?;
+        self.0.set(true);
+        Ok(MutexGuard(&*self.0))
+    }
+}
+
+pub struct MutexGuard<'a>(&'a Cell<bool>);
+
+impl<'a> Drop for MutexGuard<'a> {
+    fn drop(&mut self) {
+        self.0.set(false)
+    }
+}
+
 macro_rules! until {
     ($socket:expr, $ty:ty, |$var:ident| $cond:expr) => ({
         let (sockets, handle) = ($socket.io.sockets.clone(), $socket.handle);

artiq/firmware/runtime/session.rs

@@ -6,13 +6,13 @@ use io::{Read, Write, Error as IoError};
 use board_misoc::{ident, cache, config};
 use {mailbox, rpc_queue, kernel};
 use urc::Urc;
-use sched::{ThreadHandle, Io, TcpListener, TcpStream, Error as SchedError};
-#[cfg(has_rtio_core)]
+use sched::{ThreadHandle, Io, Mutex, TcpListener, TcpStream, Error as SchedError};
 use rtio_mgt;
 use rtio_dma::Manager as DmaManager;
 use cache::Cache;
 use kern_hwreq;
 use watchdog::WatchdogSet;
+use board_artiq::drtio_routing;
 
 use rpc_proto as rpc;
 use session_proto as host;
@@ -324,7 +324,10 @@ fn process_host_message(io: &Io,
     Ok(())
 }
 
-fn process_kern_message(io: &Io, mut stream: Option<&mut TcpStream>,
+fn process_kern_message(io: &Io, aux_mutex: &Mutex,
+                        routing_table: &drtio_routing::RoutingTable,
+                        up_destinations: &Urc<RefCell<[bool; drtio_routing::DEST_COUNT]>>,
+                        mut stream: Option<&mut TcpStream>,
                         session: &mut Session) -> Result<bool, Error<SchedError>> {
     kern_recv_notrace(io, |request| {
         match (request, session.kernel_state) {
@@ -342,7 +345,7 @@ fn process_kern_message(io: &Io, mut stream: Option<&mut TcpStream>,
 
         kern_recv_dotrace(request);
 
-        if kern_hwreq::process_kern_hwreq(io, request)? {
+        if kern_hwreq::process_kern_hwreq(io, aux_mutex, routing_table, up_destinations, request)? {
             return Ok(false)
         }
 
@@ -491,7 +494,9 @@ fn process_kern_queued_rpc(stream: &mut TcpStream,
     })
 }
 
-fn host_kernel_worker(io: &Io,
+fn host_kernel_worker(io: &Io, aux_mutex: &Mutex,
+                      routing_table: &drtio_routing::RoutingTable,
+                      up_destinations: &Urc<RefCell<[bool; drtio_routing::DEST_COUNT]>>,
                       stream: &mut TcpStream,
                       congress: &mut Congress) -> Result<(), Error<SchedError>> {
     let mut session = Session::new(congress);
@@ -508,7 +513,9 @@ fn host_kernel_worker(io: &Io,
         }
 
         if mailbox::receive() != 0 {
-            process_kern_message(io, Some(stream), &mut session)?;
+            process_kern_message(io, aux_mutex,
+                routing_table, up_destinations,
+                Some(stream), &mut session)?;
         }
 
         if session.kernel_state == KernelState::Running {
@@ -517,12 +524,9 @@ fn host_kernel_worker(io: &Io,
                 return Err(Error::WatchdogExpired(idx))
             }
 
-            #[cfg(has_rtio_core)]
-            {
-                if !rtio_mgt::crg::check() {
-                    host_write(stream, host::Reply::ClockFailure)?;
-                    return Err(Error::ClockFailure)
-                }
+            if !rtio_mgt::crg::check() {
+                host_write(stream, host::Reply::ClockFailure)?;
+                return Err(Error::ClockFailure)
             }
         }
 
@@ -530,7 +534,9 @@ fn host_kernel_worker(io: &Io,
     }
 }
 
-fn flash_kernel_worker(io: &Io,
+fn flash_kernel_worker(io: &Io, aux_mutex: &Mutex,
+                       routing_table: &drtio_routing::RoutingTable,
+                       up_destinations: &Urc<RefCell<[bool; drtio_routing::DEST_COUNT]>>,
                        congress: &mut Congress,
                        config_key: &str) -> Result<(), Error<SchedError>> {
     let mut session = Session::new(congress);
@@ -553,7 +559,7 @@ fn flash_kernel_worker(io: &Io,
         }
 
         if mailbox::receive() != 0 {
-            if process_kern_message(io, None, &mut session)? {
+            if process_kern_message(io, aux_mutex, routing_table, up_destinations, None, &mut session)? {
                 return Ok(())
             }
         }
@@ -562,11 +568,8 @@ fn flash_kernel_worker(io: &Io,
             return Err(Error::WatchdogExpired(idx))
         }
 
-        #[cfg(has_rtio_core)]
-        {
-            if !rtio_mgt::crg::check() {
-                return Err(Error::ClockFailure)
-            }
+        if !rtio_mgt::crg::check() {
+            return Err(Error::ClockFailure)
         }
 
         io.relinquish()?
@@ -588,7 +591,9 @@ fn respawn<F>(io: &Io, handle: &mut Option<ThreadHandle>, f: F)
     *handle = Some(io.spawn(16384, f))
 }
 
-pub fn thread(io: Io) {
+pub fn thread(io: Io, aux_mutex: &Mutex, 
+        routing_table: &Urc<RefCell<drtio_routing::RoutingTable>>,
+        up_destinations: &Urc<RefCell<[bool; drtio_routing::DEST_COUNT]>>) {
     let listener = TcpListener::new(&io, 65535);
     listener.listen(1381).expect("session: cannot listen");
     info!("accepting network sessions");
@@ -597,11 +602,15 @@ pub fn thread(io: Io) {
 
     let mut kernel_thread = None;
     {
+        let aux_mutex = aux_mutex.clone();
+        let routing_table = routing_table.clone();
+        let up_destinations = up_destinations.clone();
         let congress = congress.clone();
         respawn(&io, &mut kernel_thread, move |io| {
+            let routing_table = routing_table.borrow();
             let mut congress = congress.borrow_mut();
             info!("running startup kernel");
-            match flash_kernel_worker(&io, &mut congress, "startup_kernel") {
+            match flash_kernel_worker(&io, &aux_mutex, &routing_table, &up_destinations, &mut congress, "startup_kernel") {
                 Ok(()) =>
                     info!("startup kernel finished"),
                 Err(Error::KernelNotFound) =>
@@ -630,12 +639,16 @@ pub fn thread(io: Io) {
             }
             info!("new connection from {}", stream.remote_endpoint());
 
+            let aux_mutex = aux_mutex.clone();
+            let routing_table = routing_table.clone();
+            let up_destinations = up_destinations.clone();
             let congress = congress.clone();
             let stream = stream.into_handle();
             respawn(&io, &mut kernel_thread, move |io| {
+                let routing_table = routing_table.borrow();
                 let mut congress = congress.borrow_mut();
                 let mut stream = TcpStream::from_handle(&io, stream);
-                match host_kernel_worker(&io, &mut stream, &mut *congress) {
+                match host_kernel_worker(&io, &aux_mutex, &routing_table, &up_destinations, &mut stream, &mut *congress) {
                     Ok(()) => (),
                     Err(Error::Protocol(host::Error::Io(IoError::UnexpectedEnd))) =>
                         info!("connection closed"),
@@ -653,10 +666,14 @@ pub fn thread(io: Io) {
         if kernel_thread.as_ref().map_or(true, |h| h.terminated()) {
             info!("no connection, starting idle kernel");
 
+            let aux_mutex = aux_mutex.clone();
+            let routing_table = routing_table.clone();
+            let up_destinations = up_destinations.clone();
             let congress = congress.clone();
             respawn(&io, &mut kernel_thread, move |io| {
+                let routing_table = routing_table.borrow();
                 let mut congress = congress.borrow_mut();
-                match flash_kernel_worker(&io, &mut *congress, "idle_kernel") {
+                match flash_kernel_worker(&io, &aux_mutex, &routing_table, &up_destinations, &mut *congress, "idle_kernel") {
                     Ok(()) =>
                         info!("idle kernel finished, standing by"),
                     Err(Error::Protocol(host::Error::Io(

artiq/firmware/satman/main.rs

@@ -1,4 +1,4 @@
-#![feature(never_type, panic_implementation, panic_info_message)]
+#![feature(never_type, panic_implementation, panic_info_message, const_slice_len, try_from)]
 #![no_std]
 
 #[macro_use]
@@ -7,88 +7,192 @@ extern crate log;
 extern crate board_misoc;
 extern crate board_artiq;
 
-use board_misoc::{csr, ident, clock, uart_logger};
+use core::convert::TryFrom;
+use board_misoc::{csr, irq, ident, clock, uart_logger};
 use board_artiq::{i2c, spi, si5324, drtioaux};
 #[cfg(has_serwb_phy_amc)]
 use board_artiq::serwb;
+use board_artiq::drtio_routing;
 #[cfg(has_hmc830_7043)]
 use board_artiq::hmc830_7043;
 
-fn drtio_reset(reset: bool) {
+mod repeater;
+
+fn drtiosat_reset(reset: bool) {
     unsafe {
-        (csr::DRTIO[0].reset_write)(if reset { 1 } else { 0 });
+        csr::drtiosat::reset_write(if reset { 1 } else { 0 });
     }
 }
 
-fn drtio_reset_phy(reset: bool) {
+fn drtiosat_reset_phy(reset: bool) {
     unsafe {
-        (csr::DRTIO[0].reset_phy_write)(if reset { 1 } else { 0 });
+        csr::drtiosat::reset_phy_write(if reset { 1 } else { 0 });
     }
 }
 
-fn drtio_tsc_loaded() -> bool {
+fn drtiosat_link_rx_up() -> bool {
     unsafe {
-        let tsc_loaded = (csr::DRTIO[0].tsc_loaded_read)() == 1;
+        csr::drtiosat::rx_up_read() == 1
+    }
+}
+
+fn drtiosat_tsc_loaded() -> bool {
+    unsafe {
+        let tsc_loaded = csr::drtiosat::tsc_loaded_read() == 1;
         if tsc_loaded {
-            (csr::DRTIO[0].tsc_loaded_write)(1);
+            csr::drtiosat::tsc_loaded_write(1);
         }
         tsc_loaded
     }
 }
 
-fn process_aux_packet(packet: drtioaux::Packet) -> Result<(), drtioaux::Error<!>> {
+
+#[cfg(has_drtio_routing)]
+macro_rules! forward {
+    ($routing_table:expr, $destination:expr, $rank:expr, $repeaters:expr, $packet:expr) => {{
+        let hop = $routing_table.0[$destination as usize][$rank as usize];
+        if hop != 0 {
+            let repno = (hop - 1) as usize;
+            if repno < $repeaters.len() {
+                return $repeaters[repno].aux_forward($packet);
+            } else {
+                return Err(drtioaux::Error::RoutingError);
+            }
+        }
+    }}
+}
+
+#[cfg(not(has_drtio_routing))]
+macro_rules! forward {
+    ($routing_table:expr, $destination:expr, $rank:expr, $repeaters:expr, $packet:expr) => {}
+}
+
+fn process_aux_packet(_repeaters: &mut [repeater::Repeater],
+        _routing_table: &mut drtio_routing::RoutingTable, _rank: &mut u8,
+        packet: drtioaux::Packet) -> Result<(), drtioaux::Error<!>> {
     // In the code below, *_chan_sel_write takes an u8 if there are fewer than 256 channels,
     // and u16 otherwise; hence the `as _` conversion.
     match packet {
         drtioaux::Packet::EchoRequest =>
-            drtioaux::send_link(0, &drtioaux::Packet::EchoReply),
-        drtioaux::Packet::ResetRequest { phy } => {
-            if phy {
-                drtio_reset_phy(true);
-                drtio_reset_phy(false);
-            } else {
-                drtio_reset(true);
-                drtio_reset(false);
+            drtioaux::send(0, &drtioaux::Packet::EchoReply),
+        drtioaux::Packet::ResetRequest => {
+            info!("resetting RTIO");
+            drtiosat_reset(true);
+            clock::spin_us(100);
+            drtiosat_reset(false);
+            for rep in _repeaters.iter() {
+                if let Err(e) = rep.rtio_reset() {
+                    error!("failed to issue RTIO reset ({})", e);
+                }
             }
-            drtioaux::send_link(0, &drtioaux::Packet::ResetAck)
+            drtioaux::send(0, &drtioaux::Packet::ResetAck)
         },
 
-        drtioaux::Packet::RtioErrorRequest => {
-            let errors;
-            unsafe {
-                errors = (csr::DRTIO[0].rtio_error_read)();
-            }
-            if errors & 1 != 0 {
-                let channel;
+        drtioaux::Packet::DestinationStatusRequest { destination: _destination } => {
+            #[cfg(has_drtio_routing)]
+            let hop = _routing_table.0[_destination as usize][*_rank as usize];
+            #[cfg(not(has_drtio_routing))]
+            let hop = 0;
+
+            if hop == 0 {
+                let errors;
                 unsafe {
-                    channel = (csr::DRTIO[0].sequence_error_channel_read)();
-                    (csr::DRTIO[0].rtio_error_write)(1);
+                    errors = csr::drtiosat::rtio_error_read();
                 }
-                drtioaux::send_link(0,
-                    &drtioaux::Packet::RtioErrorSequenceErrorReply { channel })
-            } else if errors & 2 != 0 {
-                let channel;
-                unsafe {
-                    channel = (csr::DRTIO[0].collision_channel_read)();
-                    (csr::DRTIO[0].rtio_error_write)(2);
+                if errors & 1 != 0 {
+                    let channel;
+                    unsafe {
+                        channel = csr::drtiosat::sequence_error_channel_read();
+                        csr::drtiosat::rtio_error_write(1);
+                    }
+                    drtioaux::send(0,
+                        &drtioaux::Packet::DestinationSequenceErrorReply { channel })?;
+                } else if errors & 2 != 0 {
+                    let channel;
+                    unsafe {
+                        channel = csr::drtiosat::collision_channel_read();
+                        csr::drtiosat::rtio_error_write(2);
+                    }
+                    drtioaux::send(0,
+                        &drtioaux::Packet::DestinationCollisionReply { channel })?;
+                } else if errors & 4 != 0 {
+                    let channel;
+                    unsafe {
+                        channel = csr::drtiosat::busy_channel_read();
+                        csr::drtiosat::rtio_error_write(4);
+                    }
+                    drtioaux::send(0,
+                        &drtioaux::Packet::DestinationBusyReply { channel })?;
                 }
-                drtioaux::send_link(0,
-                    &drtioaux::Packet::RtioErrorCollisionReply { channel })
-            } else if errors & 4 != 0 {
-                let channel;
-                unsafe {
-                    channel = (csr::DRTIO[0].busy_channel_read)();
-                    (csr::DRTIO[0].rtio_error_write)(4);
+                else {
+                    drtioaux::send(0, &drtioaux::Packet::DestinationOkReply)?;
                 }
-                drtioaux::send_link(0,
-                    &drtioaux::Packet::RtioErrorBusyReply { channel })
             }
-            else {
-                drtioaux::send_link(0, &drtioaux::Packet::RtioNoErrorReply)
+
+            #[cfg(has_drtio_routing)]
+            {
+                if hop != 0 {
+                    let hop = hop as usize;
+                    if hop <= csr::DRTIOREP.len() {
+                        let repno = hop - 1;
+                        match _repeaters[repno].aux_forward(&drtioaux::Packet::DestinationStatusRequest {
+                            destination: _destination
+                        }) {
+                            Ok(()) => (),
+                            Err(drtioaux::Error::LinkDown) => drtioaux::send(0, &drtioaux::Packet::DestinationDownReply)?,
+                            Err(e) => {
+                                drtioaux::send(0, &drtioaux::Packet::DestinationDownReply)?;
+                                error!("aux error when handling destination status request: {}", e);
+                            },
+                        }
+                    } else {
+                        drtioaux::send(0, &drtioaux::Packet::DestinationDownReply)?;
+                    }
+                }
             }
+
+            Ok(())
         }
 
-        drtioaux::Packet::MonitorRequest { channel, probe } => {
+        #[cfg(has_drtio_routing)]
+        drtioaux::Packet::RoutingSetPath { destination, hops } => {
+            _routing_table.0[destination as usize] = hops;
+            for rep in _repeaters.iter() {
+                if let Err(e) = rep.set_path(destination, &hops) {
+                    error!("failed to set path ({})", e);
+                }
+            }
+            drtioaux::send(0, &drtioaux::Packet::RoutingAck)
+        }
+        #[cfg(has_drtio_routing)]
+        drtioaux::Packet::RoutingSetRank { rank } => {
+            *_rank = rank;
+            drtio_routing::interconnect_enable_all(_routing_table, rank);
+
+            let rep_rank = rank + 1;
+            for rep in _repeaters.iter() {
+                if let Err(e) = rep.set_rank(rep_rank) {
+                    error!("failed to set rank ({})", e);
+                }
+            }
+
+            info!("rank: {}", rank);
+            info!("routing table: {}", _routing_table);
+
+            drtioaux::send(0, &drtioaux::Packet::RoutingAck)
+        }
+
+        #[cfg(not(has_drtio_routing))]
+        drtioaux::Packet::RoutingSetPath { destination: _, hops: _ } => {
+            drtioaux::send(0, &drtioaux::Packet::RoutingAck)
+        }
+        #[cfg(not(has_drtio_routing))]
+        drtioaux::Packet::RoutingSetRank { rank: _ } => {
+            drtioaux::send(0, &drtioaux::Packet::RoutingAck)
+        }
+
+        drtioaux::Packet::MonitorRequest { destination: _destination, channel, probe } => {
+            forward!(_routing_table, _destination, *_rank, _repeaters, &packet);
             let value;
             #[cfg(has_rtio_moninj)]
             unsafe {
@@ -102,9 +206,10 @@ fn process_aux_packet(packet: drtioaux::Packet) -> Result<(), drtioaux::Error<!>
                 value = 0;
             }
             let reply = drtioaux::Packet::MonitorReply { value: value as u32 };
-            drtioaux::send_link(0, &reply)
+            drtioaux::send(0, &reply)
         },
-        drtioaux::Packet::InjectionRequest { channel, overrd, value } => {
+        drtioaux::Packet::InjectionRequest { destination: _destination, channel, overrd, value } => {
+            forward!(_routing_table, _destination, *_rank, _repeaters, &packet);
             #[cfg(has_rtio_moninj)]
             unsafe {
                 csr::rtio_moninj::inj_chan_sel_write(channel as _);
@@ -113,7 +218,8 @@ fn process_aux_packet(packet: drtioaux::Packet) -> Result<(), drtioaux::Error<!>
             }
             Ok(())
         },
-        drtioaux::Packet::InjectionStatusRequest { channel, overrd } => {
+        drtioaux::Packet::InjectionStatusRequest { destination: _destination, channel, overrd } => {
+            forward!(_routing_table, _destination, *_rank, _repeaters, &packet);
             let value;
             #[cfg(has_rtio_moninj)]
             unsafe {
@@ -125,53 +231,61 @@ fn process_aux_packet(packet: drtioaux::Packet) -> Result<(), drtioaux::Error<!>
             {
                 value = 0;
             }
-            drtioaux::send_link(0, &drtioaux::Packet::InjectionStatusReply { value: value })
+            drtioaux::send(0, &drtioaux::Packet::InjectionStatusReply { value: value })
         },
 
-        drtioaux::Packet::I2cStartRequest { busno } => {
+        drtioaux::Packet::I2cStartRequest { destination: _destination, busno } => {
+            forward!(_routing_table, _destination, *_rank, _repeaters, &packet);
             let succeeded = i2c::start(busno).is_ok();
-            drtioaux::send_link(0, &drtioaux::Packet::I2cBasicReply { succeeded: succeeded })
+            drtioaux::send(0, &drtioaux::Packet::I2cBasicReply { succeeded: succeeded })
         }
-        drtioaux::Packet::I2cRestartRequest { busno } => {
+        drtioaux::Packet::I2cRestartRequest { destination: _destination, busno } => {
+            forward!(_routing_table, _destination, *_rank, _repeaters, &packet);
             let succeeded = i2c::restart(busno).is_ok();
-            drtioaux::send_link(0, &drtioaux::Packet::I2cBasicReply { succeeded: succeeded })
+            drtioaux::send(0, &drtioaux::Packet::I2cBasicReply { succeeded: succeeded })
         }
-        drtioaux::Packet::I2cStopRequest { busno } => {
+        drtioaux::Packet::I2cStopRequest { destination: _destination, busno } => {
+            forward!(_routing_table, _destination, *_rank, _repeaters, &packet);
             let succeeded = i2c::stop(busno).is_ok();
-            drtioaux::send_link(0, &drtioaux::Packet::I2cBasicReply { succeeded: succeeded })
+            drtioaux::send(0, &drtioaux::Packet::I2cBasicReply { succeeded: succeeded })
         }
-        drtioaux::Packet::I2cWriteRequest { busno, data } => {
+        drtioaux::Packet::I2cWriteRequest { destination: _destination, busno, data } => {
+            forward!(_routing_table, _destination, *_rank, _repeaters, &packet);
             match i2c::write(busno, data) {
-                Ok(ack) => drtioaux::send_link(0,
+                Ok(ack) => drtioaux::send(0,
                     &drtioaux::Packet::I2cWriteReply { succeeded: true, ack: ack }),
-                Err(_) => drtioaux::send_link(0,
+                Err(_) => drtioaux::send(0,
                     &drtioaux::Packet::I2cWriteReply { succeeded: false, ack: false })
             }
         }
-        drtioaux::Packet::I2cReadRequest { busno, ack } => {
+        drtioaux::Packet::I2cReadRequest { destination: _destination, busno, ack } => {
+            forward!(_routing_table, _destination, *_rank, _repeaters, &packet);
             match i2c::read(busno, ack) {
-                Ok(data) => drtioaux::send_link(0,
+                Ok(data) => drtioaux::send(0,
                     &drtioaux::Packet::I2cReadReply { succeeded: true, data: data }),
-                Err(_) => drtioaux::send_link(0,
+                Err(_) => drtioaux::send(0,
                     &drtioaux::Packet::I2cReadReply { succeeded: false, data: 0xff })
             }
         }
 
-        drtioaux::Packet::SpiSetConfigRequest { busno, flags, length, div, cs } => {
+        drtioaux::Packet::SpiSetConfigRequest { destination: _destination, busno, flags, length, div, cs } => {
+            forward!(_routing_table, _destination, *_rank, _repeaters, &packet);
             let succeeded = spi::set_config(busno, flags, length, div, cs).is_ok();
-            drtioaux::send_link(0,
+            drtioaux::send(0,
                 &drtioaux::Packet::SpiBasicReply { succeeded: succeeded })
         },
-        drtioaux::Packet::SpiWriteRequest { busno, data } => {
+        drtioaux::Packet::SpiWriteRequest { destination: _destination, busno, data } => {
+            forward!(_routing_table, _destination, *_rank, _repeaters, &packet);
             let succeeded = spi::write(busno, data).is_ok();
-            drtioaux::send_link(0,
+            drtioaux::send(0,
                 &drtioaux::Packet::SpiBasicReply { succeeded: succeeded })
         }
-        drtioaux::Packet::SpiReadRequest { busno } => {
+        drtioaux::Packet::SpiReadRequest { destination: _destination, busno } => {
+            forward!(_routing_table, _destination, *_rank, _repeaters, &packet);
             match spi::read(busno) {
-                Ok(data) => drtioaux::send_link(0,
+                Ok(data) => drtioaux::send(0,
                     &drtioaux::Packet::SpiReadReply { succeeded: true, data: data }),
-                Err(_) => drtioaux::send_link(0,
+                Err(_) => drtioaux::send(0,
                     &drtioaux::Packet::SpiReadReply { succeeded: false, data: 0 })
             }
         }
@@ -183,11 +297,12 @@ fn process_aux_packet(packet: drtioaux::Packet) -> Result<(), drtioaux::Error<!>
     }
 }
 
-fn process_aux_packets() {
+fn process_aux_packets(repeaters: &mut [repeater::Repeater],
+        routing_table: &mut drtio_routing::RoutingTable, rank: &mut u8) {
     let result =
-        drtioaux::recv_link(0).and_then(|packet| {
+        drtioaux::recv(0).and_then(|packet| {
             if let Some(packet) = packet {
-                process_aux_packet(packet)
+                process_aux_packet(repeaters, routing_table, rank, packet)
             } else {
                 Ok(())
             }
@@ -198,10 +313,10 @@ fn process_aux_packets() {
     }
 }
 
-fn process_errors() {
+fn drtiosat_process_errors() {
     let errors;
     unsafe {
-        errors = (csr::DRTIO[0].protocol_error_read)();
+        errors = csr::drtiosat::protocol_error_read();
     }
     if errors & 1 != 0 {
         error!("received packet of an unknown type");
@@ -210,25 +325,49 @@ fn process_errors() {
         error!("received truncated packet");
     }
     if errors & 4 != 0 {
+        let destination;
+        unsafe {
+            destination = csr::drtiosat::buffer_space_timeout_dest_read();
+        }
+        error!("timeout attempting to get buffer space from CRI, destination=0x{:02x}", destination)
+    }
+    if errors & 8 != 0 {
         let channel;
         let timestamp_event;
         let timestamp_counter;
         unsafe {
-            channel = (csr::DRTIO[0].underflow_channel_read)();
-            timestamp_event = (csr::DRTIO[0].underflow_timestamp_event_read)() as i64;
-            timestamp_counter = (csr::DRTIO[0].underflow_timestamp_counter_read)() as i64;
+            channel = csr::drtiosat::underflow_channel_read();
+            timestamp_event = csr::drtiosat::underflow_timestamp_event_read() as i64;
+            timestamp_counter = csr::drtiosat::underflow_timestamp_counter_read() as i64;
         }
         error!("write underflow, channel={}, timestamp={}, counter={}, slack={}",
                channel, timestamp_event, timestamp_counter, timestamp_event-timestamp_counter);
     }
-    if errors & 8 != 0 {
+    if errors & 16 != 0 {
         error!("write overflow");
     }
     unsafe {
-        (csr::DRTIO[0].protocol_error_write)(errors);
+        csr::drtiosat::protocol_error_write(errors);
     }
 }
 
+
+#[cfg(has_rtio_crg)]
+fn init_rtio_crg() {
+    unsafe {
+        csr::rtio_crg::pll_reset_write(0);
+    }
+    clock::spin_us(150);
+    let locked = unsafe { csr::rtio_crg::pll_locked_read() != 0 };
+    if !locked {
+        error!("RTIO clock failed");
+    }
+}
+
+#[cfg(not(has_rtio_crg))]
+fn init_rtio_crg() { }
+
+
 #[cfg(rtio_frequency = "150.0")]
 const SI5324_SETTINGS: si5324::FrequencySettings
     = si5324::FrequencySettings {
@@ -255,12 +394,6 @@ const SI5324_SETTINGS: si5324::FrequencySettings
     crystal_ref: true
 };
 
-fn drtio_link_rx_up() -> bool {
-    unsafe {
-        (csr::DRTIO[0].rx_up_read)() == 1
-    }
-}
-
 const SIPHASER_PHASE: u16 = 32;
 
 #[no_mangle]
@@ -285,16 +418,30 @@ pub extern fn main() -> i32 {
     unsafe {
         csr::drtio_transceiver::stable_clkin_write(1);
     }
+    init_rtio_crg();
 
     #[cfg(has_allaki_atts)]
     board_artiq::hmc542::program_all(8/*=4dB*/);
 
+    #[cfg(has_drtio_routing)]
+    let mut repeaters = [repeater::Repeater::default(); csr::DRTIOREP.len()];
+    #[cfg(not(has_drtio_routing))]
+    let mut repeaters = [repeater::Repeater::default(); 0];
+    for i in 0..repeaters.len() {
+        repeaters[i] = repeater::Repeater::new(i as u8);
+    } 
+    let mut routing_table = drtio_routing::RoutingTable::default_empty();
+    let mut rank = 1;
+
     loop {
-        while !drtio_link_rx_up() {
-            process_errors();
+        while !drtiosat_link_rx_up() {
+            drtiosat_process_errors();
+            for mut rep in repeaters.iter_mut() {
+                rep.service(&routing_table, rank);
+            }
         }
 
-        info!("link is up, switching to recovered clock");
+        info!("uplink is up, switching to recovered clock");
         si5324::siphaser::select_recovered_clock(true).expect("failed to switch clocks");
         si5324::siphaser::calibrate_skew(SIPHASER_PHASE).expect("failed to calibrate skew");
 
@@ -318,13 +465,17 @@ pub extern fn main() -> i32 {
         }
 
         drtioaux::reset(0);
-        drtio_reset(false);
-        drtio_reset_phy(false);
+        drtiosat_reset(false);
+        drtiosat_reset_phy(false);
 
-        while drtio_link_rx_up() {
-            process_errors();
-            process_aux_packets();
-            if drtio_tsc_loaded() {
+        while drtiosat_link_rx_up() {
+            drtiosat_process_errors();
+            process_aux_packets(&mut repeaters, &mut routing_table, &mut rank);
+            for mut rep in repeaters.iter_mut() {
+                rep.service(&routing_table, rank);
+            }
+            if drtiosat_tsc_loaded() {
+                info!("TSC loaded from uplink");
                 #[cfg(has_ad9154)]
                 {
                     if let Err(e) = board_artiq::jesd204sync::sysref_auto_rtio_align() {
@@ -334,7 +485,12 @@ pub extern fn main() -> i32 {
                         error!("failed to align SYSREF at DAC: {}", e);
                     }
                 }
-                if let Err(e) = drtioaux::send_link(0, &drtioaux::Packet::TSCAck) {
+                for rep in repeaters.iter() {
+                    if let Err(e) = rep.sync_tsc() {
+                        error!("failed to sync TSC ({})", e);
+                    }
+                }
+                if let Err(e) = drtioaux::send(0, &drtioaux::Packet::TSCAck) {
                     error!("aux packet error: {}", e);
                 }
             }
@@ -343,16 +499,33 @@ pub extern fn main() -> i32 {
         #[cfg(has_ad9154)]
         board_artiq::ad9154::jesd_reset(true);
 
-        drtio_reset_phy(true);
-        drtio_reset(true);
-        drtio_tsc_loaded();
-        info!("link is down, switching to local crystal clock");
+        drtiosat_reset_phy(true);
+        drtiosat_reset(true);
+        drtiosat_tsc_loaded();
+        info!("uplink is down, switching to local crystal clock");
         si5324::siphaser::select_recovered_clock(false).expect("failed to switch clocks");
     }
 }
 
 #[no_mangle]
 pub extern fn exception(vect: u32, _regs: *const u32, pc: u32, ea: u32) {
+    let vect = irq::Exception::try_from(vect).expect("unknown exception");
+
+    fn hexdump(addr: u32) {
+        let addr = (addr - addr % 4) as *const u32;
+        let mut ptr  = addr;
+        println!("@ {:08p}", ptr);
+        for _ in 0..4 {
+            print!("+{:04x}: ", ptr as usize - addr as usize);
+            print!("{:08x} ",   unsafe { *ptr }); ptr = ptr.wrapping_offset(1);
+            print!("{:08x} ",   unsafe { *ptr }); ptr = ptr.wrapping_offset(1);
+            print!("{:08x} ",   unsafe { *ptr }); ptr = ptr.wrapping_offset(1);
+            print!("{:08x}\n",  unsafe { *ptr }); ptr = ptr.wrapping_offset(1);
+        }
+    }
+
+    hexdump(pc);
+    hexdump(ea);
     panic!("exception {:?} at PC 0x{:x}, EA 0x{:x}", vect, pc, ea)
 }
 

artiq/firmware/satman/repeater.rs

@@ -0,0 +1,278 @@
+use board_artiq::{drtioaux, drtio_routing};
+#[cfg(has_drtio_routing)]
+use board_misoc::{csr, clock};
+
+#[cfg(has_drtio_routing)]
+fn rep_link_rx_up(repno: u8) -> bool {
+    let repno = repno as usize;
+    unsafe {
+        (csr::DRTIOREP[repno].rx_up_read)() == 1
+    }
+}
+
+#[cfg(has_drtio_routing)]
+#[derive(Clone, Copy, PartialEq)]
+enum RepeaterState {
+    Down,
+    SendPing { ping_count: u16 },
+    WaitPingReply { ping_count: u16, timeout: u64 },
+    Up,
+    Failed
+}
+
+#[cfg(has_drtio_routing)]
+impl Default for RepeaterState {
+    fn default() -> RepeaterState { RepeaterState::Down }
+}
+
+#[cfg(has_drtio_routing)]
+#[derive(Clone, Copy, Default)]
+pub struct Repeater {
+    repno: u8,
+    auxno: u8,
+    state: RepeaterState
+}
+
+#[cfg(has_drtio_routing)]
+impl Repeater {
+    pub fn new(repno: u8) -> Repeater {
+        Repeater {
+            repno: repno,
+            auxno: repno + 1,
+            state: RepeaterState::Down
+        }
+    }
+
+    pub fn service(&mut self, routing_table: &drtio_routing::RoutingTable, rank: u8) {
+        self.process_local_errors();
+
+        match self.state {
+            RepeaterState::Down => {
+                if rep_link_rx_up(self.repno) {
+                    info!("[REP#{}] link RX became up, pinging", self.repno);
+                    self.state = RepeaterState::SendPing { ping_count: 0 };
+                }
+            }
+            RepeaterState::SendPing { ping_count } => {
+                if rep_link_rx_up(self.repno) {
+                    drtioaux::send(self.auxno, &drtioaux::Packet::EchoRequest).unwrap();
+                    self.state = RepeaterState::WaitPingReply {
+                        ping_count: ping_count + 1,
+                        timeout: clock::get_ms() + 100
+                    }
+                } else {
+                    error!("[REP#{}] link RX went down during ping", self.repno);
+                    self.state = RepeaterState::Down;
+                }
+            }
+            RepeaterState::WaitPingReply { ping_count, timeout } => {
+                if rep_link_rx_up(self.repno) {
+                    if let Ok(Some(drtioaux::Packet::EchoReply)) = drtioaux::recv(self.auxno) {
+                        info!("[REP#{}] remote replied after {} packets", self.repno, ping_count);
+                        self.state = RepeaterState::Up;
+                        if let Err(e) = self.sync_tsc() {
+                            error!("[REP#{}] failed to sync TSC ({})", self.repno, e);
+                            self.state = RepeaterState::Failed;
+                            return;
+                        }
+                        if let Err(e) = self.load_routing_table(routing_table) {
+                            error!("[REP#{}] failed to load routing table ({})", self.repno, e);
+                            self.state = RepeaterState::Failed;
+                            return;
+                        }
+                        if let Err(e) = self.set_rank(rank + 1) {
+                            error!("[REP#{}] failed to set rank ({})", self.repno, e);
+                            self.state = RepeaterState::Failed;
+                            return;
+                        }
+                    } else {
+                        if clock::get_ms() > timeout {
+                            if ping_count > 200 {
+                                error!("[REP#{}] ping failed", self.repno);
+                                self.state = RepeaterState::Failed;
+                            } else {
+                                self.state = RepeaterState::SendPing { ping_count: ping_count };
+                            }
+                        }
+                    }
+                } else {
+                    error!("[REP#{}] link RX went down during ping", self.repno);
+                    self.state = RepeaterState::Down;
+                }
+            }
+            RepeaterState::Up => {
+                self.process_unsolicited_aux();
+                if !rep_link_rx_up(self.repno) {
+                    info!("[REP#{}] link is down", self.repno);
+                    self.state = RepeaterState::Down;
+                }
+            }
+            RepeaterState::Failed => {
+                if !rep_link_rx_up(self.repno) {
+                    info!("[REP#{}] link is down", self.repno);
+                    self.state = RepeaterState::Down;
+                }
+            }
+        }
+    }
+
+    fn process_unsolicited_aux(&self) {
+        match drtioaux::recv(self.auxno) {
+            Ok(Some(packet)) => warn!("[REP#{}] unsolicited aux packet: {:?}", self.repno, packet),
+            Ok(None) => (),
+            Err(_) => warn!("[REP#{}] aux packet error", self.repno)
+        }
+    }
+
+    fn process_local_errors(&self) {
+        let repno = self.repno as usize;
+        let errors;
+        unsafe {
+            errors = (csr::DRTIOREP[repno].protocol_error_read)();
+        }
+        if errors & 1 != 0 {
+            error!("[REP#{}] received packet of an unknown type", repno);
+        }
+        if errors & 2 != 0 {
+            error!("[REP#{}] received truncated packet", repno);
+        }
+        if errors & 4 != 0 {
+            let cmd;
+            let chan_sel;
+            unsafe {
+                cmd = (csr::DRTIOREP[repno].command_missed_cmd_read)();
+                chan_sel = (csr::DRTIOREP[repno].command_missed_chan_sel_read)();
+            }
+            error!("[REP#{}] CRI command missed, cmd={}, chan_sel=0x{:06x}", repno, cmd, chan_sel)
+        }
+        if errors & 8 != 0 {
+            let destination;
+            unsafe {
+                destination = (csr::DRTIOREP[repno].buffer_space_timeout_dest_read)();
+            }
+            error!("[REP#{}] timeout attempting to get remote buffer space, destination=0x{:02x}", repno, destination);
+        }
+        unsafe {
+            (csr::DRTIOREP[repno].protocol_error_write)(errors);
+        }
+    }
+
+    fn recv_aux_timeout(&self, timeout: u32) -> Result<drtioaux::Packet, drtioaux::Error<!>> {
+        let max_time = clock::get_ms() + timeout as u64;
+        loop {
+            if !rep_link_rx_up(self.repno) {
+                return Err(drtioaux::Error::LinkDown);
+            }
+            if clock::get_ms() > max_time {
+                return Err(drtioaux::Error::TimedOut);
+            }
+            match drtioaux::recv(self.auxno) {
+                Ok(Some(packet)) => return Ok(packet),
+                Ok(None) => (),
+                Err(e) => return Err(e)
+            }
+        }
+    }
+
+    pub fn aux_forward(&self, request: &drtioaux::Packet) -> Result<(), drtioaux::Error<!>> {
+        if self.state != RepeaterState::Up {
+            return Err(drtioaux::Error::LinkDown);
+        }
+        drtioaux::send(self.auxno, request).unwrap();
+        let reply = self.recv_aux_timeout(200)?;
+        drtioaux::send(0, &reply).unwrap();
+        Ok(())
+    }
+
+    pub fn sync_tsc(&self) -> Result<(), drtioaux::Error<!>> {
+        if self.state != RepeaterState::Up {
+            return Ok(());
+        }
+
+        let repno = self.repno as usize;
+        unsafe {
+            (csr::DRTIOREP[repno].set_time_write)(1);
+            while (csr::DRTIOREP[repno].set_time_read)() == 1 {}
+        }
+
+        // TSCAck is the only aux packet that is sent spontaneously
+        // by the satellite, in response to a TSC set on the RT link.
+        let reply = self.recv_aux_timeout(10000)?;
+        if reply == drtioaux::Packet::TSCAck {
+            return Ok(());
+        } else {
+            return Err(drtioaux::Error::UnexpectedReply);
+        }
+    }
+
+    pub fn set_path(&self, destination: u8, hops: &[u8; drtio_routing::MAX_HOPS]) -> Result<(), drtioaux::Error<!>> {
+        if self.state != RepeaterState::Up {
+            return Ok(());
+        }
+
+        drtioaux::send(self.auxno, &drtioaux::Packet::RoutingSetPath {
+            destination: destination,
+            hops: *hops
+        }).unwrap();
+        let reply = self.recv_aux_timeout(200)?;
+        if reply != drtioaux::Packet::RoutingAck {
+            return Err(drtioaux::Error::UnexpectedReply);
+        }
+        Ok(())
+    }
+
+    pub fn load_routing_table(&self, routing_table: &drtio_routing::RoutingTable) -> Result<(), drtioaux::Error<!>> {
+        for i in 0..drtio_routing::DEST_COUNT {
+            self.set_path(i as u8, &routing_table.0[i])?;
+        }
+        Ok(())
+    }
+
+    pub fn set_rank(&self, rank: u8) -> Result<(), drtioaux::Error<!>> {
+        if self.state != RepeaterState::Up {
+            return Ok(());
+        }
+        drtioaux::send(self.auxno, &drtioaux::Packet::RoutingSetRank {
+            rank: rank
+        }).unwrap();
+        let reply = self.recv_aux_timeout(200)?;
+        if reply != drtioaux::Packet::RoutingAck {
+            return Err(drtioaux::Error::UnexpectedReply);
+        }
+        Ok(())
+    }
+
+    pub fn rtio_reset(&self) -> Result<(), drtioaux::Error<!>> {
+        let repno = self.repno as usize;
+        unsafe { (csr::DRTIOREP[repno].reset_write)(1); }
+        clock::spin_us(100);
+        unsafe { (csr::DRTIOREP[repno].reset_write)(0); }
+
+        if self.state != RepeaterState::Up {
+            return Ok(());
+        }
+
+        drtioaux::send(self.auxno, &drtioaux::Packet::ResetRequest).unwrap();
+        let reply = self.recv_aux_timeout(200)?;
+        if reply != drtioaux::Packet::ResetAck {
+            return Err(drtioaux::Error::UnexpectedReply);
+        }
+        Ok(())
+    }
+}
+
+#[cfg(not(has_drtio_routing))]
+#[derive(Clone, Copy, Default)]
+pub struct Repeater {
+}
+
+#[cfg(not(has_drtio_routing))]
+impl Repeater {
+    pub fn new(_repno: u8) -> Repeater { Repeater::default() }
+
+    pub fn service(&self, _routing_table: &drtio_routing::RoutingTable, _rank: u8) { }
+
+    pub fn sync_tsc(&self) -> Result<(), drtioaux::Error<!>> { Ok(()) }
+
+    pub fn rtio_reset(&self) -> Result<(), drtioaux::Error<!>> { Ok(()) }
+}

artiq/firmware/satman/satman.ld

@@ -49,7 +49,7 @@ SECTIONS
     .stack :
     {
         _estack = .;
-        . += 0x1000;
+        . += 0x10000;
         _fstack = . - 4;
     } > main_ram
 }

artiq/frontend/aqctl_corelog.py

@@ -16,7 +16,7 @@ def get_argparser():
     parser = argparse.ArgumentParser(
         description="ARTIQ controller for core device logs")
     simple_network_args(parser, 1068)
-    parser.add_argument("core_addr",
+    parser.add_argument("core_addr", metavar="CORE_ADDR",
                         help="hostname or IP address of the core device")
     return parser
 

artiq/frontend/artiq_route.py

@@ -0,0 +1,83 @@
+#!/usr/bin/env python3
+
+import argparse
+
+
+def get_argparser():
+    parser = argparse.ArgumentParser(description="ARTIQ DRTIO routing table "
+                                                 "manipulation tool")
+
+    parser.add_argument("file", metavar="FILE", type=str,
+                        help="target file")
+
+    action = parser.add_subparsers(dest="action")
+    action.required = True
+
+    action.add_parser("init", help="create a new empty routing table")
+    
+    action.add_parser("show", help="show contents of routing table")
+
+    a_set = action.add_parser("set", help="set routing table entry")
+    a_set.add_argument("destination", metavar="DESTINATION", type=int,
+                       help="destination to operate on")
+    a_set.add_argument("hop", metavar="HOP", type=int, nargs="*",
+                       help="hop(s) to the destination")
+
+    return parser
+
+
+DEST_COUNT = 256
+MAX_HOPS = 32
+
+
+def init(filename):
+    with open(filename, "wb") as f:
+        f.write(b"\xff"*(DEST_COUNT*MAX_HOPS))
+
+
+def show_routes(filename):
+    routes = []
+    with open(filename, "rb") as f:
+        for i in range(DEST_COUNT):
+            hops = [int.from_bytes(f.read(1), "big") for j in range(MAX_HOPS)]
+            routes.append(hops)
+
+    for destination, route in enumerate(routes):
+        if route[0] != 0xff:
+            fmt = "{:3d}:".format(destination)
+            for hop in route:
+                if hop == 0xff:
+                    break
+                fmt += " {:3d}".format(hop)
+            print(fmt)
+
+
+def set_route(filename, destination, hops):
+    with open(filename, "r+b") as f:
+        if destination >= DEST_COUNT:
+            raise ValueError("destination must be less than {}".format(DEST_COUNT))
+        f.seek(destination*MAX_HOPS)
+
+        if len(hops) + 1 >= MAX_HOPS:
+            raise ValueError("too many hops")
+        for hop in hops:
+            if hop >= 0xff:
+                raise ValueError("all hops must be less than 255")
+
+        hops = hops + [0xff]*(MAX_HOPS-len(hops))
+        f.write(bytes(hops))
+
+
+def main():
+    args = get_argparser().parse_args()
+    if args.action == "init":
+        init(args.file)
+    elif args.action == "show":
+        show_routes(args.file)
+    elif args.action == "set":
+        set_route(args.file, args.destination, args.hop)
+    else:
+        raise ValueError
+
+if __name__ == "__main__":
+    main()

artiq/gateware/drtio/__init__.py

@@ -1,2 +1,2 @@
-from artiq.gateware.drtio.core import DRTIOSatellite, DRTIOMaster
-
+from artiq.gateware.drtio.core import SyncRTIO, DRTIOSatellite, DRTIOMaster, DRTIORepeater
+from artiq.gateware.drtio.aux_controller import DRTIOAuxController

artiq/gateware/drtio/aux_controller.py

@@ -211,7 +211,7 @@ class Receiver(Module, AutoCSR):
 
 # TODO: FullMemoryWE should be applied by migen.build
 @FullMemoryWE()
-class AuxController(Module):
+class DRTIOAuxController(Module):
     def __init__(self, link_layer):
         self.bus = wishbone.Interface()
         self.submodules.transmitter = Transmitter(link_layer, len(self.bus.dat_w))

artiq/gateware/drtio/cdc.py

@@ -0,0 +1,55 @@
+from migen import *
+
+from migen.genlib.cdc import PulseSynchronizer
+
+
+class CrossDomainRequest(Module):
+    def __init__(self, domain,
+                 req_stb, req_ack, req_data,
+                 srv_stb, srv_ack, srv_data):
+        dsync = getattr(self.sync, domain)
+
+        request = PulseSynchronizer("sys", domain)
+        reply = PulseSynchronizer(domain, "sys")
+        self.submodules += request, reply
+
+        ongoing = Signal()
+        self.comb += request.i.eq(~ongoing & req_stb)
+        self.sync += [
+            req_ack.eq(reply.o),
+            If(req_stb, ongoing.eq(1)),
+            If(req_ack, ongoing.eq(0))
+        ]
+        if req_data is not None:
+            req_data_r = Signal.like(req_data)
+            req_data_r.attr.add("no_retiming")
+            self.sync += If(req_stb, req_data_r.eq(req_data))
+        dsync += [
+            If(request.o, srv_stb.eq(1)),
+            If(srv_ack, srv_stb.eq(0))
+        ]
+        if req_data is not None:
+            dsync += If(request.o, srv_data.eq(req_data_r))
+        self.comb += reply.i.eq(srv_stb & srv_ack)
+
+
+class CrossDomainNotification(Module):
+    def __init__(self, domain, rdomain,
+                 emi_stb, emi_data,
+                 rec_stb, rec_ack, rec_data):
+        emi_data_r = Signal(len(emi_data))
+        emi_data_r.attr.add("no_retiming")
+        dsync = getattr(self.sync, domain)
+        dsync += If(emi_stb, emi_data_r.eq(emi_data))
+
+        ps = PulseSynchronizer(domain, rdomain)
+        self.submodules += ps
+        self.comb += ps.i.eq(emi_stb)
+        rsync = getattr(self.sync, rdomain)
+        rsync += [
+            If(rec_ack, rec_stb.eq(0)),
+            If(ps.o,
+                rec_data.eq(emi_data_r),
+                rec_stb.eq(1)
+            )
+        ]

artiq/gateware/drtio/core.py

@@ -5,14 +5,21 @@ from migen.genlib.resetsync import AsyncResetSynchronizer
 from migen.genlib.cdc import PulseSynchronizer
 from misoc.interconnect.csr import *
 
+from artiq.gateware.rtio import cri, rtlink
 from artiq.gateware.rtio.sed.core import *
 from artiq.gateware.rtio.input_collector import *
-from artiq.gateware.drtio import (link_layer, aux_controller,
+from artiq.gateware.drtio import (link_layer,
                                   rt_packet_satellite, rt_errors_satellite,
-                                  rt_packet_master, rt_controller_master)
+                                  rt_packet_master, rt_controller_master,
+                                  rt_packet_repeater, rt_controller_repeater)
 from artiq.gateware.drtio.rx_synchronizer import GenericRXSynchronizer
 
 
+__all__ = ["ChannelInterface", "TransceiverInterface",
+           "SyncRTIO",
+           "DRTIOSatellite", "DRTIOMaster", "DRTIORepeater"]
+
+
 class ChannelInterface:
     def __init__(self, encoder, decoders):
         self.rx_ready = Signal()
@@ -30,12 +37,50 @@ class TransceiverInterface(AutoCSR):
         self.channels = channel_interfaces
 
 
+async_errors_layout = [
+    ("sequence_error", 1),
+    ("sequence_error_channel", 16),
+    ("collision", 1),
+    ("collision_channel", 16),
+    ("busy", 1),
+    ("busy_channel", 16)
+]
+
+
+class SyncRTIO(Module):
+    def __init__(self, tsc, channels, lane_count=8, fifo_depth=128):
+        self.cri = cri.Interface()
+        self.async_errors = Record(async_errors_layout)
+
+        chan_fine_ts_width = max(max(rtlink.get_fine_ts_width(channel.interface.o)
+                                     for channel in channels),
+                                 max(rtlink.get_fine_ts_width(channel.interface.i)
+                                     for channel in channels))
+        assert tsc.glbl_fine_ts_width >= chan_fine_ts_width
+
+        self.submodules.outputs = ClockDomainsRenamer("rio")(
+            SED(channels, tsc.glbl_fine_ts_width, "sync",
+                lane_count=lane_count, fifo_depth=fifo_depth,
+                enable_spread=False, report_buffer_space=True,
+                interface=self.cri))
+        self.comb += self.outputs.coarse_timestamp.eq(tsc.coarse_ts)
+        self.sync.rtio += self.outputs.minimum_coarse_timestamp.eq(tsc.coarse_ts + 16)
+
+        self.submodules.inputs = ClockDomainsRenamer("rio")(
+            InputCollector(tsc, channels, "sync", interface=self.cri))
+
+        for attr, _ in async_errors_layout:
+            self.comb += getattr(self.async_errors, attr).eq(getattr(self.outputs, attr))
+
+
 class DRTIOSatellite(Module):
-    def __init__(self, chanif, channels, rx_synchronizer=None, fine_ts_width=3,
-                 lane_count=8, fifo_depth=128):
+    def __init__(self, tsc, chanif, rx_synchronizer=None):
         self.reset = CSRStorage(reset=1)
         self.reset_phy = CSRStorage(reset=1)
         self.tsc_loaded = CSR()
+        # master interface in the rtio domain
+        self.cri = cri.Interface()
+        self.async_errors = Record(async_errors_layout)
 
         self.clock_domains.cd_rio = ClockDomain()
         self.clock_domains.cd_rio_phy = ClockDomain()
@@ -81,18 +126,13 @@ class DRTIOSatellite(Module):
         )
         self.submodules.link_stats = link_layer.LinkLayerStats(link_layer_sync, "rtio")
         self.submodules.rt_packet = ClockDomainsRenamer("rtio")(
-            rt_packet_satellite.RTPacketSatellite(link_layer_sync))
+            rt_packet_satellite.RTPacketSatellite(link_layer_sync, interface=self.cri))
         self.comb += self.rt_packet.reset.eq(self.cd_rio.rst)
 
-        coarse_ts = Signal(64 - fine_ts_width)
-        self.sync.rtio += \
-            If(self.rt_packet.tsc_load,
-                coarse_ts.eq(self.rt_packet.tsc_load_value)
-            ).Else(
-                coarse_ts.eq(coarse_ts + 1)
-            )
-        self.comb += self.rt_packet.cri.counter.eq(coarse_ts << fine_ts_width)
-        self.coarse_ts = coarse_ts
+        self.comb += [
+            tsc.load.eq(self.rt_packet.tsc_load),
+            tsc.load_value.eq(self.rt_packet.tsc_load_value)
+        ]
 
         ps_tsc_load = PulseSynchronizer("rtio", "sys")
         self.submodules += ps_tsc_load
@@ -102,33 +142,17 @@ class DRTIOSatellite(Module):
             If(ps_tsc_load.o, self.tsc_loaded.w.eq(1))
         ]
 
-        self.submodules.outputs = ClockDomainsRenamer("rio")(
-            SED(channels, fine_ts_width, "sync",
-                lane_count=lane_count, fifo_depth=fifo_depth,
-                enable_spread=False, report_buffer_space=True,
-                interface=self.rt_packet.cri))
-        self.comb += self.outputs.coarse_timestamp.eq(coarse_ts)
-        self.sync.rtio += self.outputs.minimum_coarse_timestamp.eq(coarse_ts + 16)
-
-        self.submodules.inputs = ClockDomainsRenamer("rio")(
-            InputCollector(channels, fine_ts_width, "sync",
-                           interface=self.rt_packet.cri))
-        self.comb += self.inputs.coarse_timestamp.eq(coarse_ts)
-
         self.submodules.rt_errors = rt_errors_satellite.RTErrorsSatellite(
-            self.rt_packet, self.outputs)
-
-        self.submodules.aux_controller = aux_controller.AuxController(
-            self.link_layer)
+            self.rt_packet, tsc, self.async_errors)
 
     def get_csrs(self):
         return ([self.reset, self.reset_phy, self.tsc_loaded] +
                 self.link_layer.get_csrs() + self.link_stats.get_csrs() +
-                self.rt_errors.get_csrs() + self.aux_controller.get_csrs())
+                self.rt_errors.get_csrs())
 
 
 class DRTIOMaster(Module):
-    def __init__(self, chanif, fine_ts_width=3):
+    def __init__(self, tsc, chanif):
         self.submodules.link_layer = link_layer.LinkLayer(
             chanif.encoder, chanif.decoders)
         self.comb += self.link_layer.rx_ready.eq(chanif.rx_ready)
@@ -136,19 +160,33 @@ class DRTIOMaster(Module):
         self.submodules.link_stats = link_layer.LinkLayerStats(self.link_layer, "rtio_rx")
         self.submodules.rt_packet = rt_packet_master.RTPacketMaster(self.link_layer)
         self.submodules.rt_controller = rt_controller_master.RTController(
-            self.rt_packet, fine_ts_width)
-        self.submodules.rt_manager = rt_controller_master.RTManager(self.rt_packet)
-
-        self.submodules.aux_controller = aux_controller.AuxController(
-            self.link_layer)
+            tsc, self.rt_packet)
 
     def get_csrs(self):
         return (self.link_layer.get_csrs() +
                 self.link_stats.get_csrs() +
-                self.rt_controller.get_csrs() +
-                self.rt_manager.get_csrs() +
-                self.aux_controller.get_csrs())
+                self.rt_controller.get_csrs())
 
     @property
     def cri(self):
         return self.rt_controller.cri
+
+
+class DRTIORepeater(Module):
+    def __init__(self, tsc, chanif):
+        self.submodules.link_layer = link_layer.LinkLayer(
+            chanif.encoder, chanif.decoders)
+        self.comb += self.link_layer.rx_ready.eq(chanif.rx_ready)
+
+        self.submodules.link_stats = link_layer.LinkLayerStats(self.link_layer, "rtio_rx")
+        self.submodules.rt_packet = rt_packet_repeater.RTPacketRepeater(tsc, self.link_layer)
+        self.submodules.rt_controller = rt_controller_repeater.RTController(self.rt_packet)
+
+    def get_csrs(self):
+        return (self.link_layer.get_csrs() +
+                self.link_stats.get_csrs() +
+                self.rt_controller.get_csrs())
+
+    @property
+    def cri(self):
+        return self.rt_packet.cri

artiq/gateware/drtio/rt_controller_master.py

@@ -3,71 +3,41 @@
 from migen import *
 from migen.genlib.cdc import MultiReg
 from migen.genlib.misc import WaitTimer
-from migen.genlib.resetsync import AsyncResetSynchronizer
 
 from misoc.interconnect.csr import *
 
-from artiq.gateware.rtio.cdc import GrayCodeTransfer
 from artiq.gateware.rtio import cri
 
 
 class _CSRs(AutoCSR):
     def __init__(self):
-        self.link_up = CSRStorage()
+        self.reset = CSRStorage()
 
         self.protocol_error = CSR(3)
 
         self.set_time = CSR()
         self.underflow_margin = CSRStorage(16, reset=300)
 
+        self.force_destination = CSRStorage()
+        self.destination = CSRStorage(8)
+
         self.o_get_buffer_space = CSR()
         self.o_dbg_buffer_space = CSRStatus(16)
         self.o_dbg_buffer_space_req_cnt = CSRStatus(32)
         self.o_wait = CSRStatus()
 
 
-class RTIOCounter(Module):
-    def __init__(self, width):
-        self.width = width
-        # Timestamp counter in RTIO domain
-        self.value_rtio = Signal(width)
-        # Timestamp counter resynchronized to sys domain
-        # Lags behind value_rtio, monotonic and glitch-free
-        self.value_sys = Signal(width)
-
-        # # #
-
-        # note: counter is in rtio domain and never affected by the reset CSRs
-        self.sync.rtio += self.value_rtio.eq(self.value_rtio + 1)
-        gt = GrayCodeTransfer(width)
-        self.submodules += gt
-        self.comb += gt.i.eq(self.value_rtio), self.value_sys.eq(gt.o)
-
-
 class RTController(Module):
-    def __init__(self, rt_packet, fine_ts_width):
+    def __init__(self, tsc, rt_packet):
         self.csrs = _CSRs()
         self.cri = cri.Interface()
 
-        # reset
-        local_reset = Signal(reset=1)
-        self.sync += local_reset.eq(~self.csrs.link_up.storage)
-        local_reset.attr.add("no_retiming")
-        self.clock_domains.cd_sys_with_rst = ClockDomain()
-        self.clock_domains.cd_rtio_with_rst = ClockDomain()
-        self.comb += [
-            self.cd_sys_with_rst.clk.eq(ClockSignal()),
-            self.cd_sys_with_rst.rst.eq(local_reset)
-        ]
-        self.comb += self.cd_rtio_with_rst.clk.eq(ClockSignal("rtio"))
-        self.specials += AsyncResetSynchronizer(self.cd_rtio_with_rst, local_reset)
-
         # protocol errors
         err_unknown_packet_type = Signal()
         err_packet_truncated = Signal()
         signal_buffer_space_timeout = Signal()
         err_buffer_space_timeout = Signal()
-        self.sync.sys_with_rst += [
+        self.sync += [
             If(self.csrs.protocol_error.re,
                 If(self.csrs.protocol_error.r[0], err_unknown_packet_type.eq(0)),
                 If(self.csrs.protocol_error.r[1], err_packet_truncated.eq(0)),
@@ -80,11 +50,9 @@ class RTController(Module):
         self.comb += self.csrs.protocol_error.w.eq(
             Cat(err_unknown_packet_type, err_packet_truncated, err_buffer_space_timeout))
 
-        # master RTIO counter and counter synchronization
-        self.submodules.counter = RTIOCounter(64-fine_ts_width)
+        # TSC synchronization
         self.comb += [
-            self.cri.counter.eq(self.counter.value_sys << fine_ts_width),
-            rt_packet.tsc_value.eq(self.counter.value_rtio),
+            rt_packet.tsc_value.eq(tsc.coarse_ts),
             self.csrs.set_time.w.eq(rt_packet.set_time_stb)
         ]
         self.sync += [
@@ -92,12 +60,17 @@ class RTController(Module):
             If(self.csrs.set_time.re, rt_packet.set_time_stb.eq(1))
         ]
 
+        # chan_sel forcing
+        chan_sel = Signal(24)
+        self.comb += chan_sel.eq(Mux(self.csrs.force_destination.storage,
+            self.csrs.destination.storage << 16,
+            self.cri.chan_sel))
+
         # common packet fields
-        chan_sel = self.cri.chan_sel[:16]
         rt_packet_buffer_request = Signal()
         rt_packet_read_request = Signal()
         self.comb += [
-            rt_packet.sr_channel.eq(chan_sel),
+            rt_packet.sr_chan_sel.eq(chan_sel),
             rt_packet.sr_address.eq(self.cri.o_address),
             rt_packet.sr_data.eq(self.cri.o_data),
             rt_packet.sr_timestamp.eq(self.cri.timestamp),
@@ -115,12 +88,11 @@ class RTController(Module):
         o_status_wait = Signal()
         o_status_underflow = Signal()
         self.comb += [
-            self.cri.o_status.eq(Cat(
-                o_status_wait, o_status_underflow, ~self.csrs.link_up.storage)),
+            self.cri.o_status.eq(Cat(o_status_wait, o_status_underflow)),
             self.csrs.o_wait.status.eq(o_status_wait)
         ]
         o_underflow_set = Signal()
-        self.sync.sys_with_rst += [
+        self.sync += [
             If(self.cri.cmd == cri.commands["write"],
                 o_status_underflow.eq(0)
             ),
@@ -131,21 +103,35 @@ class RTController(Module):
         self.submodules += timeout_counter
 
         cond_underflow = Signal()
-        self.comb += cond_underflow.eq((self.cri.timestamp[fine_ts_width:]
-                           - self.csrs.underflow_margin.storage[fine_ts_width:]) < self.counter.value_sys)
+        self.comb += cond_underflow.eq((self.cri.timestamp[tsc.glbl_fine_ts_width:]
+                           - self.csrs.underflow_margin.storage[tsc.glbl_fine_ts_width:]) < tsc.coarse_ts_sys)
 
-        buffer_space = Signal(16)
+        # buffer space
+        buffer_space = Memory(16, 256)
+        buffer_space_port = buffer_space.get_port(write_capable=True)
+        self.specials += buffer_space, buffer_space_port
+
+        buffer_space_load = Signal()
+        buffer_space_dec = Signal()
+        self.comb += [
+            buffer_space_port.adr.eq(chan_sel[16:]),
+            buffer_space_port.we.eq(buffer_space_load | buffer_space_dec),
+            If(buffer_space_load,
+                buffer_space_port.dat_w.eq(rt_packet.buffer_space)
+            ).Else(
+                buffer_space_port.dat_w.eq(buffer_space_port.dat_r - 1)
+            )
+        ]
 
         # input status
         i_status_wait_event = Signal()
         i_status_overflow = Signal()
         i_status_wait_status = Signal()
         self.comb += self.cri.i_status.eq(Cat(
-            i_status_wait_event, i_status_overflow, i_status_wait_status,
-            ~self.csrs.link_up.storage))
+            i_status_wait_event, i_status_overflow, i_status_wait_status))
 
         load_read_reply = Signal()
-        self.sync.sys_with_rst += [
+        self.sync += [
             If(load_read_reply,
                 i_status_wait_event.eq(0),
                 i_status_overflow.eq(0),
@@ -162,7 +148,7 @@ class RTController(Module):
         ]
 
         # FSM
-        fsm = ClockDomainsRenamer("sys_with_rst")(FSM())
+        fsm = FSM()
         self.submodules += fsm
 
         fsm.act("IDLE",
@@ -180,8 +166,8 @@ class RTController(Module):
             o_status_wait.eq(1),
             rt_packet.sr_stb.eq(1),
             If(rt_packet.sr_ack,
-                NextValue(buffer_space, buffer_space - 1),
-                If(buffer_space <= 1,
+                buffer_space_dec.eq(1),
+                If(buffer_space_port.dat_r <= 1,
                     NextState("GET_BUFFER_SPACE")
                 ).Else(
                     NextState("IDLE")
@@ -199,7 +185,7 @@ class RTController(Module):
         )
         fsm.act("GET_BUFFER_SPACE_REPLY",
             o_status_wait.eq(1),
-            NextValue(buffer_space, rt_packet.buffer_space),
+            buffer_space_load.eq(1),
             rt_packet.buffer_space_not_ack.eq(1),
             If(rt_packet.buffer_space_not,
                If(rt_packet.buffer_space != 0,
@@ -211,7 +197,7 @@ class RTController(Module):
             timeout_counter.wait.eq(1),
             If(timeout_counter.done,
                 signal_buffer_space_timeout.eq(1),
-                NextState("GET_BUFFER_SPACE")
+                NextState("IDLE")
             )
         )
         fsm.act("READ",
@@ -226,14 +212,14 @@ class RTController(Module):
         fsm.act("GET_READ_REPLY",
             i_status_wait_status.eq(1),
             rt_packet.read_not_ack.eq(1),
-            If(rt_packet.read_not,
+            If(self.csrs.reset.storage | rt_packet.read_not,
                 load_read_reply.eq(1),
                 NextState("IDLE")
             )
         )
 
         # debug CSRs
-        self.comb += self.csrs.o_dbg_buffer_space.status.eq(buffer_space),
+        self.comb += self.csrs.o_dbg_buffer_space.status.eq(buffer_space_port.dat_r),
         self.sync += \
             If((rt_packet.sr_stb & rt_packet.sr_ack & rt_packet_buffer_request),
                self.csrs.o_dbg_buffer_space_req_cnt.status.eq(
@@ -242,26 +228,3 @@ class RTController(Module):
 
     def get_csrs(self):
         return self.csrs.get_csrs()
-
-
-class RTManager(Module, AutoCSR):
-    def __init__(self, rt_packet):
-        self.request_echo = CSR()
-
-        self.update_packet_cnt = CSR()
-        self.packet_cnt_tx = CSRStatus(32)
-        self.packet_cnt_rx = CSRStatus(32)
-
-        # # #
-
-        self.comb += self.request_echo.w.eq(rt_packet.echo_stb)
-        self.sync += [
-            If(rt_packet.echo_ack, rt_packet.echo_stb.eq(0)),
-            If(self.request_echo.re, rt_packet.echo_stb.eq(1))
-        ]
-
-        self.sync += \
-            If(self.update_packet_cnt.re,
-                self.packet_cnt_tx.status.eq(rt_packet.packet_cnt_tx),
-                self.packet_cnt_rx.status.eq(rt_packet.packet_cnt_rx)
-            )

artiq/gateware/drtio/rt_controller_repeater.py

@@ -0,0 +1,62 @@
+from migen import *
+from migen.genlib.cdc import MultiReg
+
+from misoc.interconnect.csr import *
+
+from artiq.gateware.rtio.cdc import BlindTransfer
+from artiq.gateware.drtio.cdc import CrossDomainRequest
+
+
+class RTController(Module, AutoCSR):
+    def __init__(self, rt_packet):
+        self.reset = CSRStorage()
+        self.set_time = CSR()
+        self.protocol_error = CSR(4)
+        self.command_missed_cmd = CSRStatus(2)
+        self.command_missed_chan_sel = CSRStatus(24)
+        self.buffer_space_timeout_dest = CSRStatus(8)
+
+        self.specials += MultiReg(self.reset.storage, rt_packet.reset, "rtio")
+
+        set_time_stb = Signal()
+        set_time_ack = Signal()
+        self.submodules += CrossDomainRequest("rtio",
+            set_time_stb, set_time_ack, None,
+            rt_packet.set_time_stb, rt_packet.set_time_ack, None)
+        self.sync += [
+            If(set_time_ack, set_time_stb.eq(0)),
+            If(self.set_time.re, set_time_stb.eq(1))
+        ]
+        self.comb += self.set_time.w.eq(set_time_stb)
+
+        errors = [
+            (rt_packet.err_unknown_packet_type, "rtio_rx", None, None),
+            (rt_packet.err_packet_truncated, "rtio_rx", None, None),
+            (rt_packet.err_command_missed, "rtio",
+                Cat(rt_packet.command_missed_cmd, rt_packet.command_missed_chan_sel),
+                Cat(self.command_missed_cmd.status, self.command_missed_chan_sel.status)),
+            (rt_packet.err_buffer_space_timeout, "rtio",
+                rt_packet.buffer_space_destination, self.buffer_space_timeout_dest.status)
+        ]
+
+        for n, (err_i, err_cd, din, dout) in enumerate(errors):
+            if din is not None:
+                data_width = len(din)
+            else:
+                data_width = 0
+
+            xfer = BlindTransfer(err_cd, "sys", data_width=data_width)
+            self.submodules += xfer
+
+            self.comb += xfer.i.eq(err_i)
+
+            err_pending = Signal()
+            self.sync += [
+                If(self.protocol_error.re & self.protocol_error.r[n], err_pending.eq(0)),
+                If(xfer.o, err_pending.eq(1))
+            ]
+            self.comb += self.protocol_error.w[n].eq(err_pending)
+
+            if din is not None:
+                self.comb += xfer.data_i.eq(din)
+                self.sync += If(xfer.o & ~err_pending, dout.eq(xfer.data_o))

artiq/gateware/drtio/rt_errors_satellite.py

@@ -7,11 +7,12 @@ from artiq.gateware.rtio.cdc import BlindTransfer
 
 
 class RTErrorsSatellite(Module, AutoCSR):
-    def __init__(self, rt_packet, outputs):
-        self.protocol_error = CSR(4)
+    def __init__(self, rt_packet, tsc, async_errors):
+        self.protocol_error = CSR(5)
         self.underflow_channel = CSRStatus(16)
         self.underflow_timestamp_event = CSRStatus(64)
         self.underflow_timestamp_counter = CSRStatus(64)
+        self.buffer_space_timeout_dest = CSRStatus(8)
 
         self.rtio_error = CSR(3)
         self.sequence_error_channel = CSRStatus(16)
@@ -47,6 +48,7 @@ class RTErrorsSatellite(Module, AutoCSR):
                     self.comb += xfer.data_i.eq(din)
                     self.sync += If(xfer.o & ~pending, dout.eq(xfer.data_o))
 
+        cri = rt_packet.cri
 
         # The master is normally responsible for avoiding output overflows
         # and output underflows. The error reports here are only for diagnosing
@@ -56,11 +58,11 @@ class RTErrorsSatellite(Module, AutoCSR):
         underflow_error_cri = Signal(16+64+64)
         underflow_error_csr = Signal(16+64+64)
         self.comb += [
-            underflow.eq(outputs.cri.o_status[1]),
-            overflow.eq(outputs.cri.o_status[0]),
-            underflow_error_cri.eq(Cat(outputs.cri.chan_sel[:16],
-                                       outputs.cri.timestamp,
-                                       outputs.cri.counter)),
+            underflow.eq(cri.o_status[1]),
+            overflow.eq(cri.o_status[0]),
+            underflow_error_cri.eq(Cat(cri.chan_sel[:16],
+                                       cri.timestamp,
+                                       tsc.full_ts_cri)),
             Cat(self.underflow_channel.status,
                 self.underflow_timestamp_event.status,
                 self.underflow_timestamp_counter.status).eq(underflow_error_csr)
@@ -68,15 +70,17 @@ class RTErrorsSatellite(Module, AutoCSR):
         error_csr(self.protocol_error,
                   (rt_packet.unknown_packet_type, False, None, None),
                   (rt_packet.packet_truncated, False, None, None),
+                  (rt_packet.buffer_space_timeout, False,
+                        cri.chan_sel[16:], self.buffer_space_timeout_dest.status),
                   (underflow, True, underflow_error_cri, underflow_error_csr),
                   (overflow, True, None, None)
         )
 
         error_csr(self.rtio_error,
-                  (outputs.sequence_error, False,
-                   outputs.sequence_error_channel, self.sequence_error_channel.status),
-                  (outputs.collision, False,
-                   outputs.collision_channel, self.collision_channel.status),
-                  (outputs.busy, False,
-                   outputs.busy_channel, self.busy_channel.status)
+                  (async_errors.sequence_error, False,
+                   async_errors.sequence_error_channel, self.sequence_error_channel.status),
+                  (async_errors.collision, False,
+                   async_errors.collision_channel, self.collision_channel.status),
+                  (async_errors.busy, False,
+                   async_errors.busy_channel, self.busy_channel.status)
         )

artiq/gateware/drtio/rt_packet_master.py

@@ -3,70 +3,19 @@
 from migen import *
 from migen.genlib.fsm import *
 from migen.genlib.fifo import AsyncFIFO
-from migen.genlib.cdc import PulseSynchronizer
 
 from artiq.gateware.rtio.cdc import GrayCodeTransfer, BlindTransfer
+from artiq.gateware.drtio.cdc import CrossDomainRequest, CrossDomainNotification
 from artiq.gateware.drtio.rt_serializer import *
 
 
-class _CrossDomainRequest(Module):
-    def __init__(self, domain,
-                 req_stb, req_ack, req_data,
-                 srv_stb, srv_ack, srv_data):
-        dsync = getattr(self.sync, domain)
-
-        request = PulseSynchronizer("sys", domain)
-        reply = PulseSynchronizer(domain, "sys")
-        self.submodules += request, reply
-
-        ongoing = Signal()
-        self.comb += request.i.eq(~ongoing & req_stb)
-        self.sync += [
-            req_ack.eq(reply.o),
-            If(req_stb, ongoing.eq(1)),
-            If(req_ack, ongoing.eq(0))
-        ]
-        if req_data is not None:
-            req_data_r = Signal.like(req_data)
-            req_data_r.attr.add("no_retiming")
-            self.sync += If(req_stb, req_data_r.eq(req_data))
-        dsync += [
-            If(request.o, srv_stb.eq(1)),
-            If(srv_ack, srv_stb.eq(0))
-        ]
-        if req_data is not None:
-            dsync += If(request.o, srv_data.eq(req_data_r))
-        self.comb += reply.i.eq(srv_stb & srv_ack)
-
-
-class _CrossDomainNotification(Module):
-    def __init__(self, domain,
-                 emi_stb, emi_data,
-                 rec_stb, rec_ack, rec_data):
-        emi_data_r = Signal(len(emi_data))
-        emi_data_r.attr.add("no_retiming")
-        dsync = getattr(self.sync, domain)
-        dsync += If(emi_stb, emi_data_r.eq(emi_data))
-
-        ps = PulseSynchronizer(domain, "sys")
-        self.submodules += ps
-        self.comb += ps.i.eq(emi_stb)
-        self.sync += [
-            If(rec_ack, rec_stb.eq(0)),
-            If(ps.o,
-                rec_data.eq(emi_data_r),
-                rec_stb.eq(1)
-            )
-        ]
-
-
 class RTPacketMaster(Module):
     def __init__(self, link_layer, sr_fifo_depth=4):
         # all interface signals in sys domain unless otherwise specified
 
         # standard request interface
         #
-        # notwrite=1 address=0  buffer space request
+        # notwrite=1 address=0  buffer space request <destination>
         # notwrite=1 address=1  read request <channel, timestamp>
         #
         # optimized for write throughput
@@ -77,7 +26,7 @@ class RTPacketMaster(Module):
         self.sr_ack = Signal()
         self.sr_notwrite = Signal()
         self.sr_timestamp = Signal(64)
-        self.sr_channel = Signal(16)
+        self.sr_chan_sel = Signal(24)
         self.sr_address = Signal(16)
         self.sr_data = Signal(512)
 
@@ -135,20 +84,20 @@ class RTPacketMaster(Module):
         self.submodules += rx_dp
 
         # Write FIFO and extra data count
-        sr_fifo = ClockDomainsRenamer({"write": "sys_with_rst", "read": "rtio_with_rst"})(
-            AsyncFIFO(1+64+16+16+512, sr_fifo_depth))
+        sr_fifo = ClockDomainsRenamer({"write": "sys", "read": "rtio"})(
+            AsyncFIFO(1+64+24+16+512, sr_fifo_depth))
         self.submodules += sr_fifo
         sr_notwrite_d = Signal()
         sr_timestamp_d = Signal(64)
-        sr_channel_d = Signal(16)
+        sr_chan_sel_d = Signal(24)
         sr_address_d = Signal(16)
         sr_data_d = Signal(512)
         self.comb += [
             sr_fifo.we.eq(self.sr_stb),
             self.sr_ack.eq(sr_fifo.writable),
-            sr_fifo.din.eq(Cat(self.sr_notwrite, self.sr_timestamp, self.sr_channel,
+            sr_fifo.din.eq(Cat(self.sr_notwrite, self.sr_timestamp, self.sr_chan_sel,
                                self.sr_address, self.sr_data)),
-            Cat(sr_notwrite_d, sr_timestamp_d, sr_channel_d,
+            Cat(sr_notwrite_d, sr_timestamp_d, sr_chan_sel_d,
                 sr_address_d, sr_data_d).eq(sr_fifo.dout)
         ]
 
@@ -165,7 +114,7 @@ class RTPacketMaster(Module):
 
         sr_notwrite = Signal()
         sr_timestamp = Signal(64)
-        sr_channel = Signal(16)
+        sr_chan_sel = Signal(24)
         sr_address = Signal(16)
         sr_extra_data_cnt = Signal(8)
         sr_data = Signal(512)
@@ -173,7 +122,7 @@ class RTPacketMaster(Module):
         self.sync.rtio += If(sr_fifo.re,
             sr_notwrite.eq(sr_notwrite_d),
             sr_timestamp.eq(sr_timestamp_d),
-            sr_channel.eq(sr_channel_d),
+            sr_chan_sel.eq(sr_chan_sel_d),
             sr_address.eq(sr_address_d),
             sr_data.eq(sr_data_d))
 
@@ -206,19 +155,19 @@ class RTPacketMaster(Module):
         # CDC
         buffer_space_not = Signal()
         buffer_space = Signal(16)
-        self.submodules += _CrossDomainNotification("rtio_rx",
+        self.submodules += CrossDomainNotification("rtio_rx", "sys",
             buffer_space_not, buffer_space,
             self.buffer_space_not, self.buffer_space_not_ack, self.buffer_space)
 
         set_time_stb = Signal()
         set_time_ack = Signal()
-        self.submodules += _CrossDomainRequest("rtio",
+        self.submodules += CrossDomainRequest("rtio",
             self.set_time_stb, self.set_time_ack, None,
             set_time_stb, set_time_ack, None)
 
         echo_stb = Signal()
         echo_ack = Signal()
-        self.submodules += _CrossDomainRequest("rtio",
+        self.submodules += CrossDomainRequest("rtio",
             self.echo_stb, self.echo_ack, None,
             echo_stb, echo_ack, None)
 
@@ -227,7 +176,7 @@ class RTPacketMaster(Module):
         read_is_overflow = Signal()
         read_data = Signal(32)
         read_timestamp = Signal(64)
-        self.submodules += _CrossDomainNotification("rtio_rx",
+        self.submodules += CrossDomainNotification("rtio_rx", "sys",
             read_not,
             Cat(read_no_event, read_is_overflow, read_data, read_timestamp),
 
@@ -259,6 +208,10 @@ class RTPacketMaster(Module):
         self.sync.rtio += If(tsc_value_load, tsc_value.eq(self.tsc_value))
 
         tx_fsm.act("IDLE",
+            # Ensure 2 cycles between frames on the link.
+            NextState("READY")
+        )
+        tx_fsm.act("READY",
             If(sr_buf_readable,
                 If(sr_notwrite,
                     Case(sr_address[0], {
@@ -281,7 +234,7 @@ class RTPacketMaster(Module):
         tx_fsm.act("WRITE",
             tx_dp.send("write",
                 timestamp=sr_timestamp,
-                channel=sr_channel,
+                chan_sel=sr_chan_sel,
                 address=sr_address,
                 extra_data_cnt=sr_extra_data_cnt,
                 short_data=sr_data[:short_data_len]),
@@ -303,14 +256,14 @@ class RTPacketMaster(Module):
             )
         )
         tx_fsm.act("BUFFER_SPACE",
-            tx_dp.send("buffer_space_request"),
+            tx_dp.send("buffer_space_request", destination=sr_chan_sel[16:]),
             If(tx_dp.packet_last,
                 sr_buf_re.eq(1),
                 NextState("IDLE")
             )
         )
         tx_fsm.act("READ",
-            tx_dp.send("read_request", channel=sr_channel, timeout=sr_timestamp),
+            tx_dp.send("read_request", chan_sel=sr_chan_sel, timeout=sr_timestamp),
             If(tx_dp.packet_last,
                 sr_buf_re.eq(1),
                 NextState("IDLE")

artiq/gateware/drtio/rt_packet_repeater.py

@@ -0,0 +1,330 @@
+from migen import *
+from migen.genlib.fsm import *
+from migen.genlib.misc import WaitTimer
+
+
+from artiq.gateware.rtio import cri
+from artiq.gateware.drtio.cdc import CrossDomainNotification
+from artiq.gateware.drtio.rt_serializer import *
+
+
+class RTPacketRepeater(Module):
+    def __init__(self, tsc, link_layer):
+        # in rtio domain
+        self.reset = Signal()
+
+        # CRI target interface in rtio domain
+        self.cri = cri.Interface()
+
+        # in rtio_rx domain
+        self.err_unknown_packet_type = Signal()
+        self.err_packet_truncated = Signal()
+
+        # in rtio domain
+        self.err_command_missed = Signal()
+        self.command_missed_cmd = Signal(2)
+        self.command_missed_chan_sel = Signal(24)
+        self.err_buffer_space_timeout = Signal()
+        self.buffer_space_destination = Signal(8)
+
+        # set_time interface, in rtio domain
+        self.set_time_stb = Signal()
+        self.set_time_ack = Signal()
+
+        # # #
+
+        # RX/TX datapath
+        assert len(link_layer.tx_rt_data) == len(link_layer.rx_rt_data)
+        assert len(link_layer.tx_rt_data) % 8 == 0
+        ws = len(link_layer.tx_rt_data)
+        tx_plm = get_m2s_layouts(ws)
+        tx_dp = ClockDomainsRenamer("rtio")(TransmitDatapath(
+            link_layer.tx_rt_frame, link_layer.tx_rt_data, tx_plm))
+        self.submodules += tx_dp
+        rx_plm = get_s2m_layouts(ws)
+        rx_dp = ClockDomainsRenamer("rtio_rx")(ReceiveDatapath(
+            link_layer.rx_rt_frame, link_layer.rx_rt_data, rx_plm))
+        self.submodules += rx_dp
+
+        # TSC sync
+        tsc_value = Signal(64)
+        tsc_value_load = Signal()
+        self.sync.rtio += If(tsc_value_load, tsc_value.eq(tsc.coarse_ts))
+
+        # CRI buffer stage 1
+        cb0_loaded = Signal()
+        cb0_ack = Signal()
+
+        cb0_cmd = Signal(2)
+        cb0_timestamp = Signal(64)
+        cb0_chan_sel = Signal(24)
+        cb0_o_address = Signal(16)
+        cb0_o_data = Signal(512)
+        self.sync.rtio += [
+            If(self.reset | cb0_ack,
+                cb0_loaded.eq(0),
+                cb0_cmd.eq(cri.commands["nop"])
+            ),
+            If(~self.reset & ~cb0_loaded & (self.cri.cmd != cri.commands["nop"]),
+                cb0_loaded.eq(1),
+                cb0_cmd.eq(self.cri.cmd),
+                cb0_timestamp.eq(self.cri.timestamp),
+                cb0_chan_sel.eq(self.cri.chan_sel),
+                cb0_o_address.eq(self.cri.o_address),
+                cb0_o_data.eq(self.cri.o_data)
+            ),
+            self.err_command_missed.eq(cb0_loaded & (self.cri.cmd != cri.commands["nop"])),
+            self.command_missed_chan_sel.eq(self.cri.chan_sel),
+            self.command_missed_cmd.eq(self.cri.cmd)
+        ]
+
+        # CRI buffer stage 2 and write data slicer
+        cb_loaded = Signal()
+        cb_ack = Signal()
+
+        cb_cmd = Signal(2)
+        cb_timestamp = Signal(64)
+        cb_chan_sel = Signal(24)
+        cb_o_address = Signal(16)
+        cb_o_data = Signal(512)
+        self.sync.rtio += [
+            If(self.reset | cb_ack,
+                cb_loaded.eq(0),
+                cb_cmd.eq(cri.commands["nop"])
+            ),
+            If(~self.reset & ~cb_loaded & cb0_loaded,
+                cb_loaded.eq(1),
+                cb_cmd.eq(cb0_cmd),
+                cb_timestamp.eq(cb0_timestamp),
+                cb_chan_sel.eq(cb0_chan_sel),
+                cb_o_address.eq(cb0_o_address),
+                cb_o_data.eq(cb0_o_data)
+            )
+        ]
+        self.comb += cb0_ack.eq(~cb_loaded)
+
+        wb_extra_data_cnt = Signal(8)
+        short_data_len = tx_plm.field_length("write", "short_data")
+        wb_extra_data_a = Signal(512)
+        self.comb += wb_extra_data_a.eq(self.cri.o_data[short_data_len:])
+        for i in range(512//ws):
+            self.sync.rtio += If(self.cri.cmd == cri.commands["write"],
+                If(wb_extra_data_a[ws*i:ws*(i+1)] != 0, wb_extra_data_cnt.eq(i+1)))
+
+        wb_extra_data = Signal(512)
+        self.sync.rtio += If(self.cri.cmd == cri.commands["write"],
+            wb_extra_data.eq(wb_extra_data_a))
+
+        extra_data_ce = Signal()
+        extra_data_last = Signal()
+        extra_data_counter = Signal(max=512//ws+1)
+        self.comb += [
+            Case(extra_data_counter, 
+                {i+1: tx_dp.raw_data.eq(wb_extra_data[i*ws:(i+1)*ws])
+                 for i in range(512//ws)}),
+            extra_data_last.eq(extra_data_counter == wb_extra_data_cnt)
+        ]
+        self.sync.rtio += \
+            If(extra_data_ce,
+                extra_data_counter.eq(extra_data_counter + 1),
+            ).Else(
+                extra_data_counter.eq(1)
+            )
+
+        # Buffer space
+        self.sync.rtio += If(self.cri.cmd == cri.commands["get_buffer_space"],
+            self.buffer_space_destination.eq(self.cri.chan_sel[16:]))
+
+        rx_buffer_space_not = Signal()
+        rx_buffer_space = Signal(16)
+        buffer_space_not = Signal()
+        buffer_space_not_ack = Signal()
+        self.submodules += CrossDomainNotification("rtio_rx", "rtio",
+            rx_buffer_space_not, rx_buffer_space,
+            buffer_space_not, buffer_space_not_ack,
+            self.cri.o_buffer_space)
+
+        timeout_counter = ClockDomainsRenamer("rtio")(WaitTimer(8191))
+        self.submodules += timeout_counter
+
+        # Read
+        read_not = Signal()
+        read_no_event = Signal()
+        read_is_overflow = Signal()
+        read_data = Signal(32)
+        read_timestamp = Signal(64)
+        rtio_read_not = Signal()
+        rtio_read_not_ack = Signal()
+        rtio_read_no_event = Signal()
+        rtio_read_is_overflow = Signal()
+        rtio_read_data = Signal(32)
+        rtio_read_timestamp = Signal(64)
+        self.submodules += CrossDomainNotification("rtio_rx", "rtio",
+            read_not,
+            Cat(read_no_event, read_is_overflow, read_data, read_timestamp),
+
+            rtio_read_not, rtio_read_not_ack,
+            Cat(rtio_read_no_event, rtio_read_is_overflow,
+                rtio_read_data, rtio_read_timestamp))
+        self.comb += [
+            read_is_overflow.eq(rx_dp.packet_as["read_reply_noevent"].overflow),
+            read_data.eq(rx_dp.packet_as["read_reply"].data),
+            read_timestamp.eq(rx_dp.packet_as["read_reply"].timestamp)
+        ]
+
+        # input status
+        i_status_wait_event = Signal()
+        i_status_overflow = Signal()
+        self.comb += self.cri.i_status.eq(Cat(
+            i_status_wait_event, i_status_overflow, cb0_loaded | cb_loaded))
+
+        load_read_reply = Signal()
+        self.sync.rtio += [
+            If(load_read_reply,
+                i_status_wait_event.eq(0),
+                i_status_overflow.eq(0),
+                If(rtio_read_no_event,
+                    If(rtio_read_is_overflow,
+                        i_status_overflow.eq(1)
+                    ).Else(
+                        i_status_wait_event.eq(1)
+                    )
+                ),
+                self.cri.i_data.eq(rtio_read_data),
+                self.cri.i_timestamp.eq(rtio_read_timestamp)
+            )
+        ]
+
+        # TX and CRI FSM
+        tx_fsm = ClockDomainsRenamer("rtio")(FSM(reset_state="IDLE"))
+        self.submodules += tx_fsm
+
+        tx_fsm.act("IDLE",
+            # Ensure 2 cycles between frames on the link.
+            NextState("READY")
+        )
+        tx_fsm.act("READY",
+            If(self.set_time_stb,
+                tsc_value_load.eq(1),
+                NextState("SET_TIME")
+            ).Else(
+                If(cb_cmd == cri.commands["write"], NextState("WRITE")),
+                If(cb_cmd == cri.commands["get_buffer_space"], NextState("BUFFER_SPACE")),
+                If(cb_cmd == cri.commands["read"], NextState("READ"))
+            )
+        )
+
+        tx_fsm.act("SET_TIME",
+            tx_dp.send("set_time", timestamp=tsc_value),
+            If(tx_dp.packet_last,
+                self.set_time_ack.eq(1),
+                NextState("IDLE")
+            )
+        )
+
+        tx_fsm.act("WRITE",
+            tx_dp.send("write",
+                timestamp=cb_timestamp,
+                chan_sel=cb_chan_sel,
+                address=cb_o_address,
+                extra_data_cnt=wb_extra_data_cnt,
+                short_data=cb_o_data[:short_data_len]),
+            If(tx_dp.packet_last,
+                If(wb_extra_data_cnt == 0,
+                    cb_ack.eq(1),
+                    NextState("IDLE")
+                ).Else(
+                    NextState("WRITE_EXTRA")
+                )
+            )
+        )
+        tx_fsm.act("WRITE_EXTRA",
+            tx_dp.raw_stb.eq(1),
+            extra_data_ce.eq(1),
+            If(extra_data_last,
+                cb_ack.eq(1),
+                NextState("IDLE")
+            )
+        )
+
+        tx_fsm.act("BUFFER_SPACE",
+            tx_dp.send("buffer_space_request", destination=self.buffer_space_destination),
+            If(tx_dp.packet_last,
+                buffer_space_not_ack.eq(1),
+                NextState("GET_BUFFER_SPACE_REPLY")
+            )
+        )
+        tx_fsm.act("GET_BUFFER_SPACE_REPLY",
+            timeout_counter.wait.eq(1),
+            If(timeout_counter.done,
+                self.err_buffer_space_timeout.eq(1),
+                cb_ack.eq(1),
+                NextState("READY")
+            ).Else(
+                If(buffer_space_not,
+                    self.cri.o_buffer_space_valid.eq(1),
+                    cb_ack.eq(1),
+                    NextState("READY")
+                ),
+            )
+        )
+
+        tx_fsm.act("READ",
+            tx_dp.send("read_request",
+                chan_sel=cb_chan_sel,
+                timeout=cb_timestamp),
+            rtio_read_not_ack.eq(1),
+            If(tx_dp.packet_last,
+                NextState("GET_READ_REPLY")
+            )
+        )
+        tx_fsm.act("GET_READ_REPLY",
+            rtio_read_not_ack.eq(1),
+            If(self.reset | rtio_read_not,
+                load_read_reply.eq(1),
+                cb_ack.eq(1),
+                NextState("READY")
+            )
+        )
+
+        # RX FSM
+        rx_fsm = ClockDomainsRenamer("rtio_rx")(FSM(reset_state="INPUT"))
+        self.submodules += rx_fsm
+
+        ongoing_packet_next = Signal()
+        ongoing_packet = Signal()
+        self.sync.rtio_rx += ongoing_packet.eq(ongoing_packet_next)
+
+        rx_fsm.act("INPUT",
+            If(rx_dp.frame_r,
+                rx_dp.packet_buffer_load.eq(1),
+                If(rx_dp.packet_last,
+                    Case(rx_dp.packet_type, {
+                        rx_plm.types["buffer_space_reply"]: NextState("BUFFER_SPACE"),
+                        rx_plm.types["read_reply"]: NextState("READ_REPLY"),
+                        rx_plm.types["read_reply_noevent"]: NextState("READ_REPLY_NOEVENT"),
+                        "default": self.err_unknown_packet_type.eq(1)
+                    })
+                ).Else(
+                    ongoing_packet_next.eq(1)
+                )
+            ),
+            If(~rx_dp.frame_r & ongoing_packet,
+                self.err_packet_truncated.eq(1)
+            )
+        )
+        rx_fsm.act("BUFFER_SPACE",
+            rx_buffer_space_not.eq(1),
+            rx_buffer_space.eq(rx_dp.packet_as["buffer_space_reply"].space),
+            NextState("INPUT")
+        )
+        rx_fsm.act("READ_REPLY",
+            read_not.eq(1),
+            read_no_event.eq(0),
+            NextState("INPUT")
+        )
+        rx_fsm.act("READ_REPLY_NOEVENT",
+            read_not.eq(1),
+            read_no_event.eq(1),
+            NextState("INPUT")
+        )

artiq/gateware/drtio/rt_packet_satellite.py

@@ -2,22 +2,26 @@
 
 from migen import *
 from migen.genlib.fsm import *
+from migen.genlib.misc import WaitTimer
 
 from artiq.gateware.rtio import cri
 from artiq.gateware.drtio.rt_serializer import *
 
 
 class RTPacketSatellite(Module):
-    def __init__(self, link_layer):
+    def __init__(self, link_layer, interface=None):
         self.reset = Signal()
 
         self.unknown_packet_type = Signal()
         self.packet_truncated = Signal()
+        self.buffer_space_timeout = Signal()
 
         self.tsc_load = Signal()
         self.tsc_load_value = Signal(64)
 
-        self.cri = cri.Interface()
+        if interface is None:
+            interface = cri.Interface()
+        self.cri = interface
 
         # # #
 
@@ -75,18 +79,25 @@ class RTPacketSatellite(Module):
         ]
 
         # RX FSM
-        read = Signal()
+        cri_read = Signal()
+        cri_buffer_space = Signal()
         self.comb += [
             self.tsc_load_value.eq(
                 rx_dp.packet_as["set_time"].timestamp),
-            If(load_read_request | read_request_pending,
+            If(cri_read | read_request_pending,
                 self.cri.chan_sel.eq(
-                    rx_dp.packet_as["read_request"].channel),
+                    rx_dp.packet_as["read_request"].chan_sel),
+            ).Elif(cri_buffer_space,
+                self.cri.chan_sel.eq(
+                    rx_dp.packet_as["buffer_space_request"].destination << 16)
+            ).Else(
+                self.cri.chan_sel.eq(
+                    rx_dp.packet_as["write"].chan_sel),
+            ),
+            If(cri_read | read_request_pending,
                 self.cri.timestamp.eq(
                     rx_dp.packet_as["read_request"].timeout)
             ).Else(
-                self.cri.chan_sel.eq(
-                    rx_dp.packet_as["write"].channel),
                 self.cri.timestamp.eq(
                     rx_dp.packet_as["write"].timestamp)
             ),
@@ -103,6 +114,9 @@ class RTPacketSatellite(Module):
         ongoing_packet = Signal()
         self.sync += ongoing_packet.eq(ongoing_packet_next)
 
+        timeout_counter = WaitTimer(8191)
+        self.submodules += timeout_counter
+
         rx_fsm.act("INPUT",
             If(rx_dp.frame_r,
                 rx_dp.packet_buffer_load.eq(1),
@@ -113,7 +127,7 @@ class RTPacketSatellite(Module):
                         rx_plm.types["echo_request"]: echo_req.eq(1),
                         rx_plm.types["set_time"]: NextState("SET_TIME"),
                         rx_plm.types["write"]: NextState("WRITE"),
-                        rx_plm.types["buffer_space_request"]: NextState("BUFFER_SPACE"),
+                        rx_plm.types["buffer_space_request"]: NextState("BUFFER_SPACE_REQUEST"),
                         rx_plm.types["read_request"]: NextState("READ_REQUEST"),
                         "default": self.unknown_packet_type.eq(1)
                     })
@@ -130,10 +144,10 @@ class RTPacketSatellite(Module):
             NextState("INPUT")
         )
 
+        # CRI mux defaults to write information
         rx_fsm.act("WRITE",
             If(write_data_buffer_cnt == rx_dp.packet_as["write"].extra_data_cnt,
-                self.cri.cmd.eq(cri.commands["write"]),
-                NextState("INPUT")
+                NextState("WRITE_CMD")
             ).Else(
                 write_data_buffer_load.eq(1),
                 If(~rx_dp.frame_r,
@@ -142,14 +156,40 @@ class RTPacketSatellite(Module):
                 )
             )
         )
-        rx_fsm.act("BUFFER_SPACE",
-            buffer_space_set.eq(1),
-            buffer_space_update.eq(1),
+        rx_fsm.act("WRITE_CMD",
+            self.cri.cmd.eq(cri.commands["write"]),
             NextState("INPUT")
         )
 
+        rx_fsm.act("BUFFER_SPACE_REQUEST",
+            cri_buffer_space.eq(1),
+            NextState("BUFFER_SPACE_REQUEST_CMD")
+        )
+        rx_fsm.act("BUFFER_SPACE_REQUEST_CMD",
+            cri_buffer_space.eq(1),
+            self.cri.cmd.eq(cri.commands["get_buffer_space"]),
+            NextState("BUFFER_SPACE")
+        )
+        rx_fsm.act("BUFFER_SPACE",
+            cri_buffer_space.eq(1),
+            timeout_counter.wait.eq(1),
+            If(timeout_counter.done,
+                self.buffer_space_timeout.eq(1),
+                NextState("INPUT")
+            ).Elif(self.cri.o_buffer_space_valid,
+                buffer_space_set.eq(1),
+                buffer_space_update.eq(1),
+                NextState("INPUT")
+            )
+        )
+
         rx_fsm.act("READ_REQUEST",
+            cri_read.eq(1),
+            NextState("READ_REQUEST_CMD")
+        )
+        rx_fsm.act("READ_REQUEST_CMD",
             load_read_request.eq(1),
+            cri_read.eq(1),
             self.cri.cmd.eq(cri.commands["read"]),
             NextState("INPUT")
         )
@@ -187,16 +227,14 @@ class RTPacketSatellite(Module):
         tx_fsm.act("READ_OVERFLOW",
             tx_dp.send("read_reply_noevent", overflow=1),
             clear_read_request.eq(1),
-            If(tx_dp.packet_last,
-                NextState("IDLE")
-            )
+            If(tx_dp.packet_last, NextState("IDLE"))
         )
         tx_fsm.act("READ",
             tx_dp.send("read_reply",
                        timestamp=self.cri.i_timestamp,
                        data=self.cri.i_data),
-            clear_read_request.eq(1),
             If(tx_dp.packet_last,
+                clear_read_request.eq(1),
                 NextState("IDLE")
             )
         )

artiq/gateware/drtio/rt_serializer.py

@@ -49,13 +49,13 @@ def get_m2s_layouts(alignment):
     plm.add_type("set_time", ("timestamp", 64))
 
     plm.add_type("write", ("timestamp", 64),
-                          ("channel", 16),
+                          ("chan_sel", 24),
                           ("address", 16),
                           ("extra_data_cnt", 8),
                           ("short_data", short_data_len))
-    plm.add_type("buffer_space_request")
+    plm.add_type("buffer_space_request", ("destination", 8))
 
-    plm.add_type("read_request", ("channel", 16), ("timeout", 64))
+    plm.add_type("read_request", ("chan_sel", 24), ("timeout", 64))
 
     return plm
 

artiq/gateware/rtio/__init__.py

@@ -1,4 +1,5 @@
-from artiq.gateware.rtio.cri import KernelInitiator, CRIInterconnectShared
+from artiq.gateware.rtio.tsc import TSC
+from artiq.gateware.rtio.cri import KernelInitiator, CRIInterconnectShared, RoutingTableAccess
 from artiq.gateware.rtio.channel import Channel, LogChannel
 from artiq.gateware.rtio.core import Core
 from artiq.gateware.rtio.analyzer import Analyzer

artiq/gateware/rtio/analyzer.py

@@ -43,7 +43,7 @@ assert layout_len(stopped_layout) == message_len
 
 
 class MessageEncoder(Module, AutoCSR):
-    def __init__(self, cri, enable):
+    def __init__(self, tsc, cri, enable):
         self.source = stream.Endpoint([("data", message_len)])
 
         self.overflow = CSRStatus()
@@ -67,7 +67,7 @@ class MessageEncoder(Module, AutoCSR):
         self.comb += [
             input_output.channel.eq(cri.chan_sel),
             input_output.address_padding.eq(cri.o_address),
-            input_output.rtio_counter.eq(cri.counter),
+            input_output.rtio_counter.eq(tsc.full_ts_cri),
             If(cri.cmd == cri_commands["write"],
                 input_output.message_type.eq(MessageType.output.value),
                 input_output.timestamp.eq(cri.timestamp),
@@ -85,7 +85,7 @@ class MessageEncoder(Module, AutoCSR):
         self.comb += [
             exception.message_type.eq(MessageType.exception.value),
             exception.channel.eq(cri.chan_sel),
-            exception.rtio_counter.eq(cri.counter),
+            exception.rtio_counter.eq(tsc.full_ts_cri),
         ]
         just_written = Signal()
         self.sync += just_written.eq(cri.cmd == cri_commands["write"])
@@ -103,7 +103,7 @@ class MessageEncoder(Module, AutoCSR):
         stopped = Record(stopped_layout)
         self.comb += [
             stopped.message_type.eq(MessageType.stopped.value),
-            stopped.rtio_counter.eq(cri.counter),
+            stopped.rtio_counter.eq(tsc.full_ts_cri),
         ]
 
         enable_r = Signal()
@@ -193,13 +193,13 @@ class DMAWriter(Module, AutoCSR):
 
 
 class Analyzer(Module, AutoCSR):
-    def __init__(self, cri, membus, fifo_depth=128):
+    def __init__(self, tsc, cri, membus, fifo_depth=128):
         # shutdown procedure: set enable to 0, wait until busy=0
         self.enable = CSRStorage()
         self.busy = CSRStatus()
 
         self.submodules.message_encoder = MessageEncoder(
-            cri, self.enable.storage)
+            tsc, cri, self.enable.storage)
         self.submodules.fifo = stream.SyncFIFO(
             [("data", message_len)], fifo_depth, True)
         self.submodules.converter = stream.Converter(

artiq/gateware/rtio/core.py

@@ -14,8 +14,7 @@ from artiq.gateware.rtio.input_collector import *
 
 
 class Core(Module, AutoCSR):
-    def __init__(self, channels, lane_count=8, fifo_depth=128,
-                 glbl_fine_ts_width=None):
+    def __init__(self, tsc, channels, lane_count=8, fifo_depth=128):
         self.cri = cri.Interface()
         self.reset = CSR()
         self.reset_phy = CSR()
@@ -61,36 +60,23 @@ class Core(Module, AutoCSR):
                                      for channel in channels),
                                  max(rtlink.get_fine_ts_width(channel.interface.i)
                                      for channel in channels))
-        if glbl_fine_ts_width is None:
-            glbl_fine_ts_width = chan_fine_ts_width
-        assert glbl_fine_ts_width >= chan_fine_ts_width
-
-        coarse_ts = Signal(64-glbl_fine_ts_width)
-        self.sync.rtio += coarse_ts.eq(coarse_ts + 1)
-        coarse_ts_cdc = GrayCodeTransfer(len(coarse_ts))  # from rtio to sys
-        self.submodules += coarse_ts_cdc
-        self.comb += [
-            coarse_ts_cdc.i.eq(coarse_ts),
-            self.cri.counter.eq(coarse_ts_cdc.o << glbl_fine_ts_width)
-        ]
-        self.coarse_ts = coarse_ts
+        assert tsc.glbl_fine_ts_width >= chan_fine_ts_width
 
         # Outputs/Inputs
         quash_channels = [n for n, c in enumerate(channels) if isinstance(c, LogChannel)]
 
-        outputs = SED(channels, glbl_fine_ts_width, "async",
+        outputs = SED(channels, tsc.glbl_fine_ts_width, "async",
             quash_channels=quash_channels,
             lane_count=lane_count, fifo_depth=fifo_depth,
             interface=self.cri)
         self.submodules += outputs
-        self.comb += outputs.coarse_timestamp.eq(coarse_ts)
-        self.sync += outputs.minimum_coarse_timestamp.eq(coarse_ts_cdc.o + 16)
+        self.comb += outputs.coarse_timestamp.eq(tsc.coarse_ts)
+        self.sync += outputs.minimum_coarse_timestamp.eq(tsc.coarse_ts_sys + 16)
 
-        inputs = InputCollector(channels, glbl_fine_ts_width, "async",
+        inputs = InputCollector(tsc, channels, "async",
             quash_channels=quash_channels,
             interface=self.cri)
         self.submodules += inputs
-        self.comb += inputs.coarse_timestamp.eq(coarse_ts)
 
         # Asychronous output errors
         o_collision_sync = BlindTransfer(data_width=16)

artiq/gateware/rtio/cri.py

@@ -2,6 +2,7 @@
 
 from migen import *
 from migen.genlib.record import *
+from migen.genlib.cdc import MultiReg
 
 from misoc.interconnect.csr import *
 
@@ -13,41 +14,42 @@ from misoc.interconnect.csr import *
 
 commands = {
     "nop": 0,
-
     "write": 1,
     # i_status should have the "wait for status" bit set until
     # an event is available, or timestamp is reached.
-    "read": 2
+    "read": 2,
+    # targets must assert o_buffer_space_valid in response
+    # to this command
+    "get_buffer_space": 3
 }
 
 
 layout = [
     ("cmd", 2, DIR_M_TO_S),
-    # 8 MSBs of chan_sel are used to select core
+    # 8  MSBs of chan_sel = routing destination
+    # 16 LSBs of chan_sel = channel within the destination
     ("chan_sel", 24, DIR_M_TO_S),
     ("timestamp", 64, DIR_M_TO_S),
 
     ("o_data", 512, DIR_M_TO_S),
     ("o_address", 16, DIR_M_TO_S),
     # o_status bits:
-    # <0:wait> <1:underflow> <2:link error>
+    # <0:wait> <1:underflow> <2:destination unreachable>
     ("o_status", 3, DIR_S_TO_M),
-    # targets may optionally report a pessimistic estimate of the number
-    # of outputs events that can be written without waiting.
+
+    # pessimistic estimate of the number of outputs events that can be
+    # written without waiting.
+    # this feature may be omitted on systems without DRTIO.
+    ("o_buffer_space_valid", 1, DIR_S_TO_M),
     ("o_buffer_space", 16, DIR_S_TO_M),
 
     ("i_data", 32, DIR_S_TO_M),
     ("i_timestamp", 64, DIR_S_TO_M),
     # i_status bits:
     # <0:wait for event (command timeout)> <1:overflow> <2:wait for status>
-    # <3:link error>
+    # <3:destination unreachable>
     # <0> and <1> are mutually exclusive. <1> has higher priority.
     ("i_status", 4, DIR_S_TO_M),
-
-    # value of the timestamp counter transferred into the CRI clock domain.
-    # monotonic, may lag behind the counter in the IO clock domain, but
-    # not be ahead of it.
-    ("counter", 64, DIR_S_TO_M)
 ]
 
 
@@ -57,8 +59,10 @@ class Interface(Record):
 
 
 class KernelInitiator(Module, AutoCSR):
-    def __init__(self, cri=None):
+    def __init__(self, tsc, cri=None):
         self.chan_sel = CSRStorage(24)
+        # monotonic, may lag behind the counter in the IO clock domain, but
+        # not be ahead of it.
         self.timestamp = CSRStorage(64)
 
         # Writing timestamp clears o_data. This implements automatic
@@ -103,11 +107,11 @@ class KernelInitiator(Module, AutoCSR):
             self.o_data.dat_w.eq(0),
             self.o_data.we.eq(self.timestamp.re),
         ]
-        self.sync += If(self.counter_update.re, self.counter.status.eq(self.cri.counter))
+        self.sync += If(self.counter_update.re, self.counter.status.eq(tsc.full_ts_cri))
 
 
-class CRIDecoder(Module):
-    def __init__(self, slaves=2, master=None):
+class CRIDecoder(Module, AutoCSR):
+    def __init__(self, slaves=2, master=None, mode="async", enable_routing=False):
         if isinstance(slaves, int):
             slaves = [Interface() for _ in range(slaves)]
         if master is None:
@@ -117,8 +121,37 @@ class CRIDecoder(Module):
 
         # # #
 
-        selected = Signal(8, reset_less=True)
-        self.sync += selected.eq(self.master.chan_sel[16:])
+        # routing
+        if enable_routing:
+            destination_unreachable = Interface()
+            self.comb += [
+                destination_unreachable.o_status.eq(4),
+                destination_unreachable.i_status.eq(8)
+            ]
+            slaves = slaves[:]
+            slaves.append(destination_unreachable)
+            target_len = 2**(len(slaves) - 1).bit_length()
+            slaves += [destination_unreachable]*(target_len - len(slaves))
+
+        slave_bits = bits_for(len(slaves)-1)
+        selected = Signal(slave_bits)
+
+        if enable_routing:
+            self.specials.routing_table = Memory(slave_bits, 256)
+
+            if mode == "async":
+                rtp_decoder = self.routing_table.get_port()
+            elif mode == "sync":
+                rtp_decoder = self.routing_table.get_port(clock_domain="rtio")
+            else:
+                raise ValueError
+            self.specials += rtp_decoder
+            self.comb += [
+                rtp_decoder.adr.eq(self.master.chan_sel[16:]),
+                selected.eq(rtp_decoder.dat_r)
+            ]
+        else:
+            self.sync += selected.eq(self.master.chan_sel[16:])
 
         # master -> slave
         for n, slave in enumerate(slaves):
@@ -138,7 +171,7 @@ class CRIDecoder(Module):
 
 
 class CRISwitch(Module, AutoCSR):
-    def __init__(self, masters=2, slave=None):
+    def __init__(self, masters=2, slave=None, mode="async"):
         if isinstance(masters, int):
             masters = [Interface() for _ in range(masters)]
         if slave is None:
@@ -150,6 +183,15 @@ class CRISwitch(Module, AutoCSR):
 
         # # #
 
+        if mode == "async":
+            selected = self.selected.storage
+        elif mode == "sync":
+            self.selected.storage.attr.add("no_retiming")
+            selected = Signal.like(self.selected.storage)
+            self.specials += MultiReg(self.selected.storage, selected, "rtio")
+        else:
+            raise ValueError
+
         if len(masters) == 1:
             self.comb += masters[0].connect(slave)
         else:
@@ -157,7 +199,7 @@ class CRISwitch(Module, AutoCSR):
             for name, size, direction in layout:
                 if direction == DIR_M_TO_S:
                     choices = Array(getattr(m, name) for m in masters)
-                    self.comb += getattr(slave, name).eq(choices[self.selected.storage])
+                    self.comb += getattr(slave, name).eq(choices[selected])
 
             # connect slave->master signals
             for name, size, direction in layout:
@@ -167,11 +209,31 @@ class CRISwitch(Module, AutoCSR):
                         dest = getattr(m, name)
                         self.comb += dest.eq(source)
 
+
 class CRIInterconnectShared(Module):
-    def __init__(self, masters=2, slaves=2):
+    def __init__(self, masters=2, slaves=2, mode="async", enable_routing=False):
         shared = Interface()
-        self.submodules.switch = CRISwitch(masters, shared)
-        self.submodules.decoder = CRIDecoder(slaves, shared)
+        self.submodules.switch = CRISwitch(masters, shared, mode)
+        self.submodules.decoder = CRIDecoder(slaves, shared, mode, enable_routing)
 
     def get_csrs(self):
-        return self.switch.get_csrs()
+        return self.switch.get_csrs() + self.decoder.get_csrs()
+
+
+class RoutingTableAccess(Module, AutoCSR):
+    def __init__(self, interconnect):
+        if isinstance(interconnect, CRIInterconnectShared):
+            interconnect = interconnect.decoder
+
+        rtp_csr = interconnect.routing_table.get_port(write_capable=True)
+        self.specials += rtp_csr
+
+        self.destination = CSRStorage(8)
+        self.hop = CSR(len(rtp_csr.dat_w))
+
+        self.comb += [
+            rtp_csr.adr.eq(self.destination.storage),
+            rtp_csr.dat_w.eq(self.hop.r),
+            rtp_csr.we.eq(self.hop.re),
+            self.hop.w.eq(rtp_csr.dat_r)
+        ]

artiq/gateware/rtio/input_collector.py

@@ -24,11 +24,10 @@ def get_channel_layout(coarse_ts_width, interface):
 
 
 class InputCollector(Module):
-    def __init__(self, channels, glbl_fine_ts_width, mode, quash_channels=[], interface=None):
+    def __init__(self, tsc, channels, mode, quash_channels=[], interface=None):
         if interface is None:
             interface = cri.Interface()
         self.cri = interface
-        self.coarse_timestamp = Signal(64 - glbl_fine_ts_width)
 
         # # #
 
@@ -55,7 +54,7 @@ class InputCollector(Module):
                 continue
 
             # FIFO
-            layout = get_channel_layout(len(self.coarse_timestamp), iif)
+            layout = get_channel_layout(len(tsc.coarse_ts), iif)
             fifo = fifo_factory(layout_len(layout), channel.ififo_depth)
             self.submodules += fifo
             fifo_in = Record(layout)
@@ -67,10 +66,10 @@ class InputCollector(Module):
 
             # FIFO write
             if iif.delay:
-                counter_rtio = Signal.like(self.coarse_timestamp, reset_less=True)
-                sync_io += counter_rtio.eq(self.coarse_timestamp - (iif.delay + 1))
+                counter_rtio = Signal.like(tsc.coarse_ts, reset_less=True)
+                sync_io += counter_rtio.eq(tsc.coarse_ts - (iif.delay + 1))
             else:
-                counter_rtio = self.coarse_timestamp
+                counter_rtio = tsc.coarse_ts
             if hasattr(fifo_in, "data"):
                 self.comb += fifo_in.data.eq(iif.data)
             if hasattr(fifo_in, "timestamp"):
@@ -130,7 +129,7 @@ class InputCollector(Module):
                 self.cri.i_data.eq(Array(i_datas)[sel]),
                 self.cri.i_timestamp.eq(Array(i_timestamps)[sel]),
             ),
-            If((self.cri.counter >= input_timeout) | (i_status_raw != 0),
+            If((tsc.full_ts_cri >= input_timeout) | (i_status_raw != 0),
                 If(input_pending, i_ack.eq(1)),
                 input_pending.eq(0)
             ),

artiq/gateware/rtio/sed/core.py

@@ -55,7 +55,10 @@ class SED(Module):
             self.comb += i.eq(o)
 
         if report_buffer_space:
-            self.comb += self.cri.o_buffer_space.eq(self.fifos.buffer_space)
+            self.comb += [
+                self.cri.o_buffer_space_valid.eq(1),
+                self.cri.o_buffer_space.eq(self.fifos.buffer_space)
+            ]
 
     @property
     def cri(self):

artiq/gateware/rtio/tsc.py

@@ -0,0 +1,48 @@
+from migen import *
+
+from artiq.gateware.rtio.cdc import GrayCodeTransfer
+
+
+class TSC(Module):
+    def __init__(self, mode, glbl_fine_ts_width=0):
+        self.glbl_fine_ts_width = glbl_fine_ts_width
+
+        # in rtio domain
+        self.coarse_ts = Signal(64 - glbl_fine_ts_width)
+        self.full_ts = Signal(64)
+        
+        # in sys domain
+        # monotonic, may lag behind the counter in the IO clock domain, but
+        # not be ahead of it.
+        self.coarse_ts_sys = Signal.like(self.coarse_ts)
+        self.full_ts_sys = Signal(64)
+
+        # in rtio domain
+        self.load = Signal()
+        self.load_value = Signal.like(self.coarse_ts)
+
+        if mode == "async":
+            self.full_ts_cri = self.full_ts_sys
+        elif mode == "sync":
+            self.full_ts_cri = self.full_ts
+        else:
+            raise ValueError
+
+        # # #
+
+        self.sync.rtio += If(self.load,
+                self.coarse_ts.eq(self.load_value)
+            ).Else(
+                self.coarse_ts.eq(self.coarse_ts + 1)
+            )
+        coarse_ts_cdc = GrayCodeTransfer(len(self.coarse_ts))  # from rtio to sys
+        self.submodules += coarse_ts_cdc
+        self.comb += [
+            coarse_ts_cdc.i.eq(self.coarse_ts),
+            self.coarse_ts_sys.eq(coarse_ts_cdc.o)
+        ]
+
+        self.comb += [
+            self.full_ts.eq(self.coarse_ts << glbl_fine_ts_width),
+            self.full_ts_sys.eq(self.coarse_ts_sys << glbl_fine_ts_width)
+        ]

artiq/gateware/targets/kasli.py

@@ -21,14 +21,14 @@ from artiq.gateware import eem
 from artiq.gateware.drtio.transceiver import gtp_7series
 from artiq.gateware.drtio.siphaser import SiPhaser7Series
 from artiq.gateware.drtio.rx_synchronizer import XilinxRXSynchronizer
-from artiq.gateware.drtio import DRTIOMaster, DRTIOSatellite
+from artiq.gateware.drtio import *
 from artiq.build_soc import *
 
 
 class _RTIOCRG(Module, AutoCSR):
     def __init__(self, platform):
-        self._pll_reset = CSRStorage(reset=1)
-        self._pll_locked = CSRStatus()
+        self.pll_reset = CSRStorage(reset=1)
+        self.pll_locked = CSRStatus()
         self.clock_domains.cd_rtio = ClockDomain()
         self.clock_domains.cd_rtiox4 = ClockDomain(reset_less=True)
 
@@ -60,7 +60,7 @@ class _RTIOCRG(Module, AutoCSR):
                      # VCO @ 1GHz when using 125MHz input
                      p_CLKFBOUT_MULT=8, p_DIVCLK_DIVIDE=1,
                      i_CLKFBIN=self.cd_rtio.clk,
-                     i_RST=self._pll_reset.storage,
+                     i_RST=self.pll_reset.storage,
 
                      o_CLKFBOUT=rtio_clk,
 
@@ -70,7 +70,7 @@ class _RTIOCRG(Module, AutoCSR):
             Instance("BUFG", i_I=rtiox4_clk, o_O=self.cd_rtiox4.clk),
 
             AsyncResetSynchronizer(self.cd_rtio, ~pll_locked),
-            MultiReg(pll_locked, self._pll_locked.status)
+            MultiReg(pll_locked, self.pll_locked.status)
         ]
 
 
@@ -120,9 +120,10 @@ class _StandaloneBase(MiniSoC, AMPSoC):
         self.submodules.rtio_crg = _RTIOCRG(self.platform)
         self.csr_devices.append("rtio_crg")
         fix_serdes_timing_path(self.platform)
-        self.submodules.rtio_core = rtio.Core(rtio_channels)
+        self.submodules.rtio_tsc = rtio.TSC("async", glbl_fine_ts_width=3)
+        self.submodules.rtio_core = rtio.Core(self.rtio_tsc, rtio_channels)
         self.csr_devices.append("rtio_core")
-        self.submodules.rtio = rtio.KernelInitiator()
+        self.submodules.rtio = rtio.KernelInitiator(self.rtio_tsc)
         self.submodules.rtio_dma = ClockDomainsRenamer("sys_kernel")(
             rtio.DMA(self.get_native_sdram_if()))
         self.register_kernel_cpu_csrdevice("rtio")
@@ -138,7 +139,7 @@ class _StandaloneBase(MiniSoC, AMPSoC):
             self.crg.cd_sys.clk,
             self.rtio_crg.cd_rtio.clk)
 
-        self.submodules.rtio_analyzer = rtio.Analyzer(self.rtio_core.cri,
+        self.submodules.rtio_analyzer = rtio.Analyzer(self.rtio_tsc, self.rtio_core.cri,
                                                       self.get_native_sdram_if())
         self.csr_devices.append("rtio_analyzer")
 
@@ -766,19 +767,23 @@ class Tester(_StandaloneBase):
             self.rtio_crg.cd_rtio.clk, self.grabber0.deserializer.cd_cl.clk)
 
 
-class _RTIOClockMultiplier(Module):
+class _RTIOClockMultiplier(Module, AutoCSR):
     def __init__(self, rtio_clk_freq):
+        self.pll_reset = CSRStorage(reset=1)
+        self.pll_locked = CSRStatus()
         self.clock_domains.cd_rtiox4 = ClockDomain(reset_less=True)
 
         # See "Global Clock Network Deskew Using Two BUFGs" in ug472.
         clkfbout = Signal()
         clkfbin = Signal()
         rtiox4_clk = Signal()
+        pll_locked = Signal()
         self.specials += [
             Instance("MMCME2_BASE",
                 p_CLKIN1_PERIOD=1e9/rtio_clk_freq,
                 i_CLKIN1=ClockSignal("rtio"),
-                i_RST=ResetSignal("rtio"),
+                i_RST=self.pll_reset.storage,
+                o_LOCKED=pll_locked,
 
                 p_CLKFBOUT_MULT_F=8.0, p_DIVCLK_DIVIDE=1,
 
@@ -787,7 +792,9 @@ class _RTIOClockMultiplier(Module):
                 p_CLKOUT0_DIVIDE_F=2.0, o_CLKOUT0=rtiox4_clk,
             ),
             Instance("BUFG", i_I=clkfbout, o_O=clkfbin),
-            Instance("BUFG", i_I=rtiox4_clk, o_O=self.cd_rtiox4.clk)
+            Instance("BUFG", i_I=rtiox4_clk, o_O=self.cd_rtiox4.clk),
+
+            MultiReg(pll_locked, self.pll_locked.status)
         ]
 
 
@@ -796,12 +803,12 @@ class _MasterBase(MiniSoC, AMPSoC):
         "cri_con":       0x10000000,
         "rtio":          0x20000000,
         "rtio_dma":      0x30000000,
-        "drtio_aux":     0x50000000,
+        "drtioaux":      0x50000000,
         "mailbox":       0x70000000
     }
     mem_map.update(MiniSoC.mem_map)
 
-    def __init__(self, rtio_clk_freq=150e6, **kwargs):
+    def __init__(self, rtio_clk_freq=125e6, **kwargs):
         MiniSoC.__init__(self,
                          cpu_type="or1k",
                          sdram_controller_type="minicon",
@@ -836,28 +843,40 @@ class _MasterBase(MiniSoC, AMPSoC):
         self.comb += [sfp_ctl.led.eq(channel.rx_ready)
             for sfp_ctl, channel in zip(sfp_ctls, self.drtio_transceiver.channels)]
 
+        self.submodules.rtio_tsc = rtio.TSC("async", glbl_fine_ts_width=3)
+
         drtio_csr_group = []
-        drtio_memory_group = []
+        drtioaux_csr_group = []
+        drtioaux_memory_group = []
         self.drtio_cri = []
         for i in range(2):
             core_name = "drtio" + str(i)
-            memory_name = "drtio" + str(i) + "_aux"
+            coreaux_name = "drtioaux" + str(i)
+            memory_name = "drtioaux" + str(i) + "_mem"
             drtio_csr_group.append(core_name)
-            drtio_memory_group.append(memory_name)
+            drtioaux_csr_group.append(coreaux_name)
+            drtioaux_memory_group.append(memory_name)
 
-            core = ClockDomainsRenamer({"rtio_rx": "rtio_rx" + str(i)})(
-                DRTIOMaster(self.drtio_transceiver.channels[i]))
+            cdr = ClockDomainsRenamer({"rtio_rx": "rtio_rx" + str(i)})
+
+            core = cdr(DRTIOMaster(self.rtio_tsc, self.drtio_transceiver.channels[i]))
             setattr(self.submodules, core_name, core)
             self.drtio_cri.append(core.cri)
             self.csr_devices.append(core_name)
 
-            memory_address = self.mem_map["drtio_aux"] + 0x800*i
+            coreaux = cdr(DRTIOAuxController(core.link_layer))
+            setattr(self.submodules, coreaux_name, coreaux)
+            self.csr_devices.append(coreaux_name)
+
+            memory_address = self.mem_map["drtioaux"] + 0x800*i
             self.add_wb_slave(memory_address, 0x800,
-                              core.aux_controller.bus)
+                              coreaux.bus)
             self.add_memory_region(memory_name, memory_address | self.shadow_base, 0x800)
         self.config["HAS_DRTIO"] = None
+        self.config["HAS_DRTIO_ROUTING"] = None
         self.add_csr_group("drtio", drtio_csr_group)
-        self.add_memory_group("drtio_aux", drtio_memory_group)
+        self.add_csr_group("drtioaux", drtioaux_csr_group)
+        self.add_memory_group("drtioaux_mem", drtioaux_memory_group)
 
         rtio_clk_period = 1e9/rtio_clk_freq
         gtp = self.drtio_transceiver.gtps[0]
@@ -871,27 +890,31 @@ class _MasterBase(MiniSoC, AMPSoC):
             platform.add_false_path_constraints(
                 self.crg.cd_sys.clk, gtp.rxoutclk)
 
-        self.submodules.rtio_clkmul = _RTIOClockMultiplier(rtio_clk_freq)
+        self.submodules.rtio_crg = _RTIOClockMultiplier(rtio_clk_freq)
+        self.csr_devices.append("rtio_crg")
         fix_serdes_timing_path(platform)
 
     def add_rtio(self, rtio_channels):
         self.submodules.rtio_moninj = rtio.MonInj(rtio_channels)
         self.csr_devices.append("rtio_moninj")
 
-        self.submodules.rtio_core = rtio.Core(rtio_channels, glbl_fine_ts_width=3)
+        self.submodules.rtio_core = rtio.Core(self.rtio_tsc, rtio_channels)
         self.csr_devices.append("rtio_core")
 
-        self.submodules.rtio = rtio.KernelInitiator()
+        self.submodules.rtio = rtio.KernelInitiator(self.rtio_tsc)
         self.submodules.rtio_dma = ClockDomainsRenamer("sys_kernel")(
             rtio.DMA(self.get_native_sdram_if()))
         self.register_kernel_cpu_csrdevice("rtio")
         self.register_kernel_cpu_csrdevice("rtio_dma")
         self.submodules.cri_con = rtio.CRIInterconnectShared(
             [self.rtio.cri, self.rtio_dma.cri],
-            [self.rtio_core.cri] + self.drtio_cri)
+            [self.rtio_core.cri] + self.drtio_cri,
+            enable_routing=True)
         self.register_kernel_cpu_csrdevice("cri_con")
+        self.submodules.routing_table = rtio.RoutingTableAccess(self.cri_con)
+        self.csr_devices.append("routing_table")
 
-        self.submodules.rtio_analyzer = rtio.Analyzer(self.cri_con.switch.slave,
+        self.submodules.rtio_analyzer = rtio.Analyzer(self.rtio_tsc, self.cri_con.switch.slave,
                                                       self.get_native_sdram_if())
         self.csr_devices.append("rtio_analyzer")
 
@@ -930,11 +953,11 @@ class _MasterBase(MiniSoC, AMPSoC):
 
 class _SatelliteBase(BaseSoC):
     mem_map = {
-        "drtio_aux": 0x50000000,
+        "drtioaux": 0x50000000,
     }
     mem_map.update(BaseSoC.mem_map)
 
-    def __init__(self, rtio_clk_freq=150e6, **kwargs):
+    def __init__(self, rtio_clk_freq=125e6, **kwargs):
         BaseSoC.__init__(self,
                  cpu_type="or1k",
                  sdram_controller_type="minicon",
@@ -960,17 +983,63 @@ class _SatelliteBase(BaseSoC):
         qpll = QPLL(si5324_clkout_buf, qpll_drtio_settings)
         self.submodules += qpll
 
-        sfp_ctl = platform.request("sfp_ctl", 0)
-        self.comb += sfp_ctl.tx_disable.eq(0)
+        sfp_ctls = [platform.request("sfp_ctl", i) for i in range(3)]
+        self.comb += [sc.tx_disable.eq(0) for sc in sfp_ctls]
         self.submodules.drtio_transceiver = gtp_7series.GTP(
             qpll_channel=qpll.channels[0],
-            data_pads=[platform.request("sfp", 0)],
+            data_pads=[platform.request("sfp", i) for i in range(3)],
             sys_clk_freq=self.clk_freq,
             rtio_clk_freq=rtio_clk_freq)
         self.csr_devices.append("drtio_transceiver")
         self.sync += disable_si5324_ibuf.eq(
             ~self.drtio_transceiver.stable_clkin.storage)
-        self.comb += sfp_ctl.led.eq(self.drtio_transceiver.channels[0].rx_ready)
+        self.comb += [sfp_ctl.led.eq(channel.rx_ready)
+            for sfp_ctl, channel in zip(sfp_ctls, self.drtio_transceiver.channels)]
+
+        self.submodules.rtio_tsc = rtio.TSC("sync", glbl_fine_ts_width=3)
+
+        drtioaux_csr_group = []
+        drtioaux_memory_group = []
+        drtiorep_csr_group = []
+        self.drtio_cri = []
+        for i in range(3):
+            coreaux_name = "drtioaux" + str(i)
+            memory_name = "drtioaux" + str(i) + "_mem"
+            drtioaux_csr_group.append(coreaux_name)
+            drtioaux_memory_group.append(memory_name)
+
+            cdr = ClockDomainsRenamer({"rtio_rx": "rtio_rx" + str(i)})
+
+            if i == 0:
+                self.submodules.rx_synchronizer = cdr(XilinxRXSynchronizer())
+                core = cdr(DRTIOSatellite(
+                    self.rtio_tsc, self.drtio_transceiver.channels[i],
+                    self.rx_synchronizer))
+                self.submodules.drtiosat = core
+                self.csr_devices.append("drtiosat")
+            else:
+                corerep_name = "drtiorep" + str(i-1)
+                drtiorep_csr_group.append(corerep_name)
+
+                core = cdr(DRTIORepeater(
+                    self.rtio_tsc, self.drtio_transceiver.channels[i]))
+                setattr(self.submodules, corerep_name, core)
+                self.drtio_cri.append(core.cri)
+                self.csr_devices.append(corerep_name)
+
+            coreaux = cdr(DRTIOAuxController(core.link_layer))
+            setattr(self.submodules, coreaux_name, coreaux)
+            self.csr_devices.append(coreaux_name)
+
+            memory_address = self.mem_map["drtioaux"] + 0x800*i
+            self.add_wb_slave(memory_address, 0x800,
+                              coreaux.bus)
+            self.add_memory_region(memory_name, memory_address | self.shadow_base, 0x800)
+        self.config["HAS_DRTIO"] = None
+        self.config["HAS_DRTIO_ROUTING"] = None
+        self.add_csr_group("drtioaux", drtioaux_csr_group)
+        self.add_memory_group("drtioaux_mem", drtioaux_memory_group)
+        self.add_csr_group("drtiorep", drtiorep_csr_group)
 
         self.config["RTIO_FREQUENCY"] = str(rtio_clk_freq/1e6)
         self.submodules.siphaser = SiPhaser7Series(
@@ -995,26 +1064,28 @@ class _SatelliteBase(BaseSoC):
         platform.add_false_path_constraints(
             self.crg.cd_sys.clk,
             gtp.txoutclk, gtp.rxoutclk)
+        for gtp in self.drtio_transceiver.gtps[1:]:
+            platform.add_period_constraint(gtp.rxoutclk, rtio_clk_period)
+            platform.add_false_path_constraints(
+                self.crg.cd_sys.clk, gtp.rxoutclk)
 
-        self.submodules.rtio_clkmul = _RTIOClockMultiplier(rtio_clk_freq)
+        self.submodules.rtio_crg = _RTIOClockMultiplier(rtio_clk_freq)
+        self.csr_devices.append("rtio_crg")
         fix_serdes_timing_path(platform)
 
     def add_rtio(self, rtio_channels):
         self.submodules.rtio_moninj = rtio.MonInj(rtio_channels)
         self.csr_devices.append("rtio_moninj")
 
-        rx0 = ClockDomainsRenamer({"rtio_rx": "rtio_rx0"})
-        self.submodules.rx_synchronizer = rx0(XilinxRXSynchronizer())
-        self.submodules.drtio0 = rx0(DRTIOSatellite(
-            self.drtio_transceiver.channels[0], rtio_channels,
-            self.rx_synchronizer))
-        self.csr_devices.append("drtio0")
-        self.add_wb_slave(self.mem_map["drtio_aux"], 0x800,
-                          self.drtio0.aux_controller.bus)
-        self.add_memory_region("drtio0_aux", self.mem_map["drtio_aux"] | self.shadow_base, 0x800)
-        self.config["HAS_DRTIO"] = None
-        self.add_csr_group("drtio", ["drtio0"])
-        self.add_memory_group("drtio_aux", ["drtio0_aux"])
+        self.submodules.local_io = SyncRTIO(self.rtio_tsc, rtio_channels)
+        self.comb += self.drtiosat.async_errors.eq(self.local_io.async_errors)
+        self.submodules.cri_con = rtio.CRIInterconnectShared(
+            [self.drtiosat.cri],
+            [self.local_io.cri] + self.drtio_cri,
+            mode="sync", enable_routing=True)
+        self.csr_devices.append("cri_con")
+        self.submodules.routing_table = rtio.RoutingTableAccess(self.cri_con)
+        self.csr_devices.append("routing_table")
 
 
 class Master(_MasterBase):

artiq/gateware/targets/kc705.py

@@ -161,9 +161,10 @@ class _StandaloneBase(MiniSoC, AMPSoC):
         self.submodules.rtio_crg = _RTIOCRG(self.platform, self.crg.cd_sys.clk)
         self.csr_devices.append("rtio_crg")
         self.config["HAS_RTIO_CLOCK_SWITCH"] = None
-        self.submodules.rtio_core = rtio.Core(rtio_channels)
+        self.submodules.rtio_tsc = rtio.TSC("async", glbl_fine_ts_width=3)
+        self.submodules.rtio_core = rtio.Core(self.rtio_tsc, rtio_channels)
         self.csr_devices.append("rtio_core")
-        self.submodules.rtio = rtio.KernelInitiator()
+        self.submodules.rtio = rtio.KernelInitiator(self.rtio_tsc)
         self.submodules.rtio_dma = ClockDomainsRenamer("sys_kernel")(
             rtio.DMA(self.get_native_sdram_if()))
         self.register_kernel_cpu_csrdevice("rtio")
@@ -180,7 +181,7 @@ class _StandaloneBase(MiniSoC, AMPSoC):
             self.crg.cd_sys.clk,
             self.rtio_crg.cd_rtio.clk)
 
-        self.submodules.rtio_analyzer = rtio.Analyzer(self.rtio_core.cri,
+        self.submodules.rtio_analyzer = rtio.Analyzer(self.rtio_tsc, self.rtio_core.cri,
                                                       self.get_native_sdram_if())
         self.csr_devices.append("rtio_analyzer")
 
@@ -377,9 +378,10 @@ class SMA_SPI(_StandaloneBase):
                                             use_sma=False)
         self.csr_devices.append("rtio_crg")
         self.config["HAS_RTIO_CLOCK_SWITCH"] = None
-        self.submodules.rtio_core = rtio.Core(rtio_channels)
+        self.submodules.rtio_tsc = rtio.TSC("async", glbl_fine_ts_width=3)
+        self.submodules.rtio_core = rtio.Core(self.rtio_tsc, rtio_channels)
         self.csr_devices.append("rtio_core")
-        self.submodules.rtio = rtio.KernelInitiator()
+        self.submodules.rtio = rtio.KernelInitiator(self.rtio_tsc)
         self.submodules.rtio_dma = ClockDomainsRenamer("sys_kernel")(
             rtio.DMA(self.get_native_sdram_if()))
         self.register_kernel_cpu_csrdevice("rtio")
@@ -395,7 +397,7 @@ class SMA_SPI(_StandaloneBase):
             self.crg.cd_sys.clk,
             self.rtio_crg.cd_rtio.clk)
 
-        self.submodules.rtio_analyzer = rtio.Analyzer(self.rtio_core.cri,
+        self.submodules.rtio_analyzer = rtio.Analyzer(self.rtio_tsc, self.rtio_core.cri,
                                                       self.get_native_sdram_if())
         self.csr_devices.append("rtio_analyzer")
 

artiq/gateware/targets/sayma_amc.py

@@ -22,7 +22,7 @@ from artiq.gateware.rtio.phy import ttl_simple, sawg
 from artiq.gateware.drtio.transceiver import gth_ultrascale
 from artiq.gateware.drtio.siphaser import SiPhaser7Series
 from artiq.gateware.drtio.rx_synchronizer import XilinxRXSynchronizer
-from artiq.gateware.drtio import DRTIOMaster, DRTIOSatellite
+from artiq.gateware.drtio import *
 from artiq.build_soc import *
 
 
@@ -201,9 +201,10 @@ class Standalone(MiniSoC, AMPSoC, RTMCommon):
             self.cd_rtio.clk.eq(ClockSignal("jesd")),
             self.cd_rtio.rst.eq(ResetSignal("jesd"))
         ]
-        self.submodules.rtio_core = rtio.Core(rtio_channels)
+        self.submodules.rtio_tsc = rtio.TSC("async")
+        self.submodules.rtio_core = rtio.Core(self.rtio_tsc, rtio_channels)
         self.csr_devices.append("rtio_core")
-        self.submodules.rtio = rtio.KernelInitiator()
+        self.submodules.rtio = rtio.KernelInitiator(self.rtio_tsc)
         self.submodules.rtio_dma = ClockDomainsRenamer("sys_kernel")(
             rtio.DMA(self.get_native_sdram_if()))
         self.register_kernel_cpu_csrdevice("rtio")
@@ -215,12 +216,12 @@ class Standalone(MiniSoC, AMPSoC, RTMCommon):
         self.submodules.rtio_moninj = rtio.MonInj(rtio_channels)
         self.csr_devices.append("rtio_moninj")
 
-        self.submodules.rtio_analyzer = rtio.Analyzer(self.rtio_core.cri,
+        self.submodules.rtio_analyzer = rtio.Analyzer(self.rtio_tsc, self.rtio_core.cri,
                                                       self.get_native_sdram_if())
         self.csr_devices.append("rtio_analyzer")
 
         self.submodules.sysref_sampler = jesd204_tools.SysrefSampler(
-            self.rtio_core.coarse_ts, self.ad9154_crg.jref)
+            self.rtio_tsc.coarse_ts, self.ad9154_crg.jref)
         self.csr_devices.append("sysref_sampler")
 
 
@@ -233,7 +234,7 @@ class MasterDAC(MiniSoC, AMPSoC, RTMCommon):
         "rtio":          0x11000000,
         "rtio_dma":      0x12000000,
         "serwb":         0x13000000,
-        "drtio_aux":     0x14000000,
+        "drtioaux":      0x14000000,
         "mailbox":       0x70000000
     }
     mem_map.update(MiniSoC.mem_map)
@@ -283,28 +284,40 @@ class MasterDAC(MiniSoC, AMPSoC, RTMCommon):
             rtio_clk_freq=rtio_clk_freq)
         self.csr_devices.append("drtio_transceiver")
 
+        self.rtio_tsc = rtio.TSC("async", glbl_fine_ts_width=3)
+
         drtio_csr_group = []
-        drtio_memory_group = []
+        drtioaux_csr_group = []
+        drtioaux_memory_group = []
         drtio_cri = []
         for i in range(2):
             core_name = "drtio" + str(i)
-            memory_name = "drtio" + str(i) + "_aux"
+            coreaux_name = "drtioaux" + str(i)
+            memory_name = "drtioaux" + str(i) + "_mem"
             drtio_csr_group.append(core_name)
-            drtio_memory_group.append(memory_name)
+            drtioaux_csr_group.append(coreaux_name)
+            drtioaux_memory_group.append(memory_name)
 
-            core = ClockDomainsRenamer({"rtio_rx": "rtio_rx"+str(i)})(
-                DRTIOMaster(self.drtio_transceiver.channels[i]))
+            cdr = ClockDomainsRenamer({"rtio_rx": "rtio_rx" + str(i)})
+
+            core = cdr(DRTIOMaster(self.rtio_tsc, self.drtio_transceiver.channels[i]))
             setattr(self.submodules, core_name, core)
             drtio_cri.append(core.cri)
             self.csr_devices.append(core_name)
 
-            memory_address = self.mem_map["drtio_aux"] + 0x800*i
+            coreaux = cdr(DRTIOAuxController(core.link_layer))
+            setattr(self.submodules, coreaux_name, coreaux)
+            self.csr_devices.append(coreaux_name)
+
+            memory_address = self.mem_map["drtioaux"] + 0x800*i
             self.add_wb_slave(memory_address, 0x800,
-                              core.aux_controller.bus)
+                              coreaux.bus)
             self.add_memory_region(memory_name, memory_address | self.shadow_base, 0x800)
         self.config["HAS_DRTIO"] = None
+        self.config["HAS_DRTIO_ROUTING"] = None
         self.add_csr_group("drtio", drtio_csr_group)
-        self.add_memory_group("drtio_aux", drtio_memory_group)
+        self.add_csr_group("drtioaux", drtioaux_csr_group)
+        self.add_memory_group("drtioaux_mem", drtioaux_memory_group)
 
         rtio_clk_period = 1e9/rtio_clk_freq
         gth = self.drtio_transceiver.gths[0]
@@ -357,21 +370,24 @@ class MasterDAC(MiniSoC, AMPSoC, RTMCommon):
         self.submodules.rtio_moninj = rtio.MonInj(rtio_channels)
         self.csr_devices.append("rtio_moninj")
 
-        self.submodules.rtio_core = rtio.Core(rtio_channels, glbl_fine_ts_width=3)
+        self.submodules.rtio_core = rtio.Core(self.rtio_tsc, rtio_channels)
         self.csr_devices.append("rtio_core")
 
-        self.submodules.rtio = rtio.KernelInitiator()
+        self.submodules.rtio = rtio.KernelInitiator(self.rtio_tsc)
         self.submodules.rtio_dma = ClockDomainsRenamer("sys_kernel")(
             rtio.DMA(self.get_native_sdram_if()))
         self.register_kernel_cpu_csrdevice("rtio")
         self.register_kernel_cpu_csrdevice("rtio_dma")
         self.submodules.cri_con = rtio.CRIInterconnectShared(
             [self.rtio.cri, self.rtio_dma.cri],
-            [self.rtio_core.cri] + drtio_cri)
+            [self.rtio_core.cri] + drtio_cri,
+            enable_routing=True)
         self.register_kernel_cpu_csrdevice("cri_con")
+        self.submodules.routing_table = rtio.RoutingTableAccess(self.cri_con)
+        self.csr_devices.append("routing_table")
 
         self.submodules.sysref_sampler = jesd204_tools.SysrefSampler(
-            self.rtio_core.coarse_ts, self.ad9154_crg.jref)
+            self.rtio_tsc.coarse_ts, self.ad9154_crg.jref)
         self.csr_devices.append("sysref_sampler")
 
 
@@ -386,7 +402,7 @@ class Master(MiniSoC, AMPSoC):
         "cri_con":       0x10000000,
         "rtio":          0x11000000,
         "rtio_dma":      0x12000000,
-        "drtio_aux":     0x14000000,
+        "drtioaux":      0x14000000,
         "mailbox":       0x70000000
     }
     mem_map.update(MiniSoC.mem_map)
@@ -427,28 +443,40 @@ class Master(MiniSoC, AMPSoC):
             rtio_clk_freq=rtio_clk_freq)
         self.csr_devices.append("drtio_transceiver")
 
+        self.rtio_tsc = rtio.TSC("async", glbl_fine_ts_width=3)
+
         drtio_csr_group = []
-        drtio_memory_group = []
+        drtioaux_csr_group = []
+        drtioaux_memory_group = []
         drtio_cri = []
         for i in range(10):
             core_name = "drtio" + str(i)
-            memory_name = "drtio" + str(i) + "_aux"
+            coreaux_name = "drtioaux" + str(i)
+            memory_name = "drtioaux" + str(i) + "_mem"
             drtio_csr_group.append(core_name)
-            drtio_memory_group.append(memory_name)
+            drtioaux_csr_group.append(coreaux_name)
+            drtioaux_memory_group.append(memory_name)
 
-            core = ClockDomainsRenamer({"rtio_rx": "rtio_rx"+str(i)})(
-                DRTIOMaster(self.drtio_transceiver.channels[i]))
+            cdr = ClockDomainsRenamer({"rtio_rx": "rtio_rx" + str(i)})
+
+            core = cdr(DRTIOMaster(self.rtio_tsc, self.drtio_transceiver.channels[i]))
             setattr(self.submodules, core_name, core)
             drtio_cri.append(core.cri)
             self.csr_devices.append(core_name)
 
-            memory_address = self.mem_map["drtio_aux"] + 0x800*i
+            coreaux = cdr(DRTIOAuxController(core.link_layer))
+            setattr(self.submodules, coreaux_name, coreaux)
+            self.csr_devices.append(coreaux_name)
+
+            memory_address = self.mem_map["drtioaux"] + 0x800*i
             self.add_wb_slave(memory_address, 0x800,
-                              core.aux_controller.bus)
+                              coreaux.bus)
             self.add_memory_region(memory_name, memory_address | self.shadow_base, 0x800)
         self.config["HAS_DRTIO"] = None
+        self.config["HAS_DRTIO_ROUTING"] = None
         self.add_csr_group("drtio", drtio_csr_group)
-        self.add_memory_group("drtio_aux", drtio_memory_group)
+        self.add_csr_group("drtioaux", drtioaux_csr_group)
+        self.add_memory_group("drtioaux_mem", drtioaux_memory_group)
 
         rtio_clk_period = 1e9/rtio_clk_freq
         gth = self.drtio_transceiver.gths[0]
@@ -499,18 +527,21 @@ class Master(MiniSoC, AMPSoC):
         self.submodules.rtio_moninj = rtio.MonInj(rtio_channels)
         self.csr_devices.append("rtio_moninj")
 
-        self.submodules.rtio_core = rtio.Core(rtio_channels, glbl_fine_ts_width=3)
+        self.submodules.rtio_core = rtio.Core(self.rtio_tsc, rtio_channels)
         self.csr_devices.append("rtio_core")
 
-        self.submodules.rtio = rtio.KernelInitiator()
+        self.submodules.rtio = rtio.KernelInitiator(self.rtio_tsc)
         self.submodules.rtio_dma = ClockDomainsRenamer("sys_kernel")(
             rtio.DMA(self.get_native_sdram_if()))
         self.register_kernel_cpu_csrdevice("rtio")
         self.register_kernel_cpu_csrdevice("rtio_dma")
         self.submodules.cri_con = rtio.CRIInterconnectShared(
             [self.rtio.cri, self.rtio_dma.cri],
-            [self.rtio_core.cri] + drtio_cri)
+            [self.rtio_core.cri] + drtio_cri,
+            enable_routing=True)
         self.register_kernel_cpu_csrdevice("cri_con")
+        self.submodules.routing_table = rtio.RoutingTableAccess(self.cri_con)
+        self.csr_devices.append("routing_table")
 
 
 class Satellite(BaseSoC, RTMCommon):
@@ -520,7 +551,7 @@ class Satellite(BaseSoC, RTMCommon):
     """
     mem_map = {
         "serwb":         0x13000000,
-        "drtio_aux":     0x14000000,
+        "drtioaux":      0x14000000,
     }
     mem_map.update(BaseSoC.mem_map)
 
@@ -583,18 +614,29 @@ class Satellite(BaseSoC, RTMCommon):
             rtio_clk_freq=rtio_clk_freq)
         self.csr_devices.append("drtio_transceiver")
 
+        self.rtio_tsc = rtio.TSC("sync", glbl_fine_ts_width=3)
+
         rx0 = ClockDomainsRenamer({"rtio_rx": "rtio_rx0"})
         self.submodules.rx_synchronizer = rx0(XilinxRXSynchronizer())
-        self.submodules.drtio0 = rx0(DRTIOSatellite(
-            self.drtio_transceiver.channels[0], rtio_channels,
+        self.submodules.drtiosat = rx0(DRTIOSatellite(
+            self.rtio_tsc, self.drtio_transceiver.channels[0],
             self.rx_synchronizer))
-        self.csr_devices.append("drtio0")
-        self.add_wb_slave(self.mem_map["drtio_aux"], 0x800,
-                          self.drtio0.aux_controller.bus)
-        self.add_memory_region("drtio0_aux", self.mem_map["drtio_aux"] | self.shadow_base, 0x800)
+        self.csr_devices.append("drtiosat")
+        self.submodules.drtioaux0 = rx0(DRTIOAuxController(
+            self.drtiosat.link_layer))
+        self.csr_devices.append("drtioaux0")
+        self.add_wb_slave(self.mem_map["drtioaux"], 0x800,
+                          self.drtioaux0.bus)
+        self.add_memory_region("drtioaux0_mem", self.mem_map["drtioaux"] | self.shadow_base, 0x800)
         self.config["HAS_DRTIO"] = None
-        self.add_csr_group("drtio", ["drtio0"])
-        self.add_memory_group("drtio_aux", ["drtio0_aux"])
+        self.add_csr_group("drtioaux", ["drtioaux0"])
+        self.add_memory_group("drtioaux_mem", ["drtioaux0_mem"])
+
+        self.submodules.local_io = SyncRTIO(self.rtio_tsc, rtio_channels)
+        self.comb += [
+            self.drtiosat.cri.connect(self.local_io.cri),
+            self.drtiosat.async_errors.eq(self.local_io.async_errors),
+        ]
 
         self.config["RTIO_FREQUENCY"] = str(rtio_clk_freq/1e6)
         self.submodules.siphaser = SiPhaser7Series(
@@ -614,7 +656,7 @@ class Satellite(BaseSoC, RTMCommon):
         self.config["HAS_SI5324"] = None
 
         self.submodules.sysref_sampler = jesd204_tools.SysrefSampler(
-            self.drtio0.coarse_ts, self.ad9154_crg.jref)
+            self.rtio_tsc.coarse_ts, self.ad9154_crg.jref)
         self.csr_devices.append("sysref_sampler")
 
         rtio_clk_period = 1e9/rtio_clk_freq

artiq/gateware/test/drtio/packet_interface.py

@@ -0,0 +1,72 @@
+from migen import *
+
+from artiq.gateware.drtio.rt_serializer import *
+
+
+class PacketInterface:
+    def __init__(self, direction, ws):
+        if direction == "m2s":
+            self.plm = get_m2s_layouts(ws)
+        elif direction == "s2m":
+            self.plm = get_s2m_layouts(ws)
+        else:
+            raise ValueError
+        self.frame = Signal()
+        self.data = Signal(ws)
+
+    def send(self, ty, **kwargs):
+        idx = 8
+        value = self.plm.types[ty]
+        for field_name, field_size in self.plm.layouts[ty][1:]:
+            try:
+                fvalue = kwargs[field_name]
+                del kwargs[field_name]
+            except KeyError:
+                fvalue = 0
+            value = value | (fvalue << idx)
+            idx += field_size
+        if kwargs:
+            raise ValueError
+
+        ws = len(self.data)
+        yield self.frame.eq(1)
+        for i in range(idx//ws):
+            yield self.data.eq(value)
+            value >>= ws
+            yield
+        yield self.frame.eq(0)
+        yield
+
+    @passive
+    def receive(self, callback):
+        previous_frame = 0
+        frame_words = []
+        while True:
+            frame = yield self.frame
+            if frame:
+                frame_words.append((yield self.data))
+            if previous_frame and not frame:
+                packet_type = self.plm.type_names[frame_words[0] & 0xff]
+                packet_nwords = layout_len(self.plm.layouts[packet_type]) \
+                                //len(self.data)
+                packet, trailer = frame_words[:packet_nwords], \
+                                  frame_words[packet_nwords:]
+
+                n = 0
+                packet_int = 0
+                for w in packet:
+                    packet_int |= (w << n)
+                    n += len(self.data)
+
+                field_dict = dict()
+                idx = 0
+                for field_name, field_size in self.plm.layouts[packet_type]:
+                    v = (packet_int >> idx) & (2**field_size - 1)
+                    field_dict[field_name] = v
+                    idx += field_size
+
+                callback(packet_type, field_dict, trailer)
+
+                frame_words = []
+            previous_frame = frame
+            yield

artiq/gateware/test/drtio/test_aux_controller.py

@@ -36,7 +36,7 @@ class TB(Module):
     def __init__(self, nwords):
         self.submodules.link_layer = Loopback(nwords)
         self.submodules.aux_controller = ClockDomainsRenamer(
-            {"rtio": "sys", "rtio_rx": "sys"})(AuxController(self.link_layer))
+            {"rtio": "sys", "rtio_rx": "sys"})(DRTIOAuxController(self.link_layer))
 
 
 class TestAuxController(unittest.TestCase):

artiq/gateware/test/drtio/test_cdc.py

@@ -0,0 +1,94 @@
+import unittest
+import random
+
+from migen import *
+
+from artiq.gateware.drtio.cdc import CrossDomainRequest, CrossDomainNotification
+
+
+class TestCDC(unittest.TestCase):
+    def test_cross_domain_request(self):
+        prng = random.Random(1)
+        for sys_freq in 3, 6, 11:
+            for srv_freq in 3, 6, 11:
+                req_stb = Signal()
+                req_ack = Signal()
+                req_data = Signal(8)
+                srv_stb = Signal()
+                srv_ack = Signal()
+                srv_data = Signal(8)
+                test_seq = [93, 92, 19, 39, 91, 30, 12, 91, 38, 42]
+                received_seq = []
+
+                def requester():
+                    for data in test_seq:
+                        yield req_data.eq(data)
+                        yield req_stb.eq(1)
+                        yield
+                        while not (yield req_ack):
+                            yield
+                        yield req_stb.eq(0)
+                        for j in range(prng.randrange(0, 10)):
+                            yield
+
+                def server():
+                    for i in range(len(test_seq)):
+                        while not (yield srv_stb):
+                            yield
+                        received_seq.append((yield srv_data))
+                        for j in range(prng.randrange(0, 10)):
+                            yield
+                        yield srv_ack.eq(1)
+                        yield
+                        yield srv_ack.eq(0)
+                        yield
+
+                dut = CrossDomainRequest("srv",
+                         req_stb, req_ack, req_data,
+                         srv_stb, srv_ack, srv_data)
+                run_simulation(dut,
+                    {"sys": requester(), "srv": server()},
+                    {"sys": sys_freq, "srv": srv_freq})
+                self.assertEqual(test_seq, received_seq)
+
+    def test_cross_domain_notification(self):
+        prng = random.Random(1)
+
+        emi_stb = Signal()
+        emi_data = Signal(8)
+        rec_stb = Signal()
+        rec_ack = Signal()
+        rec_data = Signal(8)
+
+        test_seq = [23, 12, 8, 3, 28]
+        received_seq = []
+
+        def emitter():
+            for data in test_seq:
+                yield emi_stb.eq(1)
+                yield emi_data.eq(data)
+                yield
+                yield emi_stb.eq(0)
+                yield
+                for j in range(prng.randrange(0, 3)):
+                    yield
+
+        def receiver():
+            for i in range(len(test_seq)):
+                while not (yield rec_stb):
+                    yield
+                received_seq.append((yield rec_data))
+                yield rec_ack.eq(1)
+                yield
+                yield rec_ack.eq(0)
+                yield
+                for j in range(prng.randrange(0, 3)):
+                    yield
+
+        dut = CrossDomainNotification("emi", "sys",
+            emi_stb, emi_data,
+            rec_stb, rec_ack, rec_data)
+        run_simulation(dut,
+            {"emi": emitter(), "sys": receiver()},
+            {"emi": 13, "sys": 3})
+        self.assertEqual(test_seq, received_seq)

artiq/gateware/test/drtio/test_full_stack.py

@@ -52,10 +52,11 @@ class DUT(Module):
         self.ttl1 = Signal()
         self.transceivers = DummyTransceiverPair(nwords)
 
-        self.submodules.master = DRTIOMaster(self.transceivers.alice,
-                                             fine_ts_width=0)
-        self.submodules.master_ki = rtio.KernelInitiator(self.master.cri)
-        self.master.rt_controller.csrs.link_up.storage.reset = 1
+        self.submodules.tsc_master = rtio.TSC("async")
+        self.submodules.master = DRTIOMaster(self.tsc_master,
+                                             self.transceivers.alice)
+        self.submodules.master_ki = rtio.KernelInitiator(self.tsc_master,
+            self.master.cri)
 
         rx_synchronizer = DummyRXSynchronizer()
         self.submodules.phy0 = ttl_simple.Output(self.ttl0)
@@ -66,13 +67,19 @@ class DUT(Module):
             rtio.Channel.from_phy(self.phy1),
             rtio.Channel.from_phy(self.phy2),
         ]
+        self.submodules.tsc_satellite = rtio.TSC("sync")
         self.submodules.satellite = DRTIOSatellite(
-            self.transceivers.bob, rtio_channels, rx_synchronizer,
-            lane_count=4, fifo_depth=8, fine_ts_width=0)
+            self.tsc_satellite, self.transceivers.bob, rx_synchronizer)
         self.satellite.reset.storage.reset = 0
         self.satellite.reset.storage_full.reset = 0
         self.satellite.reset_phy.storage.reset = 0
         self.satellite.reset_phy.storage_full.reset = 0
+        self.submodules.satellite_rtio = SyncRTIO(
+            self.tsc_satellite, rtio_channels, lane_count=4, fifo_depth=8)
+        self.comb += [
+            self.satellite.cri.connect(self.satellite_rtio.cri),
+            self.satellite.async_errors.eq(self.satellite_rtio.async_errors),
+        ]
 
 
 class OutputsTestbench:
@@ -100,7 +107,7 @@ class OutputsTestbench:
 
     def sync(self):
         t = self.now + 15
-        while (yield self.dut.master.cri.counter) < t:
+        while (yield self.dut.tsc_master.full_ts_cri) < t:
             yield
 
     def write(self, channel, data):
@@ -117,7 +124,7 @@ class OutputsTestbench:
             if status & 0x2:
                 raise RTIOUnderflow
             if status & 0x4:
-                raise RTIOLinkError
+                raise RTIODestinationUnreachable
             yield
             wlen += 1
         return wlen
@@ -138,8 +145,7 @@ class OutputsTestbench:
 
 class TestFullStack(unittest.TestCase):
     clocks = {"sys": 8, "rtio": 5, "rtio_rx": 5,
-              "rio": 5, "rio_phy": 5,
-              "sys_with_rst": 8, "rtio_with_rst": 5}
+              "rio": 5, "rio_phy": 5}
 
     def test_pulses(self):
         tb = OutputsTestbench()
@@ -228,7 +234,7 @@ class TestFullStack(unittest.TestCase):
             errors = yield from saterr.protocol_error.read()
             underflow_channel = yield from saterr.underflow_channel.read()
             underflow_timestamp_event = yield from saterr.underflow_timestamp_event.read()
-            self.assertEqual(errors, 4)  # write underflow
+            self.assertEqual(errors, 8)  # write underflow
             self.assertEqual(underflow_channel, 42)
             self.assertEqual(underflow_timestamp_event, 100)
             yield from saterr.protocol_error.write(errors)
@@ -256,7 +262,7 @@ class TestFullStack(unittest.TestCase):
             if status & 0x2:
                 return "overflow"
             if status & 0x8:
-                return "link error"
+                return "destination unreachable"
             return ((yield from kcsrs.i_data.read()),
                     (yield from kcsrs.i_timestamp.read()))
 
@@ -283,26 +289,25 @@ class TestFullStack(unittest.TestCase):
 
     def test_echo(self):
         dut = DUT(2)
-        csrs = dut.master.rt_controller.csrs
-        mgr = dut.master.rt_manager
+        packet = dut.master.rt_packet
 
         def test():
             while not (yield from dut.master.link_layer.rx_up.read()):
                 yield
 
-            yield from mgr.update_packet_cnt.write(1)
-            yield
-            self.assertEqual((yield from mgr.packet_cnt_tx.read()), 0)
-            self.assertEqual((yield from mgr.packet_cnt_rx.read()), 0)
+            self.assertEqual((yield dut.master.rt_packet.packet_cnt_tx), 0)
+            self.assertEqual((yield dut.master.rt_packet.packet_cnt_rx), 0)
 
-            yield from mgr.request_echo.write(1)
+            yield dut.master.rt_packet.echo_stb.eq(1)
+            yield
+            while not (yield dut.master.rt_packet.echo_ack):
+                yield
+            yield dut.master.rt_packet.echo_stb.eq(0)
 
             for i in range(15):
                 yield
 
-            yield from mgr.update_packet_cnt.write(1)
-            yield
-            self.assertEqual((yield from mgr.packet_cnt_tx.read()), 1)
-            self.assertEqual((yield from mgr.packet_cnt_rx.read()), 1)
+            self.assertEqual((yield dut.master.rt_packet.packet_cnt_tx), 1)
+            self.assertEqual((yield dut.master.rt_packet.packet_cnt_rx), 1)
 
         run_simulation(dut, test(), self.clocks)

artiq/gateware/test/drtio/test_rt_packet.py

@@ -1,213 +0,0 @@
-import unittest
-from types import SimpleNamespace
-import random
-
-from migen import *
-
-from artiq.gateware.drtio.rt_serializer import *
-from artiq.gateware.drtio.rt_packet_satellite import RTPacketSatellite
-from artiq.gateware.drtio.rt_packet_master import (_CrossDomainRequest,
-                                                   _CrossDomainNotification)
-
-
-class PacketInterface:
-    def __init__(self, direction, ws):
-        if direction == "m2s":
-            self.plm = get_m2s_layouts(ws)
-        elif direction == "s2m":
-            self.plm = get_s2m_layouts(ws)
-        else:
-            raise ValueError
-        self.frame = Signal()
-        self.data = Signal(ws)
-
-    def send(self, ty, **kwargs):
-        idx = 8
-        value = self.plm.types[ty]
-        for field_name, field_size in self.plm.layouts[ty][1:]:
-            try:
-                fvalue = kwargs[field_name]
-                del kwargs[field_name]
-            except KeyError:
-                fvalue = 0
-            value = value | (fvalue << idx)
-            idx += field_size
-        if kwargs:
-            raise ValueError
-
-        ws = len(self.data)
-        yield self.frame.eq(1)
-        for i in range(idx//ws):
-            yield self.data.eq(value)
-            value >>= ws
-            yield
-        yield self.frame.eq(0)
-        yield
-
-    @passive
-    def receive(self, callback):
-        previous_frame = 0
-        frame_words = []
-        while True:
-            frame = yield self.frame
-            if frame:
-                frame_words.append((yield self.data))
-            if previous_frame and not frame:
-                packet_type = self.plm.type_names[frame_words[0] & 0xff]
-                packet_nwords = layout_len(self.plm.layouts[packet_type]) \
-                                //len(self.data)
-                packet, trailer = frame_words[:packet_nwords], \
-                                  frame_words[packet_nwords:]
-
-                n = 0
-                packet_int = 0
-                for w in packet:
-                    packet_int |= (w << n)
-                    n += len(self.data)
-
-                field_dict = dict()
-                idx = 0
-                for field_name, field_size in self.plm.layouts[packet_type]:
-                    v = (packet_int >> idx) & (2**field_size - 1)
-                    field_dict[field_name] = v
-                    idx += field_size
-
-                callback(packet_type, field_dict, trailer)
-
-                frame_words = []
-            previous_frame = frame
-            yield
-
-
-class TestSatellite(unittest.TestCase):
-    def create_dut(self, nwords):
-        pt = PacketInterface("m2s", nwords*8)
-        pr = PacketInterface("s2m", nwords*8)
-        dut = RTPacketSatellite(SimpleNamespace(
-            rx_rt_frame=pt.frame, rx_rt_data=pt.data,
-            tx_rt_frame=pr.frame, tx_rt_data=pr.data))
-        return pt, pr, dut
-
-    def test_echo(self):
-        for nwords in range(1, 8):
-            pt, pr, dut = self.create_dut(nwords)
-            completed = False
-            def send():
-                yield from pt.send("echo_request")
-                while not completed:
-                    yield
-            def receive(packet_type, field_dict, trailer):
-                nonlocal completed
-                self.assertEqual(packet_type, "echo_reply")
-                self.assertEqual(trailer, [])
-                completed = True
-            run_simulation(dut, [send(), pr.receive(receive)])
-
-    def test_set_time(self):
-        for nwords in range(1, 8):
-            pt, _, dut = self.create_dut(nwords)
-            tx_times = [0x12345678aabbccdd, 0x0102030405060708,
-                        0xaabbccddeeff1122]
-            def send():
-                for t in tx_times:
-                    yield from pt.send("set_time", timestamp=t)
-                # flush
-                for i in range(10):
-                    yield
-            rx_times = []
-            @passive
-            def receive():
-                while True:
-                    if (yield dut.tsc_load):
-                        rx_times.append((yield dut.tsc_load_value))
-                    yield
-            run_simulation(dut, [send(), receive()])
-            self.assertEqual(tx_times, rx_times)
-
-
-class TestCDC(unittest.TestCase):
-    def test_cross_domain_request(self):
-        prng = random.Random(1)
-        for sys_freq in 3, 6, 11:
-            for srv_freq in 3, 6, 11:
-                req_stb = Signal()
-                req_ack = Signal()
-                req_data = Signal(8)
-                srv_stb = Signal()
-                srv_ack = Signal()
-                srv_data = Signal(8)
-                test_seq = [93, 92, 19, 39, 91, 30, 12, 91, 38, 42]
-                received_seq = []
-
-                def requester():
-                    for data in test_seq:
-                        yield req_data.eq(data)
-                        yield req_stb.eq(1)
-                        yield
-                        while not (yield req_ack):
-                            yield
-                        yield req_stb.eq(0)
-                        for j in range(prng.randrange(0, 10)):
-                            yield
-
-                def server():
-                    for i in range(len(test_seq)):
-                        while not (yield srv_stb):
-                            yield
-                        received_seq.append((yield srv_data))
-                        for j in range(prng.randrange(0, 10)):
-                            yield
-                        yield srv_ack.eq(1)
-                        yield
-                        yield srv_ack.eq(0)
-                        yield
-
-                dut = _CrossDomainRequest("srv",
-                         req_stb, req_ack, req_data,
-                         srv_stb, srv_ack, srv_data)
-                run_simulation(dut,
-                    {"sys": requester(), "srv": server()},
-                    {"sys": sys_freq, "srv": srv_freq})
-                self.assertEqual(test_seq, received_seq)
-
-    def test_cross_domain_notification(self):
-        prng = random.Random(1)
-
-        emi_stb = Signal()
-        emi_data = Signal(8)
-        rec_stb = Signal()
-        rec_ack = Signal()
-        rec_data = Signal(8)
-
-        test_seq = [23, 12, 8, 3, 28]
-        received_seq = []
-
-        def emitter():
-            for data in test_seq:
-                yield emi_stb.eq(1)
-                yield emi_data.eq(data)
-                yield
-                yield emi_stb.eq(0)
-                yield
-                for j in range(prng.randrange(0, 3)):
-                    yield
-
-        def receiver():
-            for i in range(len(test_seq)):
-                while not (yield rec_stb):
-                    yield
-                received_seq.append((yield rec_data))
-                yield rec_ack.eq(1)
-                yield
-                yield rec_ack.eq(0)
-                yield
-                for j in range(prng.randrange(0, 3)):
-                    yield
-
-        dut = _CrossDomainNotification("emi",
-            emi_stb, emi_data,
-            rec_stb, rec_ack, rec_data)
-        run_simulation(dut,
-            {"emi": emitter(), "sys": receiver()},
-            {"emi": 13, "sys": 3})
-        self.assertEqual(test_seq, received_seq)

artiq/gateware/test/drtio/test_rt_packet_repeater.py

@@ -0,0 +1,192 @@
+import unittest
+from types import SimpleNamespace
+
+from migen import *
+
+from artiq.gateware.rtio import cri
+from artiq.gateware.test.drtio.packet_interface import PacketInterface
+from artiq.gateware.drtio.rt_packet_repeater import RTPacketRepeater
+
+
+def create_dut(nwords):
+    pt = PacketInterface("s2m", nwords*8)
+    pr = PacketInterface("m2s", nwords*8)
+    ts = Signal(64)
+    dut = ClockDomainsRenamer({"rtio": "sys", "rtio_rx": "sys"})(
+        RTPacketRepeater(
+            SimpleNamespace(coarse_ts=ts),
+            SimpleNamespace(
+                rx_rt_frame=pt.frame, rx_rt_data=pt.data,
+                tx_rt_frame=pr.frame, tx_rt_data=pr.data)))
+    return pt, pr, ts, dut
+
+
+class TestRepeater(unittest.TestCase):
+    def test_set_time(self):
+        nwords = 2
+        pt, pr, ts, dut = create_dut(nwords)
+
+        def send():
+            yield
+            yield ts.eq(0x12345678)
+            yield dut.set_time_stb.eq(1)
+            while not (yield dut.set_time_ack):
+                yield
+            yield dut.set_time_stb.eq(0)
+            yield
+            for _ in range(30):
+                yield
+
+        received = False
+        def receive(packet_type, field_dict, trailer):
+            nonlocal received
+            self.assertEqual(packet_type, "set_time")
+            self.assertEqual(trailer, [])
+            self.assertEqual(field_dict["timestamp"], 0x12345678)
+            self.assertEqual(received, False)
+            received = True
+
+        run_simulation(dut, [send(), pr.receive(receive)])
+        self.assertEqual(received, True)
+
+    def test_output(self):
+        test_writes = [
+            (1, 10, 21, 0x42),
+            (2, 11, 34, 0x2342),
+            (3, 12, 83, 0x2345566633),
+            (4, 13, 25, 0x98da14959a19498ae1),
+            (5, 14, 75, 0x3998a1883ae14f828ae24958ea2479)
+        ]
+
+        for nwords in range(1, 8):
+            pt, pr, ts, dut = create_dut(nwords)
+
+            def send():
+                yield
+                for channel, timestamp, address, data in test_writes:
+                    yield dut.cri.chan_sel.eq(channel)
+                    yield dut.cri.timestamp.eq(timestamp)
+                    yield dut.cri.o_address.eq(address)
+                    yield dut.cri.o_data.eq(data)
+                    yield dut.cri.cmd.eq(cri.commands["write"])
+                    yield
+                    yield dut.cri.cmd.eq(cri.commands["nop"])
+                    yield
+                    for i in range(30):
+                        yield
+                for i in range(50):
+                        yield
+
+            short_data_len = pr.plm.field_length("write", "short_data")
+
+            received = []
+            def receive(packet_type, field_dict, trailer):
+                self.assertEqual(packet_type, "write")
+                self.assertEqual(len(trailer), field_dict["extra_data_cnt"]) 
+                data = field_dict["short_data"]
+                for n, te in enumerate(trailer):
+                    data |= te << (n*nwords*8 + short_data_len)
+                received.append((field_dict["chan_sel"], field_dict["timestamp"],
+                                 field_dict["address"], data))
+
+            run_simulation(dut, [send(), pr.receive(receive)])
+            self.assertEqual(test_writes, received)
+
+    def test_buffer_space(self):
+        for nwords in range(1, 8):
+            pt, pr, ts, dut = create_dut(nwords)
+
+            def send_requests():
+                for i in range(10):
+                    yield
+                    yield dut.cri.chan_sel.eq(i << 16)
+                    yield dut.cri.cmd.eq(cri.commands["get_buffer_space"])
+                    yield
+                    yield dut.cri.cmd.eq(cri.commands["nop"])
+                    yield
+                    while not (yield dut.cri.o_buffer_space_valid):
+                        yield
+                    buffer_space = yield dut.cri.o_buffer_space
+                    self.assertEqual(buffer_space, 2*i)
+
+            current_request = None
+
+            @passive
+            def send_replies():
+                nonlocal current_request
+                while True:
+                    while current_request is None:
+                        yield
+                    yield from pt.send("buffer_space_reply", space=2*current_request)
+                    current_request = None
+
+            def receive(packet_type, field_dict, trailer):
+                nonlocal current_request
+                self.assertEqual(packet_type, "buffer_space_request")
+                self.assertEqual(trailer, [])
+                self.assertEqual(current_request, None)
+                current_request = field_dict["destination"]
+
+            run_simulation(dut, [send_requests(), send_replies(), pr.receive(receive)])
+
+    def test_input(self):
+        for nwords in range(1, 8):
+            pt, pr, ts, dut = create_dut(nwords)
+
+            def read(chan_sel, timeout):
+                yield dut.cri.chan_sel.eq(chan_sel)
+                yield dut.cri.timestamp.eq(timeout)
+                yield dut.cri.cmd.eq(cri.commands["read"])
+                yield
+                yield dut.cri.cmd.eq(cri.commands["nop"])
+                yield
+                status = yield dut.cri.i_status
+                while status & 4:
+                    yield
+                    status = yield dut.cri.i_status
+                if status & 0x1:
+                    return "timeout"
+                if status & 0x2:
+                    return "overflow"
+                if status & 0x8:
+                    return "destination unreachable"
+                return ((yield dut.cri.i_data),
+                        (yield dut.cri.i_timestamp))
+
+            def send_requests():
+                for timeout in range(20, 200000, 100000):
+                    for chan_sel in range(3):
+                        data, timestamp = yield from read(chan_sel, timeout)
+                        self.assertEqual(data, chan_sel*2)
+                        self.assertEqual(timestamp, timeout//2)
+
+                i2 = yield from read(10, 400000)
+                self.assertEqual(i2, "timeout")
+                i3 = yield from read(11, 400000)
+                self.assertEqual(i3, "overflow")
+
+            current_request = None
+
+            @passive
+            def send_replies():
+                nonlocal current_request
+                while True:
+                    while current_request is None:
+                        yield
+                    chan_sel, timeout = current_request
+                    if chan_sel == 10:
+                        yield from pt.send("read_reply_noevent", overflow=0)
+                    elif chan_sel == 11:
+                        yield from pt.send("read_reply_noevent", overflow=1)
+                    else:
+                        yield from pt.send("read_reply", data=chan_sel*2, timestamp=timeout//2)
+                    current_request = None
+
+            def receive(packet_type, field_dict, trailer):
+                nonlocal current_request
+                self.assertEqual(packet_type, "read_request")
+                self.assertEqual(trailer, [])
+                self.assertEqual(current_request, None)
+                current_request = (field_dict["chan_sel"], field_dict["timeout"])
+
+            run_simulation(dut, [send_requests(), send_replies(), pr.receive(receive)])

artiq/gateware/test/drtio/test_rt_packet_satellite.py

@@ -0,0 +1,54 @@
+import unittest
+from types import SimpleNamespace
+
+from migen import *
+
+from artiq.gateware.test.drtio.packet_interface import PacketInterface
+from artiq.gateware.drtio.rt_packet_satellite import RTPacketSatellite
+
+
+def create_dut(nwords):
+    pt = PacketInterface("m2s", nwords*8)
+    pr = PacketInterface("s2m", nwords*8)
+    dut = RTPacketSatellite(SimpleNamespace(
+        rx_rt_frame=pt.frame, rx_rt_data=pt.data,
+        tx_rt_frame=pr.frame, tx_rt_data=pr.data))
+    return pt, pr, dut
+
+
+class TestSatellite(unittest.TestCase):
+    def test_echo(self):
+        for nwords in range(1, 8):
+            pt, pr, dut = create_dut(nwords)
+            completed = False
+            def send():
+                yield from pt.send("echo_request")
+                while not completed:
+                    yield
+            def receive(packet_type, field_dict, trailer):
+                nonlocal completed
+                self.assertEqual(packet_type, "echo_reply")
+                self.assertEqual(trailer, [])
+                completed = True
+            run_simulation(dut, [send(), pr.receive(receive)])
+
+    def test_set_time(self):
+        for nwords in range(1, 8):
+            pt, _, dut = create_dut(nwords)
+            tx_times = [0x12345678aabbccdd, 0x0102030405060708,
+                        0xaabbccddeeff1122]
+            def send():
+                for t in tx_times:
+                    yield from pt.send("set_time", timestamp=t)
+                # flush
+                for i in range(10):
+                    yield
+            rx_times = []
+            @passive
+            def receive():
+                while True:
+                    if (yield dut.tsc_load):
+                        rx_times.append((yield dut.tsc_load_value))
+                    yield
+            run_simulation(dut, [send(), receive()])
+            self.assertEqual(tx_times, rx_times)

artiq/gateware/test/drtio/test_switching.py

@@ -0,0 +1,247 @@
+import unittest
+from types import SimpleNamespace
+import random
+
+from migen import *
+
+from artiq.gateware.drtio import *
+from artiq.gateware.drtio import rt_serializer, rt_packet_repeater
+from artiq.gateware import rtio
+from artiq.gateware.rtio import cri
+from artiq.coredevice.exceptions import *
+from artiq.gateware.test.drtio.packet_interface import PacketInterface
+
+
+class DummyTransceiverPair:
+    def __init__(self, nwords):
+        a2b_k = [Signal() for _ in range(nwords)]
+        a2b_d = [Signal(8) for _ in range(nwords)]
+        b2a_k = [Signal() for _ in range(nwords)]
+        b2a_d = [Signal(8) for _ in range(nwords)]
+
+        self.alice = SimpleNamespace(
+            encoder=SimpleNamespace(k=a2b_k, d=a2b_d),
+            decoders=[SimpleNamespace(k=k, d=d) for k, d in zip(b2a_k, b2a_d)],
+            rx_ready=1
+        )
+        self.bob = SimpleNamespace(
+            encoder=SimpleNamespace(k=b2a_k, d=b2a_d),
+            decoders=[SimpleNamespace(k=k, d=d) for k, d in zip(a2b_k, a2b_d)],
+            rx_ready=1
+        )
+
+
+class DummyRXSynchronizer:
+    def resync(self, signal):
+        return signal
+
+
+class DUT(Module):
+    def __init__(self, nwords):
+        self.transceivers = DummyTransceiverPair(nwords)
+
+        self.submodules.tsc_master = rtio.TSC("async")
+        self.submodules.master = DRTIOMaster(self.tsc_master,
+                                             self.transceivers.alice)
+
+        rx_synchronizer = DummyRXSynchronizer()
+        self.submodules.tsc_satellite = rtio.TSC("sync")
+        self.submodules.satellite = DRTIOSatellite(
+            self.tsc_satellite, self.transceivers.bob, rx_synchronizer)
+        self.satellite.reset.storage.reset = 0
+        self.satellite.reset.storage_full.reset = 0
+        self.satellite.reset_phy.storage.reset = 0
+        self.satellite.reset_phy.storage_full.reset = 0
+
+        self.pt = PacketInterface("s2m", nwords*8)
+        self.pr = PacketInterface("m2s", nwords*8)
+        rep_if = SimpleNamespace(
+            rx_rt_frame=self.pt.frame, rx_rt_data=self.pt.data,
+            tx_rt_frame=self.pr.frame, tx_rt_data=self.pr.data)
+        self.submodules.repeater = rt_packet_repeater.RTPacketRepeater(
+            self.tsc_satellite, rep_if)
+        self.comb += self.satellite.cri.connect(self.repeater.cri)
+
+
+class Testbench:
+    def __init__(self):
+        self.dut = DUT(2)
+        self.now = 0
+
+    def init(self, with_buffer_space=True):
+        yield from self.dut.master.rt_controller.csrs.underflow_margin.write(100)
+        while not (yield from self.dut.master.link_layer.rx_up.read()):
+            yield
+        if with_buffer_space:
+            yield from self.get_buffer_space()
+
+    def get_buffer_space(self):
+        csrs = self.dut.master.rt_controller.csrs
+        yield from csrs.o_get_buffer_space.write(1)
+        yield
+        while (yield from csrs.o_wait.read()):
+            yield
+        r = (yield from csrs.o_dbg_buffer_space.read())
+        return r
+
+    def delay(self, dt):
+        self.now += dt
+
+    def write(self, channel, data):
+        mcri = self.dut.master.cri
+        yield mcri.chan_sel.eq(channel)
+        yield mcri.timestamp.eq(self.now)
+        yield mcri.o_data.eq(data)
+        yield
+        yield mcri.cmd.eq(cri.commands["write"])
+        yield
+        yield mcri.cmd.eq(cri.commands["nop"])
+        yield
+        status = yield mcri.o_status
+        while status & 0x1:
+            yield
+            status = yield mcri.o_status
+        if status & 0x2:
+            return "underflow"
+        if status & 0x4:
+            return "destination unreachable"
+
+    def read(self, channel, timeout):
+        mcri = self.dut.master.cri
+        yield mcri.chan_sel.eq(channel)
+        yield mcri.timestamp.eq(timeout)
+        yield
+        yield mcri.cmd.eq(cri.commands["read"])
+        yield
+        yield mcri.cmd.eq(cri.commands["nop"])
+        yield
+        status = yield mcri.i_status
+        while status & 0x4:
+            yield
+            status = yield mcri.i_status
+        if status & 0x1:
+            return "timeout"
+        if status & 0x2:
+            return "overflow"
+        if status & 0x8:
+            return "destination unreachable"
+        return ((yield mcri.i_timestamp),
+                (yield mcri.i_data))
+
+
+class TestSwitching(unittest.TestCase):
+    clocks = {"sys": 8, "rtio": 5, "rtio_rx": 5,
+              "rio": 5, "rio_phy": 5}
+
+    def test_outputs(self):
+        tb = Testbench()
+
+        def test():
+            yield from tb.init()
+            tb.delay(200)
+            yield from tb.write(1, 20)
+            for _ in range(40):
+                yield
+
+        current_request = None
+
+        def get_request():
+            nonlocal current_request
+            while current_request is None:
+                yield
+            r = current_request
+            current_request = None
+            return r
+
+        def expect_buffer_space_request(destination, space):
+            packet_type, field_dict, trailer = yield from get_request()
+            self.assertEqual(packet_type, "buffer_space_request")
+            self.assertEqual(trailer, [])
+            self.assertEqual(field_dict["destination"], destination)
+            yield from tb.dut.pt.send("buffer_space_reply", space=space)
+
+        def expect_write(timestamp, channel, data):
+            packet_type, field_dict, trailer = yield from get_request()
+            self.assertEqual(packet_type, "write")
+            self.assertEqual(trailer, [])
+            self.assertEqual(field_dict["timestamp"], timestamp)
+            self.assertEqual(field_dict["chan_sel"], channel)
+            self.assertEqual(field_dict["short_data"], data)
+
+        @passive
+        def send_replies():
+            yield from expect_buffer_space_request(0, 1)
+            yield from expect_write(200, 1, 20)
+            yield from expect_buffer_space_request(0, 1)
+
+            unexpected = yield from get_request()
+            self.fail("unexpected packet: {}".format(unexpected))
+
+        def receive(packet_type, field_dict, trailer):
+            nonlocal current_request
+            self.assertEqual(current_request, None)
+            current_request = (packet_type, field_dict, trailer)
+
+        run_simulation(tb.dut,
+            {"sys": test(), "rtio": tb.dut.pr.receive(receive), "rtio_rx": send_replies()}, self.clocks)
+
+
+    def test_inputs(self):
+        tb = Testbench()
+
+        def test():
+            yield from tb.init(with_buffer_space=False)
+            reply = yield from tb.read(19, 145)
+            self.assertEqual(reply, (333, 23))
+            reply = yield from tb.read(20, 146)
+            self.assertEqual(reply, (334, 24))
+            reply = yield from tb.read(10, 34)
+            self.assertEqual(reply, "timeout")
+            reply = yield from tb.read(1, 20)
+            self.assertEqual(reply, "overflow")
+            reply = yield from tb.read(21, 147)
+            self.assertEqual(reply, (335, 25))
+            for _ in range(40):
+                yield
+
+        current_request = None
+
+        def get_request():
+            nonlocal current_request
+            while current_request is None:
+                yield
+            r = current_request
+            current_request = None
+            return r
+
+        def expect_read(chan_sel, timeout, reply):
+            packet_type, field_dict, trailer = yield from get_request()
+            self.assertEqual(packet_type, "read_request")
+            self.assertEqual(trailer, [])
+            self.assertEqual(field_dict["chan_sel"], chan_sel)
+            self.assertEqual(field_dict["timeout"], timeout)
+            if reply == "timeout":
+                yield from tb.dut.pt.send("read_reply_noevent", overflow=0)
+            elif reply == "overflow":
+                yield from tb.dut.pt.send("read_reply_noevent", overflow=1)
+            else:
+                timestamp, data = reply
+                yield from tb.dut.pt.send("read_reply", timestamp=timestamp, data=data)
+
+        @passive
+        def send_replies():
+            yield from expect_read(19, 145, (333, 23))
+            yield from expect_read(20, 146, (334, 24))
+            yield from expect_read(10, 34, "timeout")
+            yield from expect_read(1, 20, "overflow")
+            yield from expect_read(21, 147, (335, 25))
+            unexpected = yield from get_request()
+            self.fail("unexpected packet: {}".format(unexpected))
+
+        def receive(packet_type, field_dict, trailer):
+            nonlocal current_request
+            self.assertEqual(current_request, None)
+            current_request = (packet_type, field_dict, trailer)
+
+        run_simulation(tb.dut,
+            {"sys": test(), "rtio": tb.dut.pr.receive(receive), "rtio_rx": send_replies()}, self.clocks)

artiq/gateware/test/rtio/test_dma.py

@@ -5,7 +5,7 @@ import itertools
 from migen import *
 from misoc.interconnect import wishbone
 
-from artiq.coredevice.exceptions import RTIOUnderflow, RTIOLinkError
+from artiq.coredevice.exceptions import RTIOUnderflow, RTIODestinationUnreachable
 from artiq.gateware import rtio
 from artiq.gateware.rtio import dma, cri
 from artiq.gateware.rtio.phy import ttl_simple
@@ -61,7 +61,7 @@ def do_dma(dut, address):
     if error & 1:
         raise RTIOUnderflow
     if error & 2:
-        raise RTIOLinkError
+        raise RTIODestinationUnreachable
 
 
 test_writes1 = [
@@ -127,7 +127,8 @@ class FullStackTB(Module):
         self.submodules.memory = wishbone.SRAM(
             256, init=sequence, bus=bus)
         self.submodules.dut = dma.DMA(bus)
-        self.submodules.rtio = rtio.Core(rtio_channels)
+        self.submodules.tsc = rtio.TSC("async")
+        self.submodules.rtio = rtio.Core(self.tsc, rtio_channels)
         self.comb += self.dut.cri.connect(self.rtio.cri)
 
 

artiq/gateware/test/rtio/test_input_collector.py

@@ -38,9 +38,8 @@ class DUT(Module):
             rtio.Channel.from_phy(self.phy0, ififo_depth=4),
             rtio.Channel.from_phy(self.phy1, ififo_depth=4)
         ]
-        self.submodules.input_collector = InputCollector(rtio_channels, 0, "sync")
-        self.sync += self.input_collector.coarse_timestamp.eq(self.input_collector.coarse_timestamp + 1)
-        self.comb += self.input_collector.cri.counter.eq(self.input_collector.coarse_timestamp)
+        self.submodules.tsc = ClockDomainsRenamer({"rtio": "sys"})(rtio.TSC("sync"))
+        self.submodules.input_collector = InputCollector(self.tsc, rtio_channels, "sync")
 
     @property
     def cri(self):

doc/manual/installing.rst

@@ -190,7 +190,7 @@ To flash the idle kernel:
 
 The startup kernel is executed once when the core device powers up. It should initialize DDSes, set up TTL directions, etc. Proceed as with the idle kernel, but using the ``startup_kernel`` key in the ``artiq_coremgmt`` command.
 
-For DRTIO systems, the startup kernel should wait until the desired links are up, using :meth:`artiq.coredevice.Core.get_drtio_link_status`.
+For DRTIO systems, the startup kernel should wait until the desired destinations (including local RTIO) are up, using :meth:`artiq.coredevice.Core.get_rtio_destination_status`.
 
 * (optional) Select the RTIO clock source
 

setup.py

@@ -30,6 +30,7 @@ console_scripts = [
     "artiq_mkfs = artiq.frontend.artiq_mkfs:main",
     "artiq_rtiomon = artiq.frontend.artiq_rtiomon:main",
     "artiq_session = artiq.frontend.artiq_session:main",
+    "artiq_route = artiq.frontend.artiq_route:main",
     "artiq_rpctool = artiq.frontend.artiq_rpctool:main",
     "artiq_run = artiq.frontend.artiq_run:main",
     "artiq_flash = artiq.frontend.artiq_flash:main",