Merge branch 'drtio'

artiq/examples/drtio/device_db.pyon

@@ -0,0 +1,148 @@
+# This is an example device database that needs to be adapted to your setup.
+# The RTIO channel numbers here are for NIST CLOCK on KC705.
+# The list of devices here is not exhaustive.
+
+{
+    "comm": {
+        "type": "local",
+        "module": "artiq.coredevice.comm_tcp",
+        "class": "Comm",
+        "arguments": {"host": "kc705.lab.m-labs.hk"}
+    },
+    "core": {
+        "type": "local",
+        "module": "artiq.coredevice.core",
+        "class": "Core",
+        "arguments": {"ref_period": 2e-9}
+    },
+    "core_cache": {
+        "type": "local",
+        "module": "artiq.coredevice.cache",
+        "class": "CoreCache"
+    },
+
+    "rled0": {
+        "type": "local",
+        "module": "artiq.coredevice.ttl",
+        "class": "TTLOut",
+        "arguments": {"channel": 0},
+    },
+    "rled1": {
+        "type": "local",
+        "module": "artiq.coredevice.ttl",
+        "class": "TTLOut",
+        "arguments": {"channel": 1},
+    },
+    "rled2": {
+        "type": "local",
+        "module": "artiq.coredevice.ttl",
+        "class": "TTLOut",
+        "arguments": {"channel": 2},
+    },
+    "rled3": {
+        "type": "local",
+        "module": "artiq.coredevice.ttl",
+        "class": "TTLOut",
+        "arguments": {"channel": 3},
+    },
+    "rled4": {
+        "type": "local",
+        "module": "artiq.coredevice.ttl",
+        "class": "TTLOut",
+        "arguments": {"channel": 4},
+    },
+    "rled5": {
+        "type": "local",
+        "module": "artiq.coredevice.ttl",
+        "class": "TTLOut",
+        "arguments": {"channel": 5},
+    },
+    "rled6": {
+        "type": "local",
+        "module": "artiq.coredevice.ttl",
+        "class": "TTLOut",
+        "arguments": {"channel": 6},
+    },
+    "rled7": {
+        "type": "local",
+        "module": "artiq.coredevice.ttl",
+        "class": "TTLOut",
+        "arguments": {"channel": 7},
+    },
+
+    "rsmap": {
+        "type": "local",
+        "module": "artiq.coredevice.ttl",
+        "class": "TTLOut",
+        "arguments": {"channel": 8}
+    },
+    "rsman": {
+        "type": "local",
+        "module": "artiq.coredevice.ttl",
+        "class": "TTLOut",
+        "arguments": {"channel": 9}
+    },
+
+    "led0": {
+        "type": "local",
+        "module": "artiq.coredevice.ttl",
+        "class": "TTLOut",
+        "arguments": {"channel": 0x010000},
+    },
+    "led1": {
+        "type": "local",
+        "module": "artiq.coredevice.ttl",
+        "class": "TTLOut",
+        "arguments": {"channel": 0x010001},
+    },
+    "led2": {
+        "type": "local",
+        "module": "artiq.coredevice.ttl",
+        "class": "TTLOut",
+        "arguments": {"channel": 0x010002},
+    },
+    "led3": {
+        "type": "local",
+        "module": "artiq.coredevice.ttl",
+        "class": "TTLOut",
+        "arguments": {"channel": 0x010003},
+    },
+    "led4": {
+        "type": "local",
+        "module": "artiq.coredevice.ttl",
+        "class": "TTLOut",
+        "arguments": {"channel": 0x010004},
+    },
+    "led5": {
+        "type": "local",
+        "module": "artiq.coredevice.ttl",
+        "class": "TTLOut",
+        "arguments": {"channel": 0x010005},
+    },
+    "led6": {
+        "type": "local",
+        "module": "artiq.coredevice.ttl",
+        "class": "TTLOut",
+        "arguments": {"channel": 0x010006},
+    },
+    "led7": {
+        "type": "local",
+        "module": "artiq.coredevice.ttl",
+        "class": "TTLOut",
+        "arguments": {"channel": 0x010007},
+    },
+
+    "smap": {
+        "type": "local",
+        "module": "artiq.coredevice.ttl",
+        "class": "TTLOut",
+        "arguments": {"channel": 0x010008}
+    },
+    "sman": {
+        "type": "local",
+        "module": "artiq.coredevice.ttl",
+        "class": "TTLOut",
+        "arguments": {"channel": 0x010009}
+    },
+
+}

artiq/examples/drtio/repository/blink_forever.py

@@ -0,0 +1,27 @@
+from artiq.experiment import *
+
+
+class BlinkForever(EnvExperiment):
+    def build(self):
+        self.setattr_device("core")
+        self.rleds = [self.get_device("rled" + str(i)) for i in range(8)]
+        self.leds = [self.get_device("led" + str(i)) for i in range(8)]
+
+    @kernel
+    def run(self):
+        self.core.reset()
+
+        while True:
+            with parallel:
+                for led in self.leds:
+                    led.pulse(250*ms)
+                for led in self.rleds:
+                    led.pulse(250*ms)
+            t = now_mu()
+            for led in self.leds:
+                at_mu(t)
+                led.pulse(500*ms)
+            for led in self.rleds:
+                at_mu(t)
+                led.pulse(500*ms)
+            delay(250*ms)

artiq/examples/drtio/repository/pulse_rate.py

@@ -0,0 +1,25 @@
+from artiq.experiment import *
+
+
+class PulseRate(EnvExperiment):
+    def build(self):
+        self.setattr_device("core")
+        self.setattr_device("rsmap")
+
+    @kernel
+    def run(self):
+        self.core.reset()
+
+        dt = self.core.seconds_to_mu(300*ns)
+        while True:
+            for i in range(10000):
+                try:
+                    self.rsmap.pulse_mu(dt)
+                    delay_mu(dt)
+                except RTIOUnderflow:
+                    dt += 1
+                    self.core.break_realtime()
+                    break
+            else:
+                print(self.core.mu_to_seconds(dt))
+                return

artiq/gateware/drtio/__init__.py

@@ -0,0 +1,2 @@
+from artiq.gateware.drtio.core import DRTIOSatellite, DRTIOMaster
+

artiq/gateware/drtio/aux_controller.py

@@ -0,0 +1,226 @@
+from migen import *
+from migen.fhdl.simplify import FullMemoryWE
+from migen.genlib.cdc import MultiReg, PulseSynchronizer
+
+from misoc.interconnect.csr import *
+from misoc.interconnect import stream 
+from misoc.interconnect import wishbone
+
+
+max_packet = 1024
+
+
+class Transmitter(Module, AutoCSR):
+    def __init__(self, link_layer, min_mem_dw):
+        ll_dw = len(link_layer.tx_aux_data)
+        mem_dw = max(min_mem_dw, ll_dw)
+
+        self.aux_tx_length = CSRStorage(bits_for(max_packet),
+                                        alignment_bits=log2_int(mem_dw//8))
+        self.aux_tx = CSR()
+        self.specials.mem = Memory(mem_dw, max_packet//(mem_dw//8))
+
+        converter = stream.Converter(mem_dw, ll_dw)
+        self.submodules += converter
+
+        # when continuously fed, the Converter outputs data continuously
+        self.comb += [
+            converter.source.ack.eq(link_layer.tx_aux_ack),
+            link_layer.tx_aux_frame.eq(converter.source.stb),
+            link_layer.tx_aux_data.eq(converter.source.data)
+        ]
+
+        seen_eop_rst = Signal()
+        frame_r = Signal()
+        seen_eop = Signal()
+        self.sync.rtio += [
+            If(link_layer.tx_aux_ack,
+                frame_r.eq(link_layer.tx_aux_frame),
+                If(frame_r & ~link_layer.tx_aux_frame, seen_eop.eq(1))
+            ),
+            If(seen_eop_rst, seen_eop.eq(0))
+        ]
+
+        mem_port = self.mem.get_port(clock_domain="rtio")
+        self.specials += mem_port
+
+        self.aux_tx_length.storage.attr.add("no_retiming")
+        tx_length = Signal(bits_for(max_packet))
+        self.specials += MultiReg(self.aux_tx_length.storage, tx_length, "rtio")
+
+        frame_counter_nbits = bits_for(max_packet) - log2_int(mem_dw//8)
+        frame_counter = Signal(frame_counter_nbits)
+        frame_counter_next = Signal(frame_counter_nbits)
+        frame_counter_ce = Signal()
+        frame_counter_rst = Signal()
+        self.comb += [
+            frame_counter_next.eq(frame_counter),
+            If(frame_counter_rst,
+                frame_counter_next.eq(0)
+            ).Elif(frame_counter_ce,
+                frame_counter_next.eq(frame_counter + 1)
+            ),
+            mem_port.adr.eq(frame_counter_next),
+            converter.sink.data.eq(mem_port.dat_r)
+        ]
+        self.sync.rtio += frame_counter.eq(frame_counter_next)
+
+        start_tx = PulseSynchronizer("sys", "rtio")
+        tx_done = PulseSynchronizer("rtio", "sys")
+        self.submodules += start_tx, tx_done
+        self.comb += start_tx.i.eq(self.aux_tx.re)
+        self.sync += [
+            If(tx_done.o, self.aux_tx.w.eq(0)),
+            If(self.aux_tx.re, self.aux_tx.w.eq(1))
+        ]
+
+        fsm = ClockDomainsRenamer("rtio")(FSM(reset_state="IDLE"))
+        self.submodules += fsm
+
+        fsm.act("IDLE",
+            frame_counter_rst.eq(1),
+            seen_eop_rst.eq(1),
+            If(start_tx.o, NextState("TRANSMIT"))
+        )
+        fsm.act("TRANSMIT",
+            converter.sink.stb.eq(1),
+            If(converter.sink.ack,
+                frame_counter_ce.eq(1)
+            ),
+            If(frame_counter_next == tx_length, NextState("WAIT_INTERFRAME"))
+        )
+        fsm.act("WAIT_INTERFRAME",
+            If(seen_eop,
+                tx_done.i.eq(1),
+                NextState("IDLE")
+            )
+        )
+
+
+class Receiver(Module, AutoCSR):
+    def __init__(self, link_layer, min_mem_dw):
+        self.aux_rx_length = CSRStatus(bits_for(max_packet))
+        self.aux_rx_present = CSR()
+        self.aux_rx_error = CSR()
+
+        ll_dw = len(link_layer.rx_aux_data)
+        mem_dw = max(min_mem_dw, ll_dw)
+        self.specials.mem = Memory(mem_dw, max_packet//(mem_dw//8))
+
+        converter = stream.Converter(ll_dw, mem_dw)
+        self.submodules += converter
+
+        # when continuously drained, the Converter accepts data continuously
+        frame_r = Signal()
+        self.sync.rtio_rx += [
+            If(link_layer.rx_aux_stb,
+                frame_r.eq(link_layer.rx_aux_frame),
+                converter.sink.data.eq(link_layer.rx_aux_data)
+            )
+        ]
+        self.comb += [
+            converter.sink.stb.eq(link_layer.rx_aux_stb & frame_r),
+            converter.sink.eop.eq(converter.sink.stb & ~link_layer.rx_aux_frame)
+        ]
+
+        mem_port = self.mem.get_port(write_capable=True, clock_domain="rtio_rx")
+        self.specials += mem_port
+
+        frame_counter_nbits = bits_for(max_packet) - log2_int(mem_dw//8)
+        frame_counter = Signal(frame_counter_nbits)
+        self.comb += [
+            mem_port.adr.eq(frame_counter),
+            mem_port.dat_w.eq(converter.source.data),
+            converter.source.ack.eq(1)
+        ]
+
+        frame_counter.attr.add("no_retiming")
+        frame_counter_sys = Signal(frame_counter_nbits)
+        self.specials += MultiReg(frame_counter, frame_counter_sys)
+        self.comb += self.aux_rx_length.status.eq(frame_counter_sys << log2_int(mem_dw//8))
+
+        signal_frame = PulseSynchronizer("rtio_rx", "sys")
+        frame_ack = PulseSynchronizer("sys", "rtio_rx")
+        signal_error = PulseSynchronizer("rtio_rx", "sys")
+        self.submodules += signal_frame, frame_ack, signal_error
+        self.sync += [
+            If(self.aux_rx_present.re, self.aux_rx_present.w.eq(0)),
+            If(signal_frame.o, self.aux_rx_present.w.eq(1)),
+            If(self.aux_rx_error.re, self.aux_rx_error.w.eq(0)),
+            If(signal_error.o, self.aux_rx_error.w.eq(1))
+        ]
+        self.comb += frame_ack.i.eq(self.aux_rx_present.re)
+
+        fsm = ClockDomainsRenamer("rtio_rx")(FSM(reset_state="IDLE"))
+        self.submodules += fsm
+
+        sop = Signal(reset=1)
+        self.sync.rtio_rx += \
+            If(converter.source.stb,
+                If(converter.source.eop,
+                    sop.eq(1)
+                ).Else(
+                    sop.eq(0)
+                )
+            )
+
+        fsm.act("IDLE",
+            If(converter.source.stb & sop,
+                NextValue(frame_counter, frame_counter + 1),
+                mem_port.we.eq(1),
+                If(converter.source.eop,
+                    NextState("SIGNAL_FRAME")
+                ).Else(
+                    NextState("FRAME")
+                )
+            ).Else(
+                NextValue(frame_counter, 0)
+            )
+        )
+        fsm.act("FRAME",
+            If(converter.source.stb,
+                NextValue(frame_counter, frame_counter + 1),
+                mem_port.we.eq(1),
+                If(frame_counter == max_packet,
+                    mem_port.we.eq(0),
+                    signal_error.i.eq(1),
+                    NextState("IDLE")  # discard the rest of the frame
+                ),
+                If(converter.source.eop,
+                    NextState("SIGNAL_FRAME")
+                )
+            )
+        )
+        fsm.act("SIGNAL_FRAME",
+            signal_frame.i.eq(1),
+            NextState("WAIT_ACK"),
+            If(converter.source.stb, signal_error.i.eq(1))
+        )
+        fsm.act("WAIT_ACK",
+            If(frame_ack.o,
+                NextValue(frame_counter, 0), 
+                NextState("IDLE")
+            ),
+            If(converter.source.stb, signal_error.i.eq(1))
+        )
+
+
+# TODO: FullMemoryWE should be applied by migen.build
+@FullMemoryWE()
+class AuxController(Module):
+    def __init__(self, link_layer):
+        self.bus = wishbone.Interface()
+        self.submodules.transmitter = Transmitter(link_layer, len(self.bus.dat_w))
+        self.submodules.receiver = Receiver(link_layer, len(self.bus.dat_w))
+
+        tx_sdram_if = wishbone.SRAM(self.transmitter.mem, read_only=False)
+        rx_sdram_if = wishbone.SRAM(self.receiver.mem, read_only=True)
+        wsb = log2_int(len(self.bus.dat_w)//8)
+        decoder = wishbone.Decoder(self.bus,
+            [(lambda a: a[log2_int(max_packet)-wsb] == 0, tx_sdram_if.bus),
+             (lambda a: a[log2_int(max_packet)-wsb] == 1, rx_sdram_if.bus)],
+            register=True)
+        self.submodules += tx_sdram_if, rx_sdram_if, decoder
+
+    def get_csrs(self):
+        return self.transmitter.get_csrs() + self.receiver.get_csrs()

artiq/gateware/drtio/core.py

@@ -0,0 +1,68 @@
+from types import SimpleNamespace
+
+from migen import *
+
+from artiq.gateware.drtio import link_layer, rt_packets, iot, rt_controller, aux_controller
+
+
+class DRTIOSatellite(Module):
+    def __init__(self, transceiver, rx_synchronizer, channels, fine_ts_width=3, full_ts_width=63):
+        self.submodules.link_layer = link_layer.LinkLayer(
+            transceiver.encoder, transceiver.decoders)
+        self.comb += self.link_layer.rx_ready.eq(transceiver.rx_ready)
+
+        link_layer_sync = SimpleNamespace(
+            tx_aux_frame=self.link_layer.tx_aux_frame,
+            tx_aux_data=self.link_layer.tx_aux_data,
+            tx_aux_ack=self.link_layer.tx_aux_ack,
+            tx_rt_frame=self.link_layer.tx_rt_frame,
+            tx_rt_data=self.link_layer.tx_rt_data,
+
+            rx_aux_stb=rx_synchronizer.resync(self.link_layer.rx_aux_stb),
+            rx_aux_frame=rx_synchronizer.resync(self.link_layer.rx_aux_frame),
+            rx_aux_data=rx_synchronizer.resync(self.link_layer.rx_aux_data),
+            rx_rt_frame=rx_synchronizer.resync(self.link_layer.rx_rt_frame),
+            rx_rt_data=rx_synchronizer.resync(self.link_layer.rx_rt_data)
+        )
+        self.submodules.rt_packets = ClockDomainsRenamer("rtio")(
+            rt_packets.RTPacketSatellite(link_layer_sync))
+
+        self.submodules.iot = iot.IOT(
+            self.rt_packets, channels, fine_ts_width, full_ts_width)
+
+        self.clock_domains.cd_rio = ClockDomain()
+        self.clock_domains.cd_rio_phy = ClockDomain()
+        self.comb += [
+            self.cd_rio.clk.eq(ClockSignal("rtio")),
+            self.cd_rio.rst.eq(self.rt_packets.reset),
+            self.cd_rio_phy.clk.eq(ClockSignal("rtio")),
+            self.cd_rio_phy.rst.eq(self.rt_packets.reset_phy),
+        ]
+
+        self.submodules.aux_controller = aux_controller.AuxController(
+            self.link_layer)
+
+    def get_csrs(self):
+        return self.aux_controller.get_csrs()
+
+
+class DRTIOMaster(Module):
+    def __init__(self, transceiver, channel_count=1024, fine_ts_width=3):
+        self.submodules.link_layer = link_layer.LinkLayer(
+            transceiver.encoder, transceiver.decoders)
+        self.comb += self.link_layer.rx_ready.eq(transceiver.rx_ready)
+
+        self.submodules.rt_packets = rt_packets.RTPacketMaster(self.link_layer)
+        self.submodules.rt_controller = rt_controller.RTController(
+            self.rt_packets, channel_count, fine_ts_width)
+        self.submodules.rt_manager = rt_controller.RTManager(self.rt_packets)
+        self.cri = self.rt_controller.cri
+
+        self.submodules.aux_controller = aux_controller.AuxController(
+            self.link_layer)
+
+    def get_csrs(self):
+        return (self.link_layer.get_csrs() +
+                self.rt_controller.get_csrs() +
+                self.rt_manager.get_csrs() +
+                self.aux_controller.get_csrs())

artiq/gateware/drtio/iot.py

@@ -0,0 +1,86 @@
+from migen import *
+from migen.genlib.fifo import SyncFIFOBuffered
+from migen.genlib.record import *
+
+from artiq.gateware.rtio import rtlink
+
+
+class IOT(Module):
+    def __init__(self, rt_packets, channels, max_fine_ts_width, full_ts_width):
+        tsc = Signal(full_ts_width - max_fine_ts_width)
+        self.sync.rtio += \
+            If(rt_packets.tsc_load,
+                tsc.eq(rt_packets.tsc_value)
+            ).Else(
+                tsc.eq(tsc + 1)
+            )
+
+        for n, channel in enumerate(channels):
+            interface = channel.interface.o
+            data_width = rtlink.get_data_width(interface)
+            address_width = rtlink.get_address_width(interface)
+            fine_ts_width = rtlink.get_fine_ts_width(interface)
+            assert fine_ts_width <= max_fine_ts_width
+
+            # FIFO
+            ev_layout = []
+            if data_width:
+                ev_layout.append(("data", data_width))
+            if address_width:
+                ev_layout.append(("address", address_width))
+            ev_layout.append(("timestamp", len(tsc) + fine_ts_width))
+
+            fifo = ClockDomainsRenamer("rio")(
+                SyncFIFOBuffered(layout_len(ev_layout), channel.ofifo_depth))
+            self.submodules += fifo
+            fifo_in = Record(ev_layout)
+            fifo_out = Record(ev_layout)
+            self.comb += [
+                fifo.din.eq(fifo_in.raw_bits()),
+                fifo_out.raw_bits().eq(fifo.dout)
+            ]
+
+            # FIFO level
+            self.sync.rio += \
+                If(rt_packets.fifo_space_update &
+                   (rt_packets.fifo_space_channel == n),
+                    rt_packets.fifo_space.eq(channel.ofifo_depth - fifo.level))
+
+            # FIFO write
+            self.comb += fifo.we.eq(rt_packets.write_stb
+                                    & (rt_packets.write_channel == n))
+            self.sync.rio += [
+                If(rt_packets.write_overflow_ack,
+                    rt_packets.write_overflow.eq(0)),
+                If(rt_packets.write_underflow_ack,
+                    rt_packets.write_underflow.eq(0)),
+                If(fifo.we,
+                    If(~fifo.writable, rt_packets.write_overflow.eq(1)),
+                    If(rt_packets.write_timestamp[max_fine_ts_width:] < (tsc + 4),
+                        rt_packets.write_underflow.eq(1)
+                    )
+                )
+            ]
+            if data_width:
+                self.comb += fifo_in.data.eq(rt_packets.write_data)
+            if address_width:
+                self.comb += fifo_in.address.eq(rt_packets.write_address)
+            self.comb += fifo_in.timestamp.eq(
+                rt_packets.write_timestamp[max_fine_ts_width-fine_ts_width:])
+
+            # FIFO read
+            self.sync.rio += [
+                fifo.re.eq(0),
+                interface.stb.eq(0),
+                If(fifo.readable &
+                   (fifo_out.timestamp[fine_ts_width:] == tsc),
+                    fifo.re.eq(1),
+                    interface.stb.eq(1)
+                )
+            ]
+            if data_width:
+                self.sync.rio += interface.data.eq(fifo_out.data)
+            if address_width:
+                self.sync.rio += interface.address.eq(fifo_out.address)
+            if fine_ts_width:
+                self.sync.rio += interface.fine_ts.eq(fifo_out.timestamp[:fine_ts_width])

artiq/gateware/drtio/link_layer.py

@@ -0,0 +1,278 @@
+from functools import reduce
+from operator import xor, or_
+
+from migen import *
+from migen.genlib.fsm import *
+from migen.genlib.cdc import MultiReg
+from migen.genlib.misc import WaitTimer
+
+from misoc.interconnect.csr import *
+
+
+class Scrambler(Module):
+    def __init__(self, n_io1, n_io2, n_state=23, taps=[17, 22]):
+        self.i1 = Signal(n_io1)
+        self.o1 = Signal(n_io1)
+        self.i2 = Signal(n_io2)
+        self.o2 = Signal(n_io2)
+        self.sel = Signal()
+
+        # # #
+
+        state = Signal(n_state, reset=1)
+
+        stmts1 = []
+        stmts2 = []
+        for stmts, si, so in ((stmts1, self.i1, self.o1),
+                              (stmts2, self.i2, self.o2)):
+            curval = [state[i] for i in range(n_state)]
+            for i in reversed(range(len(si))):
+                out = si[i] ^ reduce(xor, [curval[tap] for tap in taps])
+                stmts += [so[i].eq(out)]
+                curval.insert(0, out)
+                curval.pop()
+
+            stmts += [state.eq(Cat(*curval[:n_state]))]
+
+        self.sync += If(self.sel, stmts2).Else(stmts1)
+
+
+class Descrambler(Module):
+    def __init__(self, n_io1, n_io2, n_state=23, taps=[17, 22]):
+        self.i1 = Signal(n_io1)
+        self.o1 = Signal(n_io1)
+        self.i2 = Signal(n_io2)
+        self.o2 = Signal(n_io2)
+        self.sel = Signal()
+
+        # # #
+
+        state = Signal(n_state, reset=1)
+
+        stmts1 = []
+        stmts2 = []
+        for stmts, si, so in ((stmts1, self.i1, self.o1),
+                              (stmts2, self.i2, self.o2)):
+            curval = [state[i] for i in range(n_state)]
+            for i in reversed(range(len(si))):
+                flip = reduce(xor, [curval[tap] for tap in taps])
+                stmts += [so[i].eq(si[i] ^ flip)]
+                curval.insert(0, si[i])
+                curval.pop()
+
+            stmts += [state.eq(Cat(*curval[:n_state]))]
+
+        self.sync += If(self.sel, stmts2).Else(stmts1)
+
+
+def K(x, y):
+    return (y << 5) | x
+
+
+aux_coding_comma = [
+    K(28, 5),
+    K(28, 0),
+    K(28, 1),
+    K(28, 2),
+    K(23, 7),
+    K(27, 7),
+    K(29, 7),
+    K(30, 7),
+]
+
+
+aux_coding_nocomma = [
+    K(28, 0),
+    K(28, 2),
+    K(28, 3),
+    K(28, 4),
+    K(23, 7),
+    K(27, 7),
+    K(29, 7),
+    K(30, 7),
+]
+
+
+class LinkLayerTX(Module):
+    def __init__(self, encoder):
+        nwords = len(encoder.k)
+        # nwords must be a power of 2
+        assert nwords & (nwords - 1) == 0
+
+        self.aux_frame = Signal()
+        self.aux_data = Signal(2*nwords)
+        self.aux_ack = Signal()
+
+        self.rt_frame = Signal()
+        self.rt_data = Signal(8*nwords)
+
+        # # #
+
+        # Idle and auxiliary traffic use special characters defined in the
+        # aux_coding_* tables.
+        # The first (or only) character uses aux_coding_comma which guarantees
+        # that commas appear regularly in the absence of traffic.
+        # The subsequent characters, if any (depending on the transceiver
+        # serialization ratio) use aux_coding_nocomma which does not contain
+        # commas. This permits aligning the comma to the first character at
+        # the receiver.
+        #
+        # A set of 8 special characters is chosen using a 3-bit control word.
+        # This control word is scrambled to reduce EMI. The control words have
+        # the following meanings:
+        #   100 idle/auxiliary framing
+        #   0AB 2 bits of auxiliary data
+        #
+        # RT traffic uses D characters and is also scrambled. The aux and RT
+        # scramblers are multiplicative and share the same state so that idle
+        # or aux traffic can synchronize the RT descrambler.
+
+        scrambler = Scrambler(3*nwords, 8*nwords)
+        self.submodules += scrambler
+
+        # scrambler input
+        aux_data_ctl = []
+        for i in range(nwords):
+            aux_data_ctl.append(self.aux_data[i*2:i*2+2])
+            aux_data_ctl.append(0)
+        self.comb += [
+            If(self.aux_frame,
+                scrambler.i1.eq(Cat(*aux_data_ctl))
+            ).Else(
+                scrambler.i1.eq(Replicate(0b100, nwords))
+            ),
+            scrambler.i2.eq(self.rt_data),
+            scrambler.sel.eq(self.rt_frame),
+            self.aux_ack.eq(~self.rt_frame)
+        ]
+
+        # compensate for scrambler latency
+        rt_frame_r = Signal()
+        self.sync += rt_frame_r.eq(self.rt_frame)
+
+        # scrambler output
+        for i in range(nwords):
+            scrambled_ctl = scrambler.o1[i*3:i*3+3]
+            if i:
+                aux_coding = aux_coding_nocomma
+            else:
+                aux_coding = aux_coding_comma
+            self.sync += [
+                encoder.k[i].eq(1),
+                encoder.d[i].eq(Array(aux_coding)[scrambled_ctl])
+            ]
+        self.sync += \
+            If(rt_frame_r,
+                [k.eq(0) for k in encoder.k],
+                [d.eq(scrambler.o2[i*8:i*8+8]) for i, d in enumerate(encoder.d)]
+            )
+
+
+class LinkLayerRX(Module):
+    def __init__(self, decoders):
+        nwords = len(decoders)
+        # nwords must be a power of 2
+        assert nwords & (nwords - 1) == 0
+
+        self.aux_stb = Signal()
+        self.aux_frame = Signal()
+        self.aux_data = Signal(2*nwords)
+
+        self.rt_frame = Signal()
+        self.rt_data = Signal(8*nwords)
+
+        # # #
+
+        descrambler = Descrambler(3*nwords, 8*nwords)
+        self.submodules += descrambler
+
+        # scrambler input
+        all_decoded_aux = []
+        for i, d in enumerate(decoders):
+            decoded_aux = Signal(3)
+            all_decoded_aux.append(decoded_aux)
+
+            if i:
+                aux_coding = aux_coding_nocomma
+            else:
+                aux_coding = aux_coding_comma
+
+            cases = {code: decoded_aux.eq(i) for i, code in enumerate(aux_coding)}
+            self.comb += Case(d.d, cases).makedefault()
+
+        self.comb += [
+            descrambler.i1.eq(Cat(*all_decoded_aux)),
+            descrambler.i2.eq(Cat(*[d.d for d in decoders])),
+            descrambler.sel.eq(~decoders[0].k)
+        ]
+
+        # scrambler output
+        self.comb += [
+            self.aux_frame.eq(~descrambler.o1[2]),
+            self.aux_data.eq(
+                Cat(*[descrambler.o1[3*i:3*i+2] for i in range(nwords)])),
+            self.rt_data.eq(descrambler.o2)
+        ]
+        self.sync += [
+            self.aux_stb.eq(decoders[0].k),
+            self.rt_frame.eq(~decoders[0].k)
+        ]
+
+
+
+class LinkLayer(Module, AutoCSR):
+    def __init__(self, encoder, decoders):
+        self.link_status = CSRStatus()
+
+        # receiver locked, comma aligned, receiving valid 8b10b symbols
+        self.rx_ready = Signal()
+
+        tx = ClockDomainsRenamer("rtio")(LinkLayerTX(encoder))
+        rx = ClockDomainsRenamer("rtio_rx")(LinkLayerRX(decoders))
+        self.submodules += tx, rx
+
+        # in rtio clock domain
+        self.tx_aux_frame = tx.aux_frame
+        self.tx_aux_data = tx.aux_data
+        self.tx_aux_ack = tx.aux_ack
+        self.tx_rt_frame = tx.rt_frame
+        self.tx_rt_data = tx.rt_data
+
+        # in rtio_rx clock domain
+        self.rx_aux_stb = rx.aux_stb
+        self.rx_aux_frame = Signal()
+        self.rx_aux_data = rx.aux_data
+        self.rx_rt_frame = Signal()
+        self.rx_rt_data = rx.rt_data
+
+        # # #
+
+        ready = Signal()
+        ready_r = Signal()
+        self.sync.rtio += ready_r.eq(ready)
+        ready_rx = Signal()
+        ready_r.attr.add("no_retiming")
+        self.specials += MultiReg(ready_r, ready_rx, "rtio_rx")
+        self.comb += [
+            self.rx_aux_frame.eq(rx.aux_frame & ready_rx),
+            self.rx_rt_frame.eq(rx.rt_frame & ready_rx),
+        ]
+        self.specials += MultiReg(ready_r, self.link_status.status)
+
+        wait_scrambler = ClockDomainsRenamer("rtio")(WaitTimer(15))
+        self.submodules += wait_scrambler
+
+        fsm = ClockDomainsRenamer("rtio")(FSM(reset_state="WAIT_RX_READY"))
+        self.submodules += fsm
+
+        fsm.act("WAIT_RX_READY",
+            If(self.rx_ready, NextState("WAIT_SCRAMBLER_SYNC"))
+        )
+        fsm.act("WAIT_SCRAMBLER_SYNC",
+            wait_scrambler.wait.eq(1),
+            If(wait_scrambler.done, NextState("READY"))
+        )
+        fsm.act("READY",
+            ready.eq(1),
+            If(~self.rx_ready, NextState("WAIT_RX_READY"))
+        )

artiq/gateware/drtio/rt_controller.py

@@ -0,0 +1,233 @@
+from migen import *
+from migen.genlib.cdc import MultiReg
+from migen.genlib.misc import WaitTimer
+
+from misoc.interconnect.csr import *
+
+from artiq.gateware.rtio.cdc import RTIOCounter
+from artiq.gateware.rtio import cri
+
+
+class _CSRs(AutoCSR):
+    def __init__(self):
+        self.chan_sel_override = CSRStorage(16)
+        self.chan_sel_override_en = CSRStorage()
+
+        self.tsc_correction = CSRStorage(64)
+        self.set_time = CSR()
+        self.underflow_margin = CSRStorage(16, reset=200)
+
+        self.o_get_fifo_space = CSR()
+        self.o_dbg_fifo_space = CSRStatus(16)
+        self.o_dbg_last_timestamp = CSRStatus(64)
+        self.o_reset_channel_status = CSR()
+        self.o_wait = CSRStatus()
+        self.o_fifo_space_timeout = CSR()
+
+
+class RTController(Module):
+    def __init__(self, rt_packets, channel_count, fine_ts_width):
+        self.csrs = _CSRs()
+        self.cri = cri.Interface()
+        self.comb += self.cri.arb_gnt.eq(1)
+
+        # channel selection
+        chan_sel = Signal(16)
+        self.comb += chan_sel.eq(
+            Mux(self.csrs.chan_sel_override_en.storage,
+                self.csrs.chan_sel_override.storage,
+                self.cri.chan_sel[:16]))
+
+        # master RTIO counter and counter synchronization
+        self.submodules.counter = RTIOCounter(64-fine_ts_width)
+        self.comb += self.cri.counter.eq(self.counter.value_sys << fine_ts_width)
+        tsc_correction = Signal(64)
+        self.csrs.tsc_correction.storage.attr.add("no_retiming")
+        self.specials += MultiReg(self.csrs.tsc_correction.storage, tsc_correction)
+        self.comb += [ 
+            rt_packets.tsc_value.eq(
+                self.counter.value_rtio + tsc_correction),
+            self.csrs.set_time.w.eq(rt_packets.set_time_stb)
+        ]
+        self.sync += [
+            If(rt_packets.set_time_ack, rt_packets.set_time_stb.eq(0)),
+            If(self.csrs.set_time.re, rt_packets.set_time_stb.eq(1))
+        ]
+
+        # reset
+        self.sync += [
+            If(rt_packets.reset_ack, rt_packets.reset_stb.eq(0)),
+            If(self.cri.cmd == cri.commands["reset"],
+                rt_packets.reset_stb.eq(1),
+                rt_packets.reset_phy.eq(0)
+            ),
+            If(self.cri.cmd == cri.commands["reset_phy"],
+                rt_packets.reset_stb.eq(1),
+                rt_packets.reset_phy.eq(1)
+            ),
+        ]
+
+        # remote channel status cache
+        fifo_spaces_mem = Memory(16, channel_count)
+        fifo_spaces = fifo_spaces_mem.get_port(write_capable=True)
+        self.specials += fifo_spaces_mem, fifo_spaces
+        last_timestamps_mem = Memory(64, channel_count)
+        last_timestamps = last_timestamps_mem.get_port(write_capable=True)
+        self.specials += last_timestamps_mem, last_timestamps
+
+        # common packet fields
+        rt_packets_fifo_request = Signal()
+        self.comb += [
+            fifo_spaces.adr.eq(chan_sel),
+            last_timestamps.adr.eq(chan_sel),
+            last_timestamps.dat_w.eq(self.cri.o_timestamp),
+            rt_packets.write_channel.eq(chan_sel),
+            rt_packets.write_address.eq(self.cri.o_address),
+            rt_packets.write_data.eq(self.cri.o_data),
+            If(rt_packets_fifo_request,
+                rt_packets.write_timestamp.eq(0xffff000000000000)
+            ).Else(
+                rt_packets.write_timestamp.eq(self.cri.o_timestamp)
+            )
+        ]
+
+        fsm = FSM()
+        self.submodules += fsm
+
+        status_wait = Signal()
+        status_underflow = Signal()
+        status_sequence_error = Signal()
+        self.comb += [
+            self.cri.o_status.eq(Cat(
+                status_wait, status_underflow, status_sequence_error)),
+            self.csrs.o_wait.status.eq(status_wait)
+        ]
+        sequence_error_set = Signal()
+        underflow_set = Signal()
+        self.sync += [
+            If(self.cri.cmd == cri.commands["o_underflow_reset"], status_underflow.eq(0)),
+            If(self.cri.cmd == cri.commands["o_sequence_error_reset"], status_sequence_error.eq(0)),
+            If(underflow_set, status_underflow.eq(1)),
+            If(sequence_error_set, status_sequence_error.eq(1)),
+        ]
+
+        signal_fifo_space_timeout = Signal()
+        self.sync += [
+            If(self.csrs.o_fifo_space_timeout.re, self.csrs.o_fifo_space_timeout.w.eq(0)),
+            If(signal_fifo_space_timeout, self.csrs.o_fifo_space_timeout.w.eq(1))
+        ]
+        timeout_counter = WaitTimer(8191)
+        self.submodules += timeout_counter
+
+        # TODO: collision, replace, busy
+        cond_sequence_error = self.cri.o_timestamp < last_timestamps.dat_r
+        cond_underflow = ((self.cri.o_timestamp[fine_ts_width:]
+                           - self.csrs.underflow_margin.storage[fine_ts_width:]) < self.counter.value_sys)
+        cond_fifo_emptied = ((last_timestamps.dat_r[fine_ts_width:] < self.counter.value_sys)
+                             & (last_timestamps.dat_r != 0))
+
+        fsm.act("IDLE",
+            If(self.cri.cmd == cri.commands["write"],
+                If(cond_sequence_error,
+                    sequence_error_set.eq(1)
+                ).Elif(cond_underflow,
+                    underflow_set.eq(1)
+                ).Else(
+                    NextState("WRITE")
+                )
+            ),
+            If(self.csrs.o_get_fifo_space.re,
+                NextState("GET_FIFO_SPACE")
+            )
+        )
+        fsm.act("WRITE",
+            status_wait.eq(1),
+            rt_packets.write_stb.eq(1),
+            If(rt_packets.write_ack,
+                fifo_spaces.we.eq(1),
+                If(cond_fifo_emptied,
+                    fifo_spaces.dat_w.eq(1),
+                ).Else(
+                    fifo_spaces.dat_w.eq(fifo_spaces.dat_r - 1)
+                ),
+                last_timestamps.we.eq(1),
+                If(~cond_fifo_emptied & (fifo_spaces.dat_r <= 1),
+                    NextState("GET_FIFO_SPACE")
+                ).Else(
+                    NextState("IDLE")
+                )
+            )
+        )
+        fsm.act("GET_FIFO_SPACE",
+            status_wait.eq(1),
+            rt_packets_fifo_request.eq(1),
+            rt_packets.write_stb.eq(1),
+            If(rt_packets.write_ack,
+                NextState("GET_FIFO_SPACE_REPLY")
+            )
+        )
+        fsm.act("GET_FIFO_SPACE_REPLY",
+            status_wait.eq(1),
+            fifo_spaces.dat_w.eq(rt_packets.fifo_space),
+            fifo_spaces.we.eq(1),
+            rt_packets.fifo_space_not_ack.eq(1),
+            If(rt_packets.fifo_space_not,
+                If(rt_packets.fifo_space > 0,
+                    NextState("IDLE")
+                ).Else(
+                    NextState("GET_FIFO_SPACE")
+                )
+            ),
+            timeout_counter.wait.eq(1),
+            If(timeout_counter.done,
+                signal_fifo_space_timeout.eq(1),
+                NextState("IDLE")
+            )
+        )
+
+        # channel state access
+        self.comb += [
+            self.csrs.o_dbg_fifo_space.status.eq(fifo_spaces.dat_r),
+            self.csrs.o_dbg_last_timestamp.status.eq(last_timestamps.dat_r),
+            If(self.csrs.o_reset_channel_status.re,
+                fifo_spaces.dat_w.eq(0),
+                fifo_spaces.we.eq(1),
+                last_timestamps.dat_w.eq(0),
+                last_timestamps.we.eq(1)
+            )
+        ]
+
+    def get_csrs(self):
+        return self.csrs.get_csrs()
+
+
+class RTManager(Module, AutoCSR):
+    def __init__(self, rt_packets):
+        self.request_echo = CSR()
+
+        self.packet_err_present = CSR()
+        self.packet_err_code = CSRStatus(8)
+
+        self.update_packet_cnt = CSR()
+        self.packet_cnt_tx = CSRStatus(32)
+        self.packet_cnt_rx = CSRStatus(32)
+
+        # # #
+
+        self.comb += self.request_echo.w.eq(rt_packets.echo_stb)
+        self.sync += [
+            If(rt_packets.echo_ack, rt_packets.echo_stb.eq(0)),
+            If(self.request_echo.re, rt_packets.echo_stb.eq(1))
+        ]
+
+        self.comb += [
+            self.packet_err_present.w.eq(rt_packets.error_not),
+            rt_packets.error_not_ack.eq(self.packet_err_present.re),
+            self.packet_err_code.status.eq(rt_packets.error_code)
+        ]
+
+        self.sync += \
+            If(self.update_packet_cnt.re,
+                self.packet_cnt_tx.status.eq(rt_packets.packet_cnt_tx),
+                self.packet_cnt_rx.status.eq(rt_packets.packet_cnt_rx)
+            )

artiq/gateware/drtio/rt_packets.py

@@ -0,0 +1,731 @@
+from types import SimpleNamespace
+
+from migen import *
+from migen.genlib.fsm import *
+from migen.genlib.fifo import AsyncFIFO
+from migen.genlib.cdc import PulseSynchronizer
+
+from artiq.gateware.rtio.cdc import GrayCodeTransfer
+
+
+def layout_len(l):
+    return sum(e[1] for e in l)
+
+
+class PacketLayoutManager:
+    def __init__(self, alignment):
+        self.alignment = alignment
+        self.layouts = dict()
+        self.types = dict()
+        self.type_names = dict()
+    
+    def add_type(self, name, *fields, pad=True):
+        type_n = len(self.types)
+        self.types[name] = type_n
+        self.type_names[type_n] = name
+        layout = [("ty", 8)] + list(fields)
+        misalignment = layout_len(layout) % self.alignment
+        if misalignment:
+            layout.append(("packet_pad", self.alignment - misalignment))
+        self.layouts[name] = layout
+
+    def field_length(self, type_name, field_name):
+        layout = self.layouts[type_name]
+        for name, length in layout:
+            if name == field_name:
+                return length
+        raise KeyError
+
+
+def get_m2s_layouts(alignment):
+    if alignment > 128:
+        short_data_len = alignment - 128 + 16
+    else:
+        short_data_len = 16
+
+    plm = PacketLayoutManager(alignment)
+    plm.add_type("echo_request")
+    plm.add_type("set_time", ("timestamp", 64))
+    plm.add_type("reset", ("phy", 1))
+    plm.add_type("write", ("timestamp", 64),
+                          ("channel", 16),
+                          ("address", 16),
+                          ("extra_data_cnt", 8),
+                          ("short_data", short_data_len))
+    plm.add_type("fifo_space_request", ("channel", 16))
+    return plm
+
+
+def get_s2m_layouts(alignment):
+    plm = PacketLayoutManager(alignment)
+    plm.add_type("error", ("code", 8))
+    plm.add_type("echo_reply")
+    plm.add_type("fifo_space_reply", ("space", 16))
+    return plm
+
+
+error_codes = {
+    "unknown_type_local": 0,
+    "unknown_type_remote": 1,
+    # The transmitter is normally responsible for avoiding
+    # overflows and underflows. Those error reports are only
+    # for diagnosing internal ARTIQ bugs.
+    "write_overflow": 2,
+    "write_underflow": 3
+}
+
+
+class ReceiveDatapath(Module):
+    def __init__(self, frame, data, plm):
+        ws = len(data)
+
+        # control
+        self.packet_buffer_load = Signal()
+
+        # outputs
+        self.frame_r = Signal()
+        self.data_r = Signal(ws)
+        self.packet_type = Signal(8)
+        self.packet_last = Signal()
+        self.packet_as = dict()
+
+        # # #
+
+        # input pipeline stage - determine packet length based on type
+        lastword_per_type = [layout_len(plm.layouts[plm.type_names[i]])//ws - 1
+                             for i in range(len(plm.layouts))]
+        packet_last_n = Signal(max=max(lastword_per_type)+1)
+        self.sync += [
+            self.frame_r.eq(frame),
+            self.data_r.eq(data),
+            If(frame & ~self.frame_r,
+                self.packet_type.eq(data[:8]),
+                packet_last_n.eq(Array(lastword_per_type)[data[:8]])
+            )
+        ]
+
+        # bufferize packet
+        packet_buffer = Signal(max(layout_len(l)
+                                   for l in plm.layouts.values()))
+        w_in_packet = len(packet_buffer)//ws
+        packet_buffer_count = Signal(max=w_in_packet+1)
+        self.sync += \
+            If(self.packet_buffer_load,
+                Case(packet_buffer_count, 
+                     {i: packet_buffer[i*ws:(i+1)*ws].eq(self.data_r)
+                      for i in range(w_in_packet)}),
+                packet_buffer_count.eq(packet_buffer_count + 1)
+            ).Else(
+                packet_buffer_count.eq(0)
+            )
+        self.comb += self.packet_last.eq(packet_buffer_count == packet_last_n)
+
+        # dissect packet
+        for name, layout in plm.layouts.items():
+            fields = SimpleNamespace()
+            idx = 0
+            for field_name, field_size in layout:
+                setattr(fields, field_name, packet_buffer[idx:idx+field_size])
+                idx += field_size
+            self.packet_as[name] = fields
+
+
+class TransmitDatapath(Module):
+    def __init__(self, frame, data, plm):
+        ws = len(data)
+        assert ws % 8 == 0
+        self.ws = ws
+        self.plm = plm
+
+        self.packet_buffer = Signal(max(layout_len(l)
+                                        for l in plm.layouts.values()))
+        w_in_packet = len(self.packet_buffer)//ws
+        self.packet_last_n = Signal(max=max(w_in_packet, 2))
+        self.packet_stb = Signal()
+        self.packet_last = Signal()
+
+        self.raw_stb = Signal()
+        self.raw_data = Signal(ws)
+
+        # # #
+
+        self.sync += frame.eq(0)
+
+        if w_in_packet > 1:
+            packet_buffer_count = Signal(max=w_in_packet)
+            self.comb += self.packet_last.eq(packet_buffer_count == self.packet_last_n)
+            self.sync += [
+                packet_buffer_count.eq(0),
+                If(self.packet_stb,
+                    frame.eq(1),
+                    Case(packet_buffer_count,
+                         {i: data.eq(self.packet_buffer[i*ws:(i+1)*ws])
+                          for i in range(w_in_packet)}),
+                    packet_buffer_count.eq(packet_buffer_count + 1)
+                )
+            ]
+        else:
+            self.comb += self.packet_last.eq(1)
+            self.sync += \
+                If(self.packet_stb,
+                    frame.eq(1),
+                    data.eq(self.packet_buffer)
+                )
+
+        self.sync += [
+            If(self.raw_stb,
+                frame.eq(1),
+                data.eq(self.raw_data)
+            )
+        ]
+
+    def send(self, ty, **kwargs):
+        idx = 8
+        value = self.plm.types[ty]
+        for field_name, field_size in self.plm.layouts[ty][1:]:
+            try:
+                fvalue = kwargs[field_name]
+                del kwargs[field_name]
+            except KeyError:
+                fvalue = 0
+            value = value | (fvalue << idx)
+            idx += field_size
+        if kwargs:
+            raise ValueError
+        return [
+            self.packet_stb.eq(1),
+            self.packet_buffer.eq(value),
+            self.packet_last_n.eq(idx//self.ws-1)
+        ]
+
+
+class RTPacketSatellite(Module):
+    def __init__(self, link_layer):
+        self.tsc_load = Signal()
+        self.tsc_value = Signal(64)
+
+        self.reset = Signal(reset=1)
+        self.reset_phy = Signal(reset=1)
+
+        self.fifo_space_channel = Signal(16)
+        self.fifo_space_update = Signal()
+        self.fifo_space = Signal(16)
+        
+        self.write_stb = Signal()
+        self.write_timestamp = Signal(64)
+        self.write_channel = Signal(16)
+        self.write_address = Signal(16)
+        self.write_data = Signal(512)
+        self.write_overflow = Signal()
+        self.write_overflow_ack = Signal()
+        self.write_underflow = Signal()
+        self.write_underflow_ack = Signal()
+
+        # # #
+
+        # RX/TX datapath
+        assert len(link_layer.tx_rt_data) == len(link_layer.rx_rt_data)
+        assert len(link_layer.tx_rt_data) % 8 == 0
+        ws = len(link_layer.tx_rt_data)
+        rx_plm = get_m2s_layouts(ws)
+        rx_dp = ReceiveDatapath(
+            link_layer.rx_rt_frame, link_layer.rx_rt_data, rx_plm)
+        self.submodules += rx_dp
+        tx_plm = get_s2m_layouts(ws)
+        tx_dp = TransmitDatapath(
+            link_layer.tx_rt_frame, link_layer.tx_rt_data, tx_plm)
+        self.submodules += tx_dp
+
+        # RX write data buffer
+        write_data_buffer_load = Signal()
+        write_data_buffer_cnt = Signal(max=512//ws+1)
+        write_data_buffer = Signal(512)
+        self.sync += \
+            If(write_data_buffer_load,
+                Case(write_data_buffer_cnt,
+                     {i: write_data_buffer[i*ws:(i+1)*ws].eq(rx_dp.data_r)
+                      for i in range(512//ws)}),
+                write_data_buffer_cnt.eq(write_data_buffer_cnt + 1)
+            ).Else(
+                write_data_buffer_cnt.eq(0)
+            )
+
+        # RX->TX
+        echo_req = Signal()
+        err_set = Signal()
+        err_req = Signal()
+        err_ack = Signal()
+        fifo_space_set = Signal()
+        fifo_space_req = Signal()
+        fifo_space_ack = Signal()
+        self.sync += [
+            If(err_ack, err_req.eq(0)),
+            If(err_set, err_req.eq(1)),
+            If(fifo_space_ack, fifo_space_req.eq(0)),
+            If(fifo_space_set, fifo_space_req.eq(1)),
+        ]
+        err_code = Signal(max=len(error_codes)+1)
+
+        # RX FSM
+        self.comb += [
+            self.tsc_value.eq(
+                rx_dp.packet_as["set_time"].timestamp),
+            self.fifo_space_channel.eq(
+                rx_dp.packet_as["fifo_space_request"].channel),
+            self.write_timestamp.eq(
+                rx_dp.packet_as["write"].timestamp),
+            self.write_channel.eq(
+                rx_dp.packet_as["write"].channel),
+            self.write_address.eq(
+                rx_dp.packet_as["write"].address),
+            self.write_data.eq(
+                Cat(rx_dp.packet_as["write"].short_data, write_data_buffer))
+        ]
+
+        reset = Signal()
+        reset_phy = Signal()
+        self.sync += [
+            self.reset.eq(reset),
+            self.reset_phy.eq(reset_phy)
+        ]
+
+        rx_fsm = FSM(reset_state="INPUT")
+        self.submodules += rx_fsm
+
+        rx_fsm.act("INPUT",
+            If(rx_dp.frame_r,
+                rx_dp.packet_buffer_load.eq(1),
+                If(rx_dp.packet_last,
+                    Case(rx_dp.packet_type, {
+                        # echo must have fixed latency, so there is no memory
+                        # mechanism
+                        rx_plm.types["echo_request"]: echo_req.eq(1),
+                        rx_plm.types["set_time"]: NextState("SET_TIME"),
+                        rx_plm.types["reset"]: NextState("RESET"),
+                        rx_plm.types["write"]: NextState("WRITE"),
+                        rx_plm.types["fifo_space_request"]:
+                            NextState("FIFO_SPACE"),
+                        "default": [
+                            err_set.eq(1),
+                            NextValue(err_code, error_codes["unknown_type_remote"])]
+                    })
+                )
+            )
+        )
+        rx_fsm.act("SET_TIME",
+            self.tsc_load.eq(1),
+            NextState("INPUT")
+        )
+        rx_fsm.act("RESET",
+            If(rx_dp.packet_as["reset"].phy,
+                reset_phy.eq(1)
+            ).Else(
+                reset.eq(1)
+            ),
+            NextState("INPUT")
+        )
+        rx_fsm.act("WRITE",
+            write_data_buffer_load.eq(1),
+            If(write_data_buffer_cnt == rx_dp.packet_as["write"].extra_data_cnt,
+                self.write_stb.eq(1),
+                NextState("INPUT")
+            )
+        )
+        rx_fsm.act("FIFO_SPACE",
+            fifo_space_set.eq(1),
+            self.fifo_space_update.eq(1),
+            NextState("INPUT")
+        )
+
+        # TX FSM
+        tx_fsm = FSM(reset_state="IDLE")
+        self.submodules += tx_fsm
+
+        tx_fsm.act("IDLE",
+            If(echo_req, NextState("ECHO")),
+            If(fifo_space_req, NextState("FIFO_SPACE")),
+            If(self.write_overflow, NextState("ERROR_WRITE_OVERFLOW")),
+            If(self.write_underflow, NextState("ERROR_WRITE_UNDERFLOW")),
+            If(err_req, NextState("ERROR"))
+        )
+        tx_fsm.act("ECHO",
+            tx_dp.send("echo_reply"),
+            If(tx_dp.packet_last, NextState("IDLE"))
+        )
+        tx_fsm.act("FIFO_SPACE",
+            fifo_space_ack.eq(1),
+            tx_dp.send("fifo_space_reply", space=self.fifo_space),
+            If(tx_dp.packet_last, NextState("IDLE"))
+        )
+        tx_fsm.act("ERROR_WRITE_OVERFLOW",
+            self.write_overflow_ack.eq(1),
+            tx_dp.send("error", code=error_codes["write_overflow"]),
+            If(tx_dp.packet_last, NextState("IDLE"))
+        )
+        tx_fsm.act("ERROR_WRITE_UNDERFLOW",
+            self.write_underflow_ack.eq(1),
+            tx_dp.send("error", code=error_codes["write_underflow"]),
+            If(tx_dp.packet_last, NextState("IDLE"))
+        )
+        tx_fsm.act("ERROR",
+            err_ack.eq(1),
+            tx_dp.send("error", code=err_code),
+            If(tx_dp.packet_last, NextState("IDLE"))
+        )
+
+
+class _CrossDomainRequest(Module):
+    def __init__(self, domain,
+                 req_stb, req_ack, req_data,
+                 srv_stb, srv_ack, srv_data):
+        dsync = getattr(self.sync, domain)
+
+        request = PulseSynchronizer("sys", domain)
+        reply = PulseSynchronizer(domain, "sys")
+        self.submodules += request, reply
+
+        ongoing = Signal()
+        self.comb += request.i.eq(~ongoing & req_stb)
+        self.sync += [
+            req_ack.eq(reply.o),
+            If(req_stb, ongoing.eq(1)),
+            If(req_ack, ongoing.eq(0))
+        ]
+        if req_data is not None:
+            req_data_r = Signal.like(req_data)
+            req_data_r.attr.add("no_retiming")
+            self.sync += If(req_stb, req_data_r.eq(req_data))
+        dsync += [
+            If(request.o, srv_stb.eq(1)),
+            If(srv_ack, srv_stb.eq(0))
+        ]
+        if req_data is not None:
+            dsync += If(request.o, srv_data.eq(req_data_r))
+        self.comb += reply.i.eq(srv_stb & srv_ack)
+
+
+class _CrossDomainNotification(Module):
+    def __init__(self, domain,
+                 emi_stb, emi_data,
+                 rec_stb, rec_ack, rec_data):
+        emi_data_r = Signal.like(emi_data)
+        emi_data_r.attr.add("no_retiming")
+        dsync = getattr(self.sync, domain)
+        dsync += If(emi_stb, emi_data_r.eq(emi_data))
+
+        ps = PulseSynchronizer(domain, "sys")
+        self.submodules += ps
+        self.comb += ps.i.eq(emi_stb)
+        self.sync += [
+            If(rec_ack, rec_stb.eq(0)),
+            If(ps.o,
+                rec_data.eq(emi_data_r),
+                rec_stb.eq(1)
+            )
+        ]
+
+
+class RTPacketMaster(Module):
+    def __init__(self, link_layer, write_fifo_depth=4):
+        # all interface signals in sys domain unless otherwise specified
+
+        # write interface, optimized for throughput
+        self.write_stb = Signal()
+        self.write_ack = Signal()
+        self.write_timestamp = Signal(64)
+        self.write_channel = Signal(16)
+        self.write_address = Signal(16)
+        self.write_data = Signal(512)
+
+        # fifo space interface
+        # write with timestamp[48:] == 0xffff to make a fifo space request
+        # (space requests have to be ordered wrt writes)
+        self.fifo_space_not = Signal()
+        self.fifo_space_not_ack = Signal()
+        self.fifo_space = Signal(16)
+
+        # echo interface
+        self.echo_stb = Signal()
+        self.echo_ack = Signal()
+        self.echo_sent_now = Signal()  # in rtio domain
+        self.echo_received_now = Signal()  # in rtio_rx domain
+
+        # set_time interface
+        self.set_time_stb = Signal()
+        self.set_time_ack = Signal()
+        # in rtio domain, must be valid all time while there is
+        # a set_time request pending
+        self.tsc_value = Signal(64)
+
+        # reset interface
+        self.reset_stb = Signal()
+        self.reset_ack = Signal()
+        self.reset_phy = Signal()
+
+        # errors
+        self.error_not = Signal()
+        self.error_not_ack = Signal()
+        self.error_code = Signal(8)
+
+        # packet counters
+        self.packet_cnt_tx = Signal(32)
+        self.packet_cnt_rx = Signal(32)
+
+        # # #
+
+        # RX/TX datapath
+        assert len(link_layer.tx_rt_data) == len(link_layer.rx_rt_data)
+        assert len(link_layer.tx_rt_data) % 8 == 0
+        ws = len(link_layer.tx_rt_data)
+        tx_plm = get_m2s_layouts(ws)
+        tx_dp = ClockDomainsRenamer("rtio")(TransmitDatapath(
+            link_layer.tx_rt_frame, link_layer.tx_rt_data, tx_plm))
+        self.submodules += tx_dp
+        rx_plm = get_s2m_layouts(ws)
+        rx_dp = ClockDomainsRenamer("rtio_rx")(ReceiveDatapath(
+            link_layer.rx_rt_frame, link_layer.rx_rt_data, rx_plm))
+        self.submodules += rx_dp
+
+        # Write FIFO and extra data count
+        wfifo = ClockDomainsRenamer({"write": "sys", "read": "rtio"})(
+            AsyncFIFO(64+16+16+512, write_fifo_depth))
+        self.submodules += wfifo
+        write_timestamp_d = Signal(64)
+        write_channel_d = Signal(16)
+        write_address_d = Signal(16)
+        write_data_d = Signal(512)
+        self.comb += [
+            wfifo.we.eq(self.write_stb),
+            self.write_ack.eq(wfifo.writable),
+            wfifo.din.eq(Cat(self.write_timestamp, self.write_channel,
+                             self.write_address, self.write_data)),
+            Cat(write_timestamp_d, write_channel_d,
+                write_address_d, write_data_d).eq(wfifo.dout)
+        ]
+
+        wfb_readable = Signal()
+        wfb_re = Signal()
+
+        self.comb += wfifo.re.eq(wfifo.readable & (~wfb_readable | wfb_re))
+        self.sync.rtio += \
+            If(wfifo.re,
+                wfb_readable.eq(1),
+            ).Elif(wfb_re,
+                wfb_readable.eq(0),
+            )
+
+        write_timestamp = Signal(64)
+        write_channel = Signal(16)
+        write_address = Signal(16)
+        write_extra_data_cnt = Signal(8)
+        write_data = Signal(512)
+
+        self.sync.rtio += If(wfifo.re,
+            write_timestamp.eq(write_timestamp_d),
+            write_channel.eq(write_channel_d),
+            write_address.eq(write_address_d),
+            write_data.eq(write_data_d))
+
+        short_data_len = tx_plm.field_length("write", "short_data")
+        write_extra_data_d = Signal(512)
+        self.comb += write_extra_data_d.eq(write_data_d[short_data_len:])
+        for i in range(512//ws):
+            self.sync.rtio += If(wfifo.re,
+                If(write_extra_data_d[ws*i:ws*(i+1)] != 0, write_extra_data_cnt.eq(i+1)))
+
+        write_extra_data = Signal(512)
+        self.sync.rtio += If(wfifo.re, write_extra_data.eq(write_extra_data_d))
+
+        extra_data_ce = Signal()
+        extra_data_last = Signal()
+        extra_data_counter = Signal(max=512//ws+1)
+        self.comb += [
+            Case(extra_data_counter, 
+                {i+1: tx_dp.raw_data.eq(write_extra_data[i*ws:(i+1)*ws])
+                 for i in range(512//ws)}),
+            extra_data_last.eq(extra_data_counter == write_extra_data_cnt)
+        ]
+        self.sync.rtio += \
+            If(extra_data_ce,
+                extra_data_counter.eq(extra_data_counter + 1),
+            ).Else(
+                extra_data_counter.eq(1)
+            )
+
+        # CDC
+        fifo_space_not = Signal()
+        fifo_space = Signal(16)
+        self.submodules += _CrossDomainNotification("rtio_rx",
+            fifo_space_not, fifo_space,
+            self.fifo_space_not, self.fifo_space_not_ack, self.fifo_space)
+
+        set_time_stb = Signal()
+        set_time_ack = Signal()
+        self.submodules += _CrossDomainRequest("rtio",
+            self.set_time_stb, self.set_time_ack, None,
+            set_time_stb, set_time_ack, None)
+
+        reset_stb = Signal()
+        reset_ack = Signal()
+        reset_phy = Signal()
+        self.submodules += _CrossDomainRequest("rtio",
+            self.reset_stb, self.reset_ack, self.reset_phy,
+            reset_stb, reset_ack, reset_phy)
+
+        echo_stb = Signal()
+        echo_ack = Signal()
+        self.submodules += _CrossDomainRequest("rtio",
+            self.echo_stb, self.echo_ack, None,
+            echo_stb, echo_ack, None)
+
+        error_not = Signal()
+        error_code = Signal(8)
+        self.submodules += _CrossDomainNotification("rtio_rx",
+            error_not, error_code,
+            self.error_not, self.error_not_ack, self.error_code)
+
+        # TX FSM
+        tx_fsm = ClockDomainsRenamer("rtio")(FSM(reset_state="IDLE"))
+        self.submodules += tx_fsm
+
+        echo_sent_now = Signal()
+        self.sync.rtio += self.echo_sent_now.eq(echo_sent_now)
+        tsc_value = Signal(64)
+        tsc_value_load = Signal()
+        self.sync.rtio += If(tsc_value_load, tsc_value.eq(self.tsc_value))
+
+        tx_fsm.act("IDLE",
+            If(wfb_readable,
+                If(write_timestamp[48:] == 0xffff,
+                    NextState("FIFO_SPACE")
+                ).Else(
+                    NextState("WRITE")
+                )
+            ).Else(
+                If(echo_stb,
+                    echo_sent_now.eq(1),
+                    NextState("ECHO")
+                ).Elif(set_time_stb,
+                    tsc_value_load.eq(1),
+                    NextState("SET_TIME")
+                ).Elif(reset_stb,
+                    NextState("RESET")
+                )
+            )
+        )
+        tx_fsm.act("WRITE",
+            tx_dp.send("write",
+                timestamp=write_timestamp,
+                channel=write_channel,
+                address=write_address,
+                extra_data_cnt=write_extra_data_cnt,
+                short_data=write_data[:short_data_len]),
+            If(tx_dp.packet_last,
+                If(write_extra_data_cnt == 0,
+                    wfb_re.eq(1),
+                    NextState("IDLE")
+                ).Else(
+                    NextState("WRITE_EXTRA")
+                )
+            )
+        )
+        tx_fsm.act("WRITE_EXTRA",
+            tx_dp.raw_stb.eq(1),
+            extra_data_ce.eq(1),
+            If(extra_data_last,
+                wfb_re.eq(1),
+                NextState("IDLE")
+            )
+        )
+        tx_fsm.act("FIFO_SPACE",
+            tx_dp.send("fifo_space_request", channel=write_channel),
+            If(tx_dp.packet_last,
+                wfb_re.eq(1),
+                NextState("IDLE")
+            )
+        )
+        tx_fsm.act("ECHO",
+            tx_dp.send("echo_request"),
+            If(tx_dp.packet_last,
+                echo_ack.eq(1),
+                NextState("IDLE")
+            )
+        )
+        tx_fsm.act("SET_TIME",
+            tx_dp.send("set_time", timestamp=tsc_value),
+            If(tx_dp.packet_last,
+                set_time_ack.eq(1),
+                NextState("IDLE")
+            )
+        )
+        tx_fsm.act("RESET",
+            tx_dp.send("reset", phy=reset_phy),
+            If(tx_dp.packet_last,
+                reset_ack.eq(1),
+                NextState("IDLE")
+            )
+        )
+
+        # RX FSM
+        rx_fsm = ClockDomainsRenamer("rtio_rx")(FSM(reset_state="INPUT"))
+        self.submodules += rx_fsm
+
+        echo_received_now = Signal()
+        self.sync.rtio_rx += self.echo_received_now.eq(echo_received_now)
+
+        rx_fsm.act("INPUT",
+            If(rx_dp.frame_r,
+                rx_dp.packet_buffer_load.eq(1),
+                If(rx_dp.packet_last,
+                    Case(rx_dp.packet_type, {
+                        rx_plm.types["error"]: NextState("ERROR"),
+                        rx_plm.types["echo_reply"]: echo_received_now.eq(1),
+                        rx_plm.types["fifo_space_reply"]: NextState("FIFO_SPACE"),
+                        "default": [
+                            error_not.eq(1),
+                            error_code.eq(error_codes["unknown_type_local"])
+                        ]
+                    })
+                )
+            )
+        )
+        rx_fsm.act("ERROR",
+            error_not.eq(1),
+            error_code.eq(rx_dp.packet_as["error"].code),
+            NextState("INPUT")
+        )
+        rx_fsm.act("FIFO_SPACE",
+            fifo_space_not.eq(1),
+            fifo_space.eq(rx_dp.packet_as["fifo_space_reply"].space),
+            NextState("INPUT")
+        )
+
+        # packet counters
+        tx_frame_r = Signal()
+        packet_cnt_tx = Signal(32)
+        self.sync.rtio += [
+            tx_frame_r.eq(link_layer.tx_rt_frame),
+            If(link_layer.tx_rt_frame & ~tx_frame_r,
+                packet_cnt_tx.eq(packet_cnt_tx + 1))
+        ]
+        cdc_packet_cnt_tx = GrayCodeTransfer(32)
+        self.submodules += cdc_packet_cnt_tx
+        self.comb += [
+            cdc_packet_cnt_tx.i.eq(packet_cnt_tx),
+            self.packet_cnt_tx.eq(cdc_packet_cnt_tx.o)
+        ]
+
+        rx_frame_r = Signal()
+        packet_cnt_rx = Signal(32)
+        self.sync.rtio_rx += [
+            rx_frame_r.eq(link_layer.rx_rt_frame),
+            If(link_layer.rx_rt_frame & ~rx_frame_r,
+                packet_cnt_rx.eq(packet_cnt_rx + 1))
+        ]
+        cdc_packet_cnt_rx = ClockDomainsRenamer({"rtio": "rtio_rx"})(
+            GrayCodeTransfer(32))
+        self.submodules += cdc_packet_cnt_rx
+        self.comb += [
+            cdc_packet_cnt_rx.i.eq(packet_cnt_rx),
+            self.packet_cnt_rx.eq(cdc_packet_cnt_rx.o)
+        ]

artiq/gateware/drtio/transceiver/gtx_7series.py

@@ -0,0 +1,265 @@
+from migen import *
+from migen.genlib.resetsync import AsyncResetSynchronizer
+
+from misoc.cores.code_8b10b import Encoder, Decoder
+from misoc.interconnect.csr import *
+
+from artiq.gateware.drtio.transceiver.gtx_7series_init import *
+
+
+class GTX_20X(Module):
+    # The transceiver clock on clock_pads must be at the RTIO clock
+    # frequency when clock_div2=False, and 2x that frequency when
+    # clock_div2=True.
+    def __init__(self, clock_pads, tx_pads, rx_pads, sys_clk_freq,
+                 clock_div2=False):
+        self.submodules.encoder = ClockDomainsRenamer("rtio")(
+            Encoder(2, True))
+        self.decoders = [ClockDomainsRenamer("rtio_rx")(
+            Decoder(True)) for _ in range(2)]
+        self.submodules += self.decoders
+
+        self.rx_ready = Signal()
+
+        # transceiver direct clock outputs
+        # useful to specify clock constraints in a way palatable to Vivado
+        self.txoutclk = Signal()
+        self.rxoutclk = Signal()
+
+        # # #
+
+        refclk = Signal()
+        if clock_div2:
+            self.specials += Instance("IBUFDS_GTE2",
+                i_CEB=0,
+                i_I=clock_pads.p,
+                i_IB=clock_pads.n,
+                o_ODIV2=refclk
+            )
+        else:
+            self.specials += Instance("IBUFDS_GTE2",
+                i_CEB=0,
+                i_I=clock_pads.p,
+                i_IB=clock_pads.n,
+                o_O=refclk
+            )
+
+        cplllock = Signal()
+        # TX generates RTIO clock, init must be in system domain
+        tx_init = GTXInit(sys_clk_freq, False)
+        # RX receives restart commands from RTIO domain
+        rx_init = ClockDomainsRenamer("rtio")(
+            GTXInit(self.rtio_clk_freq, True))
+        self.submodules += tx_init, rx_init
+        self.comb += tx_init.cplllock.eq(cplllock), \
+                     rx_init.cplllock.eq(cplllock)
+
+        txdata = Signal(20)
+        rxdata = Signal(20)
+        self.specials += \
+            Instance("GTXE2_CHANNEL",
+                # PMA Attributes
+                p_PMA_RSV=0x00018480,
+                p_PMA_RSV2=0x2050,
+                p_PMA_RSV3=0,
+                p_PMA_RSV4=0,
+                p_RX_BIAS_CFG=0b100,
+                p_RX_CM_TRIM=0b010,
+                p_RX_OS_CFG=0b10000000,
+                p_RX_CLK25_DIV=5,
+                p_TX_CLK25_DIV=5,
+
+                # Power-Down Attributes
+                p_PD_TRANS_TIME_FROM_P2=0x3c,
+                p_PD_TRANS_TIME_NONE_P2=0x3c,
+                p_PD_TRANS_TIME_TO_P2=0x64,
+
+                # CPLL
+                p_CPLL_CFG=0xBC07DC,
+                p_CPLL_FBDIV=4,
+                p_CPLL_FBDIV_45=5,
+                p_CPLL_REFCLK_DIV=1,
+                p_RXOUT_DIV=2,
+                p_TXOUT_DIV=2,
+                o_CPLLLOCK=cplllock,
+                i_CPLLLOCKEN=1,
+                i_CPLLREFCLKSEL=0b001,
+                i_TSTIN=2**20-1,
+                i_GTREFCLK0=refclk,
+
+                # TX clock
+                p_TXBUF_EN="FALSE",
+                p_TX_XCLK_SEL="TXUSR",
+                o_TXOUTCLK=self.txoutclk,
+                i_TXSYSCLKSEL=0b00,
+                i_TXOUTCLKSEL=0b11,
+
+                # TX Startup/Reset
+                i_GTTXRESET=tx_init.gtXxreset,
+                o_TXRESETDONE=tx_init.Xxresetdone,
+                i_TXDLYSRESET=tx_init.Xxdlysreset,
+                o_TXDLYSRESETDONE=tx_init.Xxdlysresetdone,
+                o_TXPHALIGNDONE=tx_init.Xxphaligndone,
+                i_TXUSERRDY=tx_init.Xxuserrdy,
+
+                # TX data
+                p_TX_DATA_WIDTH=20,
+                p_TX_INT_DATAWIDTH=0,
+                i_TXCHARDISPMODE=Cat(txdata[9], txdata[19]),
+                i_TXCHARDISPVAL=Cat(txdata[8], txdata[18]),
+                i_TXDATA=Cat(txdata[:8], txdata[10:18]),
+                i_TXUSRCLK=ClockSignal("rtio"),
+                i_TXUSRCLK2=ClockSignal("rtio"),
+
+                # TX electrical
+                i_TXBUFDIFFCTRL=0b100,
+                i_TXDIFFCTRL=0b1000,
+
+                # RX Startup/Reset
+                i_GTRXRESET=rx_init.gtXxreset,
+                o_RXRESETDONE=rx_init.Xxresetdone,
+                i_RXDLYSRESET=rx_init.Xxdlysreset,
+                o_RXDLYSRESETDONE=rx_init.Xxdlysresetdone,
+                o_RXPHALIGNDONE=rx_init.Xxphaligndone,
+                i_RXUSERRDY=rx_init.Xxuserrdy,
+
+                # RX AFE
+                p_RX_DFE_XYD_CFG=0,
+                i_RXDFEXYDEN=1,
+                i_RXDFEXYDHOLD=0,
+                i_RXDFEXYDOVRDEN=0,
+                i_RXLPMEN=0,
+
+                # RX clock
+                p_RXBUF_EN="FALSE",
+                p_RX_XCLK_SEL="RXUSR",
+                i_RXDDIEN=1,
+                i_RXSYSCLKSEL=0b00,
+                i_RXOUTCLKSEL=0b010,
+                o_RXOUTCLK=self.rxoutclk,
+                i_RXUSRCLK=ClockSignal("rtio_rx"),
+                i_RXUSRCLK2=ClockSignal("rtio_rx"),
+                p_RXCDR_CFG=0x03000023FF10100020,
+
+                # RX Clock Correction Attributes
+                p_CLK_CORRECT_USE="FALSE",
+                p_CLK_COR_SEQ_1_1=0b0100000000,
+                p_CLK_COR_SEQ_2_1=0b0100000000,
+                p_CLK_COR_SEQ_1_ENABLE=0b1111,
+                p_CLK_COR_SEQ_2_ENABLE=0b1111,
+
+                # RX data
+                p_RX_DATA_WIDTH=20,
+                p_RX_INT_DATAWIDTH=0,
+                o_RXDISPERR=Cat(rxdata[9], rxdata[19]),
+                o_RXCHARISK=Cat(rxdata[8], rxdata[18]),
+                o_RXDATA=Cat(rxdata[:8], rxdata[10:18]),
+
+                # Pads
+                i_GTXRXP=rx_pads.p,
+                i_GTXRXN=rx_pads.n,
+                o_GTXTXP=tx_pads.p,
+                o_GTXTXN=tx_pads.n,
+            )
+
+        tx_reset_deglitched = Signal()
+        tx_reset_deglitched.attr.add("no_retiming")
+        self.sync += tx_reset_deglitched.eq(~tx_init.done)
+        self.clock_domains.cd_rtio = ClockDomain()
+        self.specials += [
+            Instance("BUFG", i_I=self.txoutclk, o_O=self.cd_rtio.clk),
+            AsyncResetSynchronizer(self.cd_rtio, tx_reset_deglitched)
+        ]
+        rx_reset_deglitched = Signal()
+        rx_reset_deglitched.attr.add("no_retiming")
+        self.sync.rtio += rx_reset_deglitched.eq(~rx_init.done)
+        self.clock_domains.cd_rtio_rx = ClockDomain()
+        self.specials += [
+            Instance("BUFG", i_I=self.rxoutclk, o_O=self.cd_rtio_rx.clk),
+            AsyncResetSynchronizer(self.cd_rtio_rx, rx_reset_deglitched)
+        ]
+
+        self.comb += [
+            txdata.eq(Cat(self.encoder.output[0], self.encoder.output[1])),
+            self.decoders[0].input.eq(rxdata[:10]),
+            self.decoders[1].input.eq(rxdata[10:])
+        ]
+
+        clock_aligner = BruteforceClockAligner(0b0101111100, self.rtio_clk_freq)
+        self.submodules += clock_aligner
+        self.comb += [
+            clock_aligner.rxdata.eq(rxdata),
+            rx_init.restart.eq(clock_aligner.restart),
+            self.rx_ready.eq(clock_aligner.ready)
+        ]
+
+
+class GTX_1000BASE_BX10(GTX_20X):
+    rtio_clk_freq = 62.5e6
+
+
+class GTX_3G(GTX_20X):
+    rtio_clk_freq = 150e6
+
+
+class RXSynchronizer(Module, AutoCSR):
+    """Delays the data received in the rtio_rx by a configurable amount
+    so that it meets s/h in the rtio domain, and recapture it in the rtio
+    domain. This has fixed latency.
+
+    Since Xilinx doesn't provide decent on-chip delay lines, we implement the
+    delay with MMCM that provides a clock and a finely configurable phase, used
+    to resample the data.
+
+    The phase has to be determined either empirically or by making sense of the
+    Xilinx scriptures (when existent) and should be constant for a given design
+    placement.
+    """
+    def __init__(self, rtio_clk_freq, initial_phase=0.0):
+        self.phase_shift = CSR()
+        self.phase_shift_done = CSRStatus()
+
+        self.clock_domains.cd_rtio_delayed = ClockDomain()
+
+        mmcm_output = Signal()
+        mmcm_fb = Signal()
+        mmcm_locked = Signal()
+        # maximize VCO frequency to maximize phase shift resolution
+        mmcm_mult = 1200e6//rtio_clk_freq
+        self.specials += [
+            Instance("MMCME2_ADV",
+                p_CLKIN1_PERIOD=1e9/rtio_clk_freq,
+                i_CLKIN1=ClockSignal("rtio_rx"),
+                i_RST=ResetSignal("rtio_rx"),
+                i_CLKINSEL=1,  # yes, 1=CLKIN1 0=CLKIN2
+
+                p_CLKFBOUT_MULT_F=mmcm_mult,
+                p_CLKOUT0_DIVIDE_F=mmcm_mult,
+                p_CLKOUT0_PHASE=initial_phase,
+                p_DIVCLK_DIVIDE=1,
+
+                # According to Xilinx, there is no guarantee of input/output
+                # phase relationship when using internal feedback. We assume
+                # here that the input/ouput skew is constant to save BUFGs.
+                o_CLKFBOUT=mmcm_fb,
+                i_CLKFBIN=mmcm_fb,
+
+                p_CLKOUT0_USE_FINE_PS="TRUE",
+                o_CLKOUT0=mmcm_output,
+                o_LOCKED=mmcm_locked,
+
+                i_PSCLK=ClockSignal(),
+                i_PSEN=self.phase_shift.re,
+                i_PSINCDEC=self.phase_shift.r,
+                o_PSDONE=self.phase_shift_done.status,
+            ),
+            Instance("BUFR", i_I=mmcm_output, o_O=self.cd_rtio_delayed.clk),
+            AsyncResetSynchronizer(self.cd_rtio_delayed, ~mmcm_locked)
+        ]
+
+    def resync(self, signal):
+        delayed = Signal.like(signal, related=signal)
+        synchronized = Signal.like(signal, related=signal)
+        self.sync.rtio_delayed += delayed.eq(signal)
+        self.sync.rtio += synchronized.eq(delayed)
+        return synchronized

artiq/gateware/drtio/transceiver/gtx_7series_init.py

@@ -0,0 +1,226 @@
+from math import ceil
+from functools import reduce
+from operator import add
+
+from migen import *
+from migen.genlib.cdc import MultiReg, PulseSynchronizer
+from migen.genlib.misc import WaitTimer
+from migen.genlib.fsm import FSM
+
+
+class GTXInit(Module):
+    # Based on LiteSATA by Enjoy-Digital
+    def __init__(self, sys_clk_freq, rx):
+        self.done = Signal()
+        self.restart = Signal()
+
+        # GTX signals
+        self.cplllock = Signal()
+        self.gtXxreset = Signal()
+        self.Xxresetdone = Signal()
+        self.Xxdlysreset = Signal()
+        self.Xxdlysresetdone = Signal()
+        self.Xxphaligndone = Signal()
+        self.Xxuserrdy = Signal()
+
+        # # #
+
+        # Double-latch transceiver asynch outputs
+        cplllock = Signal()
+        Xxresetdone = Signal()
+        Xxdlysresetdone = Signal()
+        Xxphaligndone = Signal()
+        self.specials += [
+            MultiReg(self.cplllock, cplllock),
+            MultiReg(self.Xxresetdone, Xxresetdone),
+            MultiReg(self.Xxdlysresetdone, Xxdlysresetdone),
+            MultiReg(self.Xxphaligndone, Xxphaligndone),
+        ]
+
+        # Deglitch FSM outputs driving transceiver asynch inputs
+        gtXxreset = Signal()
+        Xxdlysreset = Signal()
+        Xxuserrdy = Signal()
+        self.sync += [
+            self.gtXxreset.eq(gtXxreset),
+            self.Xxdlysreset.eq(Xxdlysreset),
+            self.Xxuserrdy.eq(Xxuserrdy)
+        ]
+
+        # After configuration, transceiver resets have to stay low for
+        # at least 500ns (see AR43482)
+        startup_cycles = ceil(500*sys_clk_freq/1000000000)
+        startup_timer = WaitTimer(startup_cycles)
+        self.submodules += startup_timer
+
+        startup_fsm = FSM(reset_state="INITIAL")
+        self.submodules += startup_fsm
+
+        if rx:
+            cdr_stable_timer = WaitTimer(1024)
+            self.submodules += cdr_stable_timer
+
+        Xxphaligndone_r = Signal(reset=1)
+        Xxphaligndone_rising = Signal()
+        self.sync += Xxphaligndone_r.eq(Xxphaligndone)
+        self.comb += Xxphaligndone_rising.eq(Xxphaligndone & ~Xxphaligndone_r)
+
+        startup_fsm.act("INITIAL",
+            startup_timer.wait.eq(1),
+            If(startup_timer.done, NextState("RESET_GTX"))
+        )
+        startup_fsm.act("RESET_GTX",
+            gtXxreset.eq(1),
+            NextState("WAIT_CPLL")
+        )
+        startup_fsm.act("WAIT_CPLL",
+            gtXxreset.eq(1),
+            If(cplllock, NextState("RELEASE_RESET"))
+        )
+        # Release GTX reset and wait for GTX resetdone
+        # (from UG476, GTX is reset on falling edge
+        # of gttxreset)
+        if rx:
+            startup_fsm.act("RELEASE_RESET",
+                Xxuserrdy.eq(1),
+                cdr_stable_timer.wait.eq(1),
+                If(Xxresetdone & cdr_stable_timer.done, NextState("ALIGN"))
+            )
+        else:
+            startup_fsm.act("RELEASE_RESET",
+                Xxuserrdy.eq(1),
+                If(Xxresetdone, NextState("ALIGN"))
+            )
+        # Start delay alignment (pulse)
+        startup_fsm.act("ALIGN",
+            Xxuserrdy.eq(1),
+            Xxdlysreset.eq(1),
+            NextState("WAIT_ALIGN")
+        )
+        # Wait for delay alignment
+        startup_fsm.act("WAIT_ALIGN",
+            Xxuserrdy.eq(1),
+            If(Xxdlysresetdone, NextState("WAIT_FIRST_ALIGN_DONE"))
+        )
+        # Wait 2 rising edges of rxphaligndone
+        # (from UG476 in buffer bypass config)
+        startup_fsm.act("WAIT_FIRST_ALIGN_DONE",
+            Xxuserrdy.eq(1),
+            If(Xxphaligndone_rising, NextState("WAIT_SECOND_ALIGN_DONE"))
+        )
+        startup_fsm.act("WAIT_SECOND_ALIGN_DONE",
+            Xxuserrdy.eq(1),
+            If(Xxphaligndone_rising, NextState("READY"))
+        )
+        startup_fsm.act("READY",
+            Xxuserrdy.eq(1),
+            self.done.eq(1),
+            If(self.restart, NextState("RESET_GTX"))
+        )
+
+
+# Changes the phase of the transceiver RX clock to align the comma to
+# the LSBs of RXDATA, fixing the latency.
+#
+# This is implemented by repeatedly resetting the transceiver until it
+# gives out the correct phase. Each reset gives a random phase.
+#
+# If Xilinx had designed the GTX transceiver correctly, RXSLIDE_MODE=PMA
+# would achieve this faster and in a cleaner way. But:
+#  * the phase jumps are of 2 UI at every second RXSLIDE pulse, instead
+#    of 1 UI at every pulse. It is unclear what the latency becomes.
+#  * RXSLIDE_MODE=PMA cannot be used with the RX buffer bypassed.
+# Those design flaws make RXSLIDE_MODE=PMA yet another broken and useless
+# transceiver "feature".
+#
+# Warning: Xilinx transceivers are LSB first, and comma needs to be flipped
+# compared to the usual 8b10b binary representation.
+class BruteforceClockAligner(Module):
+    def __init__(self, comma, rtio_clk_freq, check_period=6e-3):
+        self.rxdata = Signal(20)
+        self.restart = Signal()
+
+        self.ready = Signal()
+
+        check_max_val = ceil(check_period*rtio_clk_freq)
+        check_counter = Signal(max=check_max_val+1)
+        check = Signal()
+        reset_check_counter = Signal()
+        self.sync.rtio += [
+            check.eq(0),
+            If(reset_check_counter,
+                check_counter.eq(check_max_val)
+            ).Else(
+                If(check_counter == 0,
+                    check.eq(1),
+                    check_counter.eq(check_max_val)
+                ).Else(
+                    check_counter.eq(check_counter-1)
+                )
+            )
+        ]
+
+        checks_reset = PulseSynchronizer("rtio", "rtio_rx")
+        self.submodules += checks_reset
+
+        comma_n = ~comma & 0b1111111111
+        comma_seen_rxclk = Signal()
+        comma_seen = Signal()
+        comma_seen_rxclk.attr.add("no_retiming")
+        self.specials += MultiReg(comma_seen_rxclk, comma_seen)
+        self.sync.rtio_rx += \
+            If(checks_reset.o,
+                comma_seen_rxclk.eq(0)
+            ).Elif((self.rxdata[:10] == comma) | (self.rxdata[:10] == comma_n),
+                comma_seen_rxclk.eq(1)
+            )
+
+        error_seen_rxclk = Signal()
+        error_seen = Signal()
+        error_seen_rxclk.attr.add("no_retiming")
+        self.specials += MultiReg(error_seen_rxclk, error_seen)
+        rx1cnt = Signal(max=11)
+        self.sync.rtio_rx += [
+            rx1cnt.eq(reduce(add, [self.rxdata[i] for i in range(10)])),
+            If(checks_reset.o,
+                error_seen_rxclk.eq(0)
+            ).Elif((rx1cnt != 4) & (rx1cnt != 5) & (rx1cnt != 6),
+                error_seen_rxclk.eq(1)
+            )
+        ]
+
+        fsm = ClockDomainsRenamer("rtio")(FSM(reset_state="WAIT_COMMA"))
+        self.submodules += fsm
+
+        fsm.act("WAIT_COMMA",
+            If(check,
+                # Errors are still OK at this stage, as the transceiver
+                # has just been reset and may output garbage data.
+                If(comma_seen,
+                    NextState("WAIT_NOERROR")
+                ).Else(
+                    self.restart.eq(1)
+                ),
+                checks_reset.i.eq(1)
+            )
+        )
+        fsm.act("WAIT_NOERROR",
+            If(check,
+                If(comma_seen & ~error_seen,
+                    NextState("READY")
+                ).Else(
+                    self.restart.eq(1),
+                    NextState("WAIT_COMMA")
+                ),
+                checks_reset.i.eq(1)
+            )
+        )
+        fsm.act("READY",
+            reset_check_counter.eq(1),
+            self.ready.eq(1),
+            If(error_seen,
+                checks_reset.i.eq(1),
+                self.restart.eq(1),
+                NextState("WAIT_COMMA")
+            )
+        )

artiq/gateware/rtio/__init__.py

@@ -1,3 +1,5 @@
-from artiq.gateware.rtio.core import Channel, LogChannel, RTIO
+from artiq.gateware.rtio.cri import KernelInitiator, CRIInterconnectShared
+from artiq.gateware.rtio.core import Channel, LogChannel, Core
 from artiq.gateware.rtio.analyzer import Analyzer
 from artiq.gateware.rtio.moninj import MonInj
+from artiq.gateware.rtio.dma import DMA

artiq/gateware/rtio/analyzer.py

@@ -42,7 +42,7 @@ assert layout_len(stopped_layout) == message_len
 
 
 class MessageEncoder(Module, AutoCSR):
-    def __init__(self, rtio_core, enable):
+    def __init__(self, kcsrs, rtio_counter, enable):
         self.source = stream.Endpoint([("data", message_len)])
 
         self.overflow = CSRStatus()
@@ -50,27 +50,15 @@ class MessageEncoder(Module, AutoCSR):
 
         # # #
 
-        kcsrs = rtio_core.kcsrs
-
         input_output_stb = Signal()
         input_output = Record(input_output_layout)
-        if hasattr(kcsrs, "o_data"):
+        o_data = kcsrs.o_data.storage
-            o_data = kcsrs.o_data.storage
+        o_address = kcsrs.o_address.storage
-        else:
+        i_data = kcsrs.i_data.status
-            o_data = 0
-        if hasattr(kcsrs, "o_address"):
-            o_address = kcsrs.o_address.storage
-        else:
-            o_address = 0
-        if hasattr(kcsrs, "i_data"):
-            i_data = kcsrs.i_data.status
-        else:
-            i_data = 0
         self.comb += [
             input_output.channel.eq(kcsrs.chan_sel.storage),
             input_output.address_padding.eq(o_address),
-            input_output.rtio_counter.eq(
+            input_output.rtio_counter.eq(rtio_counter),
-                rtio_core.counter.value_sys << rtio_core.fine_ts_width),
             If(kcsrs.o_we.re,
                 input_output.message_type.eq(MessageType.output.value),
                 input_output.timestamp.eq(kcsrs.o_timestamp.storage),
@@ -88,10 +76,10 @@ class MessageEncoder(Module, AutoCSR):
         self.comb += [
             exception.message_type.eq(MessageType.exception.value),
             exception.channel.eq(kcsrs.chan_sel.storage),
-            exception.rtio_counter.eq(
+            exception.rtio_counter.eq(rtio_counter),
-                rtio_core.counter.value_sys << rtio_core.fine_ts_width),
         ]
-        for ename in ("o_underflow_reset", "o_sequence_error_reset",
+        for ename in ("reset", "reset_phy",
+                      "o_underflow_reset", "o_sequence_error_reset",
                       "o_collision_reset", "i_overflow_reset"):
             self.comb += \
                 If(getattr(kcsrs, ename).re,
@@ -99,28 +87,11 @@ class MessageEncoder(Module, AutoCSR):
                     exception.exception_type.eq(
                         getattr(ExceptionType, ename).value)
                 )
-        for rname in "reset", "reset_phy":
-            r_d = Signal(reset=1)
-            r = getattr(kcsrs, rname).storage
-            self.sync += r_d.eq(r)
-            self.comb += [
-                If(r & ~r_d,
-                    exception_stb.eq(1),
-                    exception.exception_type.eq(
-                        getattr(ExceptionType, rname+"_rising").value)
-                ),
-                If(~r & r_d,
-                    exception_stb.eq(1),
-                    exception.exception_type.eq(
-                        getattr(ExceptionType, rname+"_falling").value)
-                )
-            ]
 
         stopped = Record(stopped_layout)
         self.comb += [
             stopped.message_type.eq(MessageType.stopped.value),
-            stopped.rtio_counter.eq(
+            stopped.rtio_counter.eq(rtio_counter),
-                rtio_core.counter.value_sys << rtio_core.fine_ts_width),
         ]
 
         enable_r = Signal()
@@ -210,13 +181,13 @@ class DMAWriter(Module, AutoCSR):
 
 
 class Analyzer(Module, AutoCSR):
-    def __init__(self, rtio_core, membus, fifo_depth=128):
+    def __init__(self, kcsrs, rtio_counter, membus, fifo_depth=128):
         # shutdown procedure: set enable to 0, wait until busy=0
         self.enable = CSRStorage()
         self.busy = CSRStatus()
 
         self.submodules.message_encoder = MessageEncoder(
-            rtio_core, self.enable.storage)
+            kcsrs, rtio_counter, self.enable.storage)
         self.submodules.fifo = stream.SyncFIFO(
             [("data", message_len)], fifo_depth, True)
         self.submodules.converter = stream.Converter(

artiq/gateware/rtio/cdc.py

@@ -0,0 +1,66 @@
+from migen import *
+from migen.genlib.cdc import *
+
+
+__all__ = ["GrayCodeTransfer", "RTIOCounter", "BlindTransfer"]
+
+
+# note: transfer is in rtio/sys domains and not affected by the reset CSRs
+class GrayCodeTransfer(Module):
+    def __init__(self, width):
+        self.i = Signal(width)  # in rtio domain
+        self.o = Signal(width)  # in sys domain
+
+        # # #
+
+        # convert to Gray code
+        value_gray_rtio = Signal(width)
+        self.sync.rtio += value_gray_rtio.eq(self.i ^ self.i[1:])
+        # transfer to system clock domain
+        value_gray_sys = Signal(width)
+        value_gray_rtio.attr.add("no_retiming")
+        self.specials += MultiReg(value_gray_rtio, value_gray_sys)
+        # convert back to binary
+        value_sys = Signal(width)
+        self.comb += value_sys[-1].eq(value_gray_sys[-1])
+        for i in reversed(range(width-1)):
+            self.comb += value_sys[i].eq(value_sys[i+1] ^ value_gray_sys[i])
+        self.sync += self.o.eq(value_sys)
+
+
+class RTIOCounter(Module):
+    def __init__(self, width):
+        self.width = width
+        # Timestamp counter in RTIO domain
+        self.value_rtio = Signal(width)
+        # Timestamp counter resynchronized to sys domain
+        # Lags behind value_rtio, monotonic and glitch-free
+        self.value_sys = Signal(width)
+
+        # # #
+
+        # note: counter is in rtio domain and never affected by the reset CSRs
+        self.sync.rtio += self.value_rtio.eq(self.value_rtio + 1)
+        gt = GrayCodeTransfer(width)
+        self.submodules += gt
+        self.comb += gt.i.eq(self.value_rtio), self.value_sys.eq(gt.o)
+
+
+class BlindTransfer(Module):
+    def __init__(self):
+        self.i = Signal()
+        self.o = Signal()
+
+        ps = PulseSynchronizer("rio", "rsys")
+        ps_ack = PulseSynchronizer("rsys", "rio")
+        self.submodules += ps, ps_ack
+        blind = Signal()
+        self.sync.rio += [
+            If(self.i, blind.eq(1)),
+            If(ps_ack.o, blind.eq(0))
+        ]
+        self.comb += [
+            ps.i.eq(self.i & ~blind),
+            ps_ack.i.eq(ps.o),
+            self.o.eq(ps.o)
+        ]

artiq/gateware/rtio/core.py

@@ -3,73 +3,11 @@ from operator import and_
 
 from migen import *
 from migen.genlib.record import Record
-from migen.genlib.cdc import *
 from migen.genlib.fifo import AsyncFIFO
 from migen.genlib.resetsync import AsyncResetSynchronizer
-from misoc.interconnect.csr import *
 
-from artiq.gateware.rtio import rtlink
+from artiq.gateware.rtio import cri, rtlink
-
+from artiq.gateware.rtio.cdc import *
-
-# note: transfer is in rtio/sys domains and not affected by the reset CSRs
-class _GrayCodeTransfer(Module):
-    def __init__(self, width):
-        self.i = Signal(width)  # in rtio domain
-        self.o = Signal(width)  # in sys domain
-
-        # # #
-
-        # convert to Gray code
-        value_gray_rtio = Signal(width)
-        self.sync.rtio += value_gray_rtio.eq(self.i ^ self.i[1:])
-        # transfer to system clock domain
-        value_gray_sys = Signal(width)
-        value_gray_rtio.attr.add("no_retiming")
-        self.specials += MultiReg(value_gray_rtio, value_gray_sys)
-        # convert back to binary
-        value_sys = Signal(width)
-        self.comb += value_sys[-1].eq(value_gray_sys[-1])
-        for i in reversed(range(width-1)):
-            self.comb += value_sys[i].eq(value_sys[i+1] ^ value_gray_sys[i])
-        self.sync += self.o.eq(value_sys)
-
-
-class _RTIOCounter(Module):
-    def __init__(self, width):
-        self.width = width
-        # Timestamp counter in RTIO domain
-        self.value_rtio = Signal(width)
-        # Timestamp counter resynchronized to sys domain
-        # Lags behind value_rtio, monotonic and glitch-free
-        self.value_sys = Signal(width)
-
-        # # #
-
-        # note: counter is in rtio domain and never affected by the reset CSRs
-        self.sync.rtio += self.value_rtio.eq(self.value_rtio + 1)
-        gt = _GrayCodeTransfer(width)
-        self.submodules += gt
-        self.comb += gt.i.eq(self.value_rtio), self.value_sys.eq(gt.o)
-
-
-class _BlindTransfer(Module):
-    def __init__(self):
-        self.i = Signal()
-        self.o = Signal()
-
-        ps = PulseSynchronizer("rio", "rsys")
-        ps_ack = PulseSynchronizer("rsys", "rio")
-        self.submodules += ps, ps_ack
-        blind = Signal()
-        self.sync.rio += [
-            If(self.i, blind.eq(1)),
-            If(ps_ack.o, blind.eq(0))
-        ]
-        self.comb += [
-            ps.i.eq(self.i & ~blind),
-            ps_ack.i.eq(ps.o),
-            self.o.eq(ps.o)
-        ]
 
 
 # CHOOSING A GUARD TIME
@@ -227,7 +165,7 @@ class _OutputManager(Module):
             interface.stb.eq(dout_stb & dout_ack)
         ]
 
-        busy_transfer = _BlindTransfer()
+        busy_transfer = BlindTransfer()
         self.submodules += busy_transfer
         self.comb += [
             busy_transfer.i.eq(interface.stb & interface.busy),
@@ -289,7 +227,7 @@ class _InputManager(Module):
             fifo.re.eq(self.re)
         ]
 
-        overflow_transfer = _BlindTransfer()
+        overflow_transfer = BlindTransfer()
         self.submodules += overflow_transfer
         self.comb += [
             overflow_transfer.i.eq(fifo.we & ~fifo.writable),
@@ -326,81 +264,47 @@ class LogChannel:
         self.overrides = []
 
 
-class _KernelCSRs(AutoCSR):
+class Core(Module):
-    def __init__(self, chan_sel_width,
+    def __init__(self, channels, fine_ts_width=None, guard_io_cycles=20):
-                 data_width, address_width, full_ts_width):
+        if fine_ts_width is None:
-        self.reset = CSRStorage(reset=1)
+            fine_ts_width = max(rtlink.get_fine_ts_width(c.interface)
-        self.reset_phy = CSRStorage(reset=1)
+                                for c in channels)
-        self.chan_sel = CSRStorage(chan_sel_width)
 
-        if data_width:
+        self.cri = cri.Interface()
-            self.o_data = CSRStorage(data_width, write_from_dev=True)
+        self.comb += self.cri.arb_gnt.eq(1)
-        if address_width:
-            self.o_address = CSRStorage(address_width, write_from_dev=True)
-        self.o_timestamp = CSRStorage(full_ts_width)
-        self.o_we = CSR()
-        self.o_status = CSRStatus(5)
-        self.o_underflow_reset = CSR()
-        self.o_sequence_error_reset = CSR()
-        self.o_collision_reset = CSR()
-        self.o_busy_reset = CSR()
-
-        if data_width:
-            self.i_data = CSRStatus(data_width)
-        self.i_timestamp = CSRStatus(full_ts_width)
-        self.i_re = CSR()
-        self.i_status = CSRStatus(2)
-        self.i_overflow_reset = CSR()
-
-        self.counter = CSRStatus(full_ts_width)
-        self.counter_update = CSR()
-
-
-class RTIO(Module):
-    def __init__(self, channels, full_ts_width=63, guard_io_cycles=20):
-        data_width = max(rtlink.get_data_width(c.interface)
-                         for c in channels)
-        address_width = max(rtlink.get_address_width(c.interface)
-                            for c in channels)
-        fine_ts_width = max(rtlink.get_fine_ts_width(c.interface)
-                            for c in channels)
-
-        self.data_width = data_width
-        self.address_width = address_width
-        self.fine_ts_width = fine_ts_width
-
-        # CSRs
-        self.kcsrs = _KernelCSRs(bits_for(len(channels)-1),
-                                 data_width, address_width,
-                                 full_ts_width)
 
         # Clocking/Reset
         # Create rsys, rio and rio_phy domains based on sys and rtio
-        # with reset controlled by CSR.
+        # with reset controlled by CRI.
+        cmd_reset = Signal(reset=1)
+        cmd_reset_phy = Signal(reset=1)
+        self.sync += [
+            cmd_reset.eq(self.cri.cmd == cri.commands["reset"]),
+            cmd_reset_phy.eq(self.cri.cmd == cri.commands["reset_phy"])
+        ]
+        cmd_reset.attr.add("no_retiming")
+        cmd_reset_phy.attr.add("no_retiming")
+
         self.clock_domains.cd_rsys = ClockDomain()
         self.clock_domains.cd_rio = ClockDomain()
         self.clock_domains.cd_rio_phy = ClockDomain()
         self.comb += [
             self.cd_rsys.clk.eq(ClockSignal()),
-            self.cd_rsys.rst.eq(self.kcsrs.reset.storage)
+            self.cd_rsys.rst.eq(cmd_reset)
         ]
         self.comb += self.cd_rio.clk.eq(ClockSignal("rtio"))
         self.specials += AsyncResetSynchronizer(
-            self.cd_rio,
+            self.cd_rio, cmd_reset)
-            self.kcsrs.reset.storage | ResetSignal("rtio",
-                                                   allow_reset_less=True))
         self.comb += self.cd_rio_phy.clk.eq(ClockSignal("rtio"))
         self.specials += AsyncResetSynchronizer(
-            self.cd_rio_phy,
+            self.cd_rio_phy, cmd_reset_phy)
-            self.kcsrs.reset_phy.storage | ResetSignal("rtio",
-                                                       allow_reset_less=True))
 
         # Managers
-        self.submodules.counter = _RTIOCounter(full_ts_width - fine_ts_width)
+        self.submodules.counter = RTIOCounter(len(self.cri.o_timestamp) - fine_ts_width)
 
         i_datas, i_timestamps = [], []
         o_statuses, i_statuses = [], []
-        sel = self.kcsrs.chan_sel.storage
+        sel = self.cri.chan_sel[:16]
         for n, channel in enumerate(channels):
             if isinstance(channel, LogChannel):
                 i_datas.append(0)
@@ -416,30 +320,26 @@ class RTIO(Module):
             self.submodules += o_manager
 
             if hasattr(o_manager.ev, "data"):
-                self.comb += o_manager.ev.data.eq(
+                self.comb += o_manager.ev.data.eq(self.cri.o_data)
-                    self.kcsrs.o_data.storage)
             if hasattr(o_manager.ev, "address"):
-                self.comb += o_manager.ev.address.eq(
+                self.comb += o_manager.ev.address.eq(self.cri.o_address)
-                    self.kcsrs.o_address.storage)
+            ts_shift = len(self.cri.o_timestamp) - len(o_manager.ev.timestamp)
-            ts_shift = (len(self.kcsrs.o_timestamp.storage)
+            self.comb += o_manager.ev.timestamp.eq(self.cri.o_timestamp[ts_shift:])
-                        - len(o_manager.ev.timestamp))
-            self.comb += o_manager.ev.timestamp.eq(
-                self.kcsrs.o_timestamp.storage[ts_shift:])
 
-            self.comb += o_manager.we.eq(selected & self.kcsrs.o_we.re)
+            self.comb += o_manager.we.eq(selected & (self.cri.cmd == cri.commands["write"]))
 
             underflow = Signal()
             sequence_error = Signal()
             collision = Signal()
             busy = Signal()
             self.sync.rsys += [
-                If(selected & self.kcsrs.o_underflow_reset.re,
+                If(selected & (self.cri.cmd == cri.commands["o_underflow_reset"]),
                    underflow.eq(0)),
-                If(selected & self.kcsrs.o_sequence_error_reset.re,
+                If(selected & (self.cri.cmd == cri.commands["o_sequence_error_reset"]),
                    sequence_error.eq(0)),
-                If(selected & self.kcsrs.o_collision_reset.re,
+                If(selected & (self.cri.cmd == cri.commands["o_collision_reset"]),
                    collision.eq(0)),
-                If(selected & self.kcsrs.o_busy_reset.re,
+                If(selected & (self.cri.cmd == cri.commands["o_busy_reset"]),
                    busy.eq(0)),
                 If(o_manager.underflow, underflow.eq(1)),
                 If(o_manager.sequence_error, sequence_error.eq(1)),
@@ -462,17 +362,16 @@ class RTIO(Module):
                 else:
                     i_datas.append(0)
                 if channel.interface.i.timestamped:
-                    ts_shift = (len(self.kcsrs.i_timestamp.status)
+                    ts_shift = (len(self.cri.i_timestamp) - len(i_manager.ev.timestamp))
-                                - len(i_manager.ev.timestamp))
                     i_timestamps.append(i_manager.ev.timestamp << ts_shift)
                 else:
                     i_timestamps.append(0)
 
-                self.comb += i_manager.re.eq(selected & self.kcsrs.i_re.re)
+                self.comb += i_manager.re.eq(selected & (self.cri.cmd == cri.commands["read"]))
 
                 overflow = Signal()
                 self.sync.rsys += [
-                    If(selected & self.kcsrs.i_overflow_reset.re,
+                    If(selected & (self.cri.cmd == cri.commands["i_overflow_reset"]),
                        overflow.eq(0)),
                     If(i_manager.overflow,
                        overflow.eq(1))
@@ -483,28 +382,11 @@ class RTIO(Module):
                 i_datas.append(0)
                 i_timestamps.append(0)
                 i_statuses.append(0)
-        if data_width:
-            self.comb += self.kcsrs.i_data.status.eq(Array(i_datas)[sel])
         self.comb += [
-            self.kcsrs.i_timestamp.status.eq(Array(i_timestamps)[sel]),
+            self.cri.i_data.eq(Array(i_datas)[sel]),
-            self.kcsrs.o_status.status.eq(Array(o_statuses)[sel]),
+            self.cri.i_timestamp.eq(Array(i_timestamps)[sel]),
-            self.kcsrs.i_status.status.eq(Array(i_statuses)[sel])
+            self.cri.o_status.eq(Array(o_statuses)[sel]),
+            self.cri.i_status.eq(Array(i_statuses)[sel])
         ]
 
-        # Counter access
+        self.comb += self.cri.counter.eq(self.counter.value_sys << fine_ts_width)
-        self.sync += \
-           If(self.kcsrs.counter_update.re,
-               self.kcsrs.counter.status.eq(self.counter.value_sys
-                                                << fine_ts_width)
-           )
-
-        # Auto clear/zero pad event data
-        self.comb += [
-            self.kcsrs.o_data.dat_w.eq(0),
-            self.kcsrs.o_data.we.eq(self.kcsrs.o_timestamp.re),
-            self.kcsrs.o_address.dat_w.eq(0),
-            self.kcsrs.o_address.we.eq(self.kcsrs.o_timestamp.re),
-        ]
-
-    def get_csrs(self):
-        return self.kcsrs.get_csrs()

artiq/gateware/rtio/cri.py

@@ -0,0 +1,196 @@
+"""Common RTIO Interface"""
+
+from migen import *
+from migen.genlib.record import *
+
+from misoc.interconnect.csr import *
+
+
+commands = {
+    "nop": 0,
+    "reset": 1,
+    "reset_phy": 2,
+
+    "write": 3,
+    "read": 4,
+
+    "o_underflow_reset": 5,
+    "o_sequence_error_reset": 6,
+    "o_collision_reset": 7,
+    "o_busy_reset": 8,
+    "i_overflow_reset": 9
+}
+
+
+layout = [
+    ("arb_req", 1, DIR_M_TO_S),
+    ("arb_gnt", 1, DIR_S_TO_M),
+
+    ("cmd", 4, DIR_M_TO_S),
+    # 8 MSBs of chan_sel are used to select core
+    ("chan_sel", 24, DIR_M_TO_S),
+
+    ("o_data", 512, DIR_M_TO_S),
+    ("o_address", 16, DIR_M_TO_S),
+    ("o_timestamp", 64, DIR_M_TO_S),
+    # o_status bits:
+    # <0:wait> <1:underflow> <2:sequence_error> <3:collision> <4:busy>
+    ("o_status", 5, DIR_S_TO_M),
+
+    ("i_data", 32, DIR_S_TO_M),
+    ("i_timestamp", 64, DIR_S_TO_M),
+    # i_status bits:
+    # <0:wait> <1:overflow>
+    ("i_status", 2, DIR_S_TO_M),
+
+    ("counter", 64, DIR_S_TO_M)
+]
+
+
+class Interface(Record):
+    def __init__(self):
+        Record.__init__(self, layout)
+
+
+class KernelInitiator(Module, AutoCSR):
+    def __init__(self, cri=None):
+        self.arb_req = CSRStorage()
+        self.arb_gnt = CSRStatus()
+
+        self.reset = CSR()
+        self.reset_phy = CSR()
+        self.chan_sel = CSRStorage(24)
+
+        self.o_data = CSRStorage(512, write_from_dev=True)
+        self.o_address = CSRStorage(16)
+        self.o_timestamp = CSRStorage(64)
+        self.o_we = CSR()
+        self.o_status = CSRStatus(5)
+        self.o_underflow_reset = CSR()
+        self.o_sequence_error_reset = CSR()
+        self.o_collision_reset = CSR()
+        self.o_busy_reset = CSR()
+
+        self.i_data = CSRStatus(32)
+        self.i_timestamp = CSRStatus(64)
+        self.i_re = CSR()
+        self.i_status = CSRStatus(2)
+        self.i_overflow_reset = CSR()
+
+        self.counter = CSRStatus(64)
+        self.counter_update = CSR()
+
+        if cri is None:
+            cri = Interface()
+        self.cri = cri
+
+        # # #
+
+        self.comb += [
+            self.cri.arb_req.eq(self.arb_req.storage),
+            self.arb_gnt.status.eq(self.cri.arb_gnt),
+
+            self.cri.cmd.eq(commands["nop"]),
+            If(self.reset.re, self.cri.cmd.eq(commands["reset"])),
+            If(self.reset_phy.re, self.cri.cmd.eq(commands["reset_phy"])),
+            If(self.o_we.re, self.cri.cmd.eq(commands["write"])),
+            If(self.i_re.re, self.cri.cmd.eq(commands["read"])),
+            If(self.o_underflow_reset.re, self.cri.cmd.eq(commands["o_underflow_reset"])),
+            If(self.o_sequence_error_reset.re, self.cri.cmd.eq(commands["o_sequence_error_reset"])),
+            If(self.o_collision_reset.re, self.cri.cmd.eq(commands["o_collision_reset"])),
+            If(self.o_busy_reset.re, self.cri.cmd.eq(commands["o_busy_reset"])),
+            If(self.i_overflow_reset.re, self.cri.cmd.eq(commands["i_overflow_reset"])),
+
+            self.cri.chan_sel.eq(self.chan_sel.storage),
+
+            self.cri.o_data.eq(self.o_data.storage),
+            self.cri.o_address.eq(self.o_address.storage),
+            self.cri.o_timestamp.eq(self.o_timestamp.storage),
+            self.o_status.status.eq(self.cri.o_status),
+
+            self.i_data.status.eq(self.cri.i_data),
+            self.i_timestamp.status.eq(self.cri.i_timestamp),
+            self.i_status.status.eq(self.cri.i_status),
+
+            self.o_data.dat_w.eq(0),
+            self.o_data.we.eq(self.o_timestamp.re),
+        ]
+        self.sync += If(self.counter_update.re, self.counter.status.eq(self.cri.counter))
+
+
+class CRIDecoder(Module):
+    def __init__(self, slaves=2, master=None):
+        if isinstance(slaves, int):
+            slaves = [Interface() for _ in range(slaves)]
+        if master is None:
+            master = Interface()
+        self.slaves = slaves
+        self.master = master
+
+        # # #
+
+        selected = Signal(8)
+        self.sync += selected.eq(self.master.chan_sel[16:])
+
+        # master -> slave
+        for n, slave in enumerate(slaves):
+            for name, size, direction in layout:
+                if direction == DIR_M_TO_S and name != "cmd":
+                    self.comb += getattr(slave, name).eq(getattr(master, name))
+            self.comb += If(selected == n, slave.cmd.eq(master.cmd))
+
+        # slave -> master
+        cases = dict()
+        for n, slave in enumerate(slaves):
+            cases[n] = []
+            for name, size, direction in layout:
+                if direction == DIR_S_TO_M:
+                    cases[n].append(getattr(master, name).eq(getattr(slave, name)))
+        self.comb += Case(selected, cases)
+
+
+class CRIArbiter(Module):
+    def __init__(self, masters=2, slave=None):
+        if isinstance(masters, int):
+            masters = [Interface() for _ in range(masters)]
+        if slave is None:
+            slave = Interface()
+        self.masters = masters
+        self.slave = slave
+
+        # # #
+
+        if len(masters) == 1:
+            self.comb += masters[0].connect(slave)
+        else:
+            selected = Signal(max=len(masters))
+
+            # mux master->slave signals
+            for name, size, direction in layout:
+                if direction == DIR_M_TO_S:
+                    choices = Array(getattr(m, name) for m in masters)
+                    self.comb += getattr(slave, name).eq(choices[selected])
+
+            # connect slave->master signals
+            for name, size, direction in layout:
+                if direction == DIR_S_TO_M:
+                    source = getattr(slave, name)
+                    for i, m in enumerate(masters):
+                        dest = getattr(m, name)
+                        if name == "arb_gnt":
+                            self.comb += dest.eq(source & (selected == i))
+                        else:
+                            self.comb += dest.eq(source)
+
+            # select master
+            self.sync += \
+                If(~slave.arb_req,
+                    [If(m.arb_req, selected.eq(i)) for i, m in enumerate(masters)]
+                )
+
+
+class CRIInterconnectShared(Module):
+    def __init__(self, masters=2, slaves=2):
+        shared = Interface()
+        self.submodules.arbiter = CRIArbiter(masters, shared)
+        self.submodules.decoder = CRIDecoder(slaves, shared)

artiq/gateware/rtio/dma.py

@@ -0,0 +1,332 @@
+from migen import *
+from migen.genlib.record import Record, layout_len
+from migen.genlib.fsm import FSM
+from misoc.interconnect.csr import *
+from misoc.interconnect import stream, wishbone
+
+from artiq.gateware.rtio import cri
+
+
+class WishboneReader(Module):
+    def __init__(self, bus=None):
+        if bus is None:
+            bus = wishbone.Interface
+        self.bus = bus
+
+        aw = len(bus.adr)
+        dw = len(bus.dat_w)        
+        self.sink = stream.Endpoint([("address", aw)])
+        self.source = stream.Endpoint([("data", dw)])
+
+        # # #
+
+        bus_stb = Signal()
+        data_reg_loaded = Signal()
+
+        self.comb += [
+            bus_stb.eq(self.sink.stb & (~data_reg_loaded | self.source.ack)),
+            bus.cyc.eq(bus_stb),
+            bus.stb.eq(bus_stb),
+            bus.adr.eq(self.sink.address),
+            self.sink.ack.eq(bus.ack),
+            self.source.stb.eq(data_reg_loaded),
+        ]
+        self.sync += [
+            If(self.source.ack, data_reg_loaded.eq(0)),
+            If(bus.ack,
+                data_reg_loaded.eq(1),
+                self.source.data.eq(bus.dat_r),
+                self.source.eop.eq(self.sink.eop)
+            )
+        ]
+
+
+class DMAReader(Module, AutoCSR):
+    def __init__(self, membus, enable):
+        aw = len(membus.adr)
+        data_alignment = log2_int(len(membus.dat_w)//8)
+
+        self.submodules.wb_reader = WishboneReader(membus)
+        self.source = self.wb_reader.source
+
+        # All numbers in bytes
+        self.base_address = CSRStorage(aw + data_alignment,
+                                       alignment_bits=data_alignment)
+        self.last_address = CSRStorage(aw + data_alignment,
+                                       alignment_bits=data_alignment)
+
+        # # #
+
+        enable_r = Signal()
+        address = self.wb_reader.sink
+        self.sync += [
+            enable_r.eq(enable),
+            If(enable & ~enable_r,
+                address.address.eq(self.base_address.storage),
+                address.eop.eq(0),
+                address.stb.eq(1)
+            ),
+            If(address.stb & address.ack,
+                If(address.eop,
+                    address.stb.eq(0)
+                ).Else(
+                    address.address.eq(address.address + 1),
+                    If(~enable | (address.address == self.last_address.storage),
+                        address.eop.eq(1)
+                    )
+                )
+            )
+        ]
+
+
+class RawSlicer(Module):
+    def __init__(self, in_size, out_size, granularity):
+        g = granularity
+
+        self.sink = stream.Endpoint([("data", in_size*g)])
+        self.source = Signal(out_size*g)
+        self.source_stb = Signal()
+        self.source_consume = Signal(max=out_size+1)
+
+        # # #
+
+        # worst-case buffer space required (when loading):
+        #          <data being shifted out>   <new incoming word>   <EOP marker>   
+        buf_size =       out_size - 1       +       in_size       +      1
+        buf = Signal(buf_size*g)
+        self.comb += self.source.eq(buf[:out_size])
+
+        level = Signal(max=buf_size+1)
+        next_level = Signal(max=buf_size+1)
+        self.sync += level.eq(next_level)
+        self.comb += next_level.eq(level)
+
+        load_buf = Signal()
+        shift_buf = Signal()
+
+        self.sync += [
+            If(load_buf, Case(level,
+                # note how the MSBs of the buffer are set to 0
+                # (including the EOP marker position) 
+                {i: buf[i*g:].eq(self.sink.data)
+                 for i in range(out_size)})),
+            If(shift_buf, buf.eq(buf >> self.source_consume*g))
+        ]
+
+        fsm = FSM(reset_state="FETCH")
+        self.submodules += fsm
+
+        fsm.act("FETCH",
+            self.sink.ack.eq(1),
+            load_buf.eq(1),
+            If(self.sink.stb,
+                If(self.sink.eop,
+                    # insert <granularity> bits of 0 to mark EOP
+                    next_level.eq(level + in_size + 1)
+                ).Else(
+                    next_level.eq(level + in_size)
+                )
+            ),
+            If(next_level >= out_size, NextState("OUTPUT"))
+        )
+        fsm.act("OUTPUT",
+            self.source_stb.eq(1),
+            shift_buf.eq(1),
+            next_level.eq(level - self.source_consume),
+            If(next_level < out_size, NextState("FETCH"))
+        )
+
+
+record_layout = [
+    ("length", 8),  # of whole record (header+data)
+    ("channel", 24),
+    ("timestamp", 64),
+    ("address", 16),
+    ("data", 512)  # variable length
+]
+
+
+class RecordConverter(Module):
+    def __init__(self, stream_slicer):
+        self.source = stream.Endpoint(record_layout)
+
+        hdrlen = layout_len(record_layout) - 512
+        record_raw = Record(record_layout)
+        self.comb += [
+            record_raw.raw_bits().eq(stream_slicer.source),
+
+            self.source.channel.eq(record_raw.channel),
+            self.source.timestamp.eq(record_raw.timestamp),
+            self.source.address.eq(record_raw.address),
+            Case(record_raw.length,
+                 {hdrlen+i*8: self.source.data.eq(record_raw.data[:])
+                  for i in range(512//8)}),
+
+            self.source.stb.eq(stream_slicer.source_stb),
+            self.source.eop.eq(record_raw.length == 0),
+            If(self.source.ack,
+                If(record_raw.length == 0,
+                    stream_slicer.source_consume.eq(1)
+                ).Else(
+                    stream_slicer.source_consume.eq(record_raw.length)
+                )
+            )
+        ]
+
+
+class RecordSlicer(Module):
+    def __init__(self, in_size):
+        self.submodules.raw_slicer = RawSlicer(
+            in_size, layout_len(record_layout)//8, 8)
+        self.submodules.record_converter = RecordConverter(self.raw_slicer)
+        self.sink = self.raw_slicer.sink
+        self.source = self.record_converter.source
+
+
+class TimeOffset(Module, AutoCSR):
+    def __init__(self):
+        self.time_offset = CSRStorage(64)
+        self.source = stream.Endpoint(record_layout)
+        self.sink = stream.Endpoint(record_layout)
+
+        # # #
+
+        pipe_ce = Signal()
+        self.sync += \
+            If(pipe_ce,
+                self.sink.payload.connect(self.source.payload,
+                                          leave_out={"timestamp"}),
+                self.source.payload.timestamp.eq(self.sink.payload.timestamp
+                                                 + self.time_offset.storage),
+                self.source.stb.eq(self.sink.stb)
+            )
+        self.comb += [
+            pipe_ce.eq(self.source.ack | ~self.source.stb),
+            self.sink.ack.eq(pipe_ce)
+        ]
+
+
+class CRIMaster(Module, AutoCSR):
+    def __init__(self):
+        self.arb_req = CSRStorage()
+        self.arb_gnt = CSRStatus()
+
+        self.error_status = CSRStatus(5)  # same encoding as RTIO status
+        self.error_underflow_reset = CSR()
+        self.error_sequence_error_reset = CSR()
+        self.error_collision_reset = CSR()
+        self.error_busy_reset = CSR()
+
+        self.error_channel = CSRStatus(24)
+        self.error_timestamp = CSRStatus(64)
+        self.error_address = CSRStatus(16)
+
+        self.sink = stream.Endpoint(record_layout)
+        self.cri = cri.Interface()
+        self.busy = Signal()
+
+        # # #
+
+        self.comb += [
+            self.cri.arb_req.eq(self.arb_req.storage),
+            self.arb_gnt.status.eq(self.cri.arb_gnt)
+        ]
+
+        error_set = Signal(4)
+        for i, rcsr in enumerate([self.error_underflow_reset, self.error_sequence_error_reset,
+                                  self.error_collision_reset, self.error_busy_reset]):
+            # bit 0 is RTIO wait and always 0 here
+            bit = i + 1
+            self.sync += [
+                If(error_set[i],
+                    self.error_status.status[bit].eq(1),
+                    self.error_channel.status.eq(self.sink.channel),
+                    self.error_timestamp.status.eq(self.sink.timestamp),
+                    self.error_address.status.eq(self.sink.address)
+                ),
+                If(rcsr.re, self.error_status.status[bit].eq(0))
+            ]
+
+        self.comb += [
+            self.cri.chan_sel.eq(self.sink.channel),
+            self.cri.o_timestamp.eq(self.sink.timestamp),
+            self.cri.o_address.eq(self.sink.address),
+        ]
+
+        fsm = FSM(reset_state="IDLE")
+        self.submodules += fsm
+
+        fsm.act("IDLE",
+            If(self.error_status.status == 0,
+                If(self.sink.stb, NextState("WRITE"))
+            ).Else(
+                # discard all data until errors are acked
+                self.sink.ack.eq(1)
+            )
+        )
+        fsm.act("WRITE",
+            self.busy.eq(1),
+            self.cri.cmd.eq(cri.commands["write"]),
+            NextState("CHECK_STATE")
+        )
+        fsm.act("CHECK_STATE",
+            self.busy.eq(1),
+            If(~self.cri.o_status,
+                self.sink.ack.eq(1),
+                NextState("IDLE")
+            ),
+            If(self.cri.o_status[1], NextState("UNDERFLOW")),
+            If(self.cri.o_status[2], NextState("SEQUENCE_ERROR")),
+            If(self.cri.o_status[3], NextState("COLLISION")),
+            If(self.cri.o_status[4], NextState("BUSY"))
+        )
+        for n, name in enumerate(["UNDERFLOW", "SEQUENCE_ERROR",
+                                  "COLLISION", "BUSY"]):
+            fsm.act(name,
+                self.busy.eq(1),
+                error_set.eq(1 << n),
+                self.cri.cmd.eq(cri.commands["o_" + name.lower() + "_reset"]),
+                self.sink.ack.eq(1),
+                NextState("IDLE")
+            )
+
+
+class DMA(Module):
+    def __init__(self, membus):
+        # shutdown procedure: set enable to 0, wait until busy=0
+        self.enable = CSRStorage()
+        self.busy = CSRStatus()
+
+        self.submodules.dma = DMAReader(membus, self.enable.storage)
+        self.submodules.slicer = RecordSlicer(len(membus.dat_w))
+        self.submodules.time_offset = TimeOffset()
+        self.submodules.cri_master = CRIMaster()
+        self.cri = self.cri_master.cri
+
+        self.comb += [
+            self.dma.source.connect(self.slicer.sink),
+            self.slicer.source.connect(self.time_offset.sink),
+            self.time_offset.source.connect(self.cri_master.sink)
+        ]
+
+        fsm = FSM(reset_state="IDLE")
+        self.submodules += fsm
+
+        fsm.act("IDLE",
+            If(self.enable.storage, NextState("FLOWING"))
+        )
+        fsm.act("FLOWING",
+            self.busy.status.eq(1),
+            If(self.cri_master.sink.stb & self.cri_master.sink.ack & self.cri_master.sink.eop,
+                NextState("WAIT_CRI_MASTER")
+            )
+        )
+        fsm.act("WAIT_CRI_MASTER",
+            self.busy.status.eq(1),
+            If(~self.cri_master.busy, NextState("IDLE"))
+        )
+
+    def get_csrs(self):
+        return ([self.enable, self.busy] +
+                self.dma.get_csrs() + self.time_offset.get_csrs() +
+                self.cri_master.get_csrs())

artiq/gateware/soc.py

@@ -39,8 +39,9 @@ class AMPSoC:
         self.submodules.timer_kernel = timer.Timer()
         self.register_kernel_cpu_csrdevice("timer_kernel")
 
-    def register_kernel_cpu_csrdevice(self, name):
+    def register_kernel_cpu_csrdevice(self, name, csrs=None):
-        csrs = getattr(self, name).get_csrs()
+        if csrs is None:
+            csrs = getattr(self, name).get_csrs()
         bank = wishbone.CSRBank(csrs)
         self.submodules += bank
         self.kernel_cpu.add_wb_slave(mem_decoder(self.mem_map[name]),

artiq/gateware/targets/kc705.py

@@ -101,10 +101,11 @@ _ams101_dac = [
 
 class _NIST_Ions(MiniSoC, AMPSoC):
     mem_map = {
-        "timer_kernel":  0x10000000, # (shadow @0x90000000)
+        "timer_kernel":  0x10000000,
-        "rtio":          0x20000000, # (shadow @0xa0000000)
+        "rtio":          0x20000000,
-        "i2c":           0x30000000, # (shadow @0xb0000000)
+        "rtio_dma":      0x30000000,
-        "mailbox":       0x70000000  # (shadow @0xf0000000)
+        "i2c":           0x50000000,
+        "mailbox":       0x70000000
     }
     mem_map.update(MiniSoC.mem_map)
 
@@ -142,8 +143,14 @@ class _NIST_Ions(MiniSoC, AMPSoC):
     def add_rtio(self, rtio_channels):
         self.submodules.rtio_crg = _RTIOCRG(self.platform, self.crg.cd_sys.clk)
         self.csr_devices.append("rtio_crg")
-        self.submodules.rtio = rtio.RTIO(rtio_channels)
+        self.submodules.rtio_core = rtio.Core(rtio_channels)
+        self.submodules.rtio = rtio.KernelInitiator()
+        self.submodules.rtio_dma = rtio.DMA(self.get_native_sdram_if())
         self.register_kernel_cpu_csrdevice("rtio")
+        self.register_kernel_cpu_csrdevice("rtio_dma")
+        self.submodules.cri_con = rtio.CRIInterconnectShared(
+            [self.rtio.cri, self.rtio_dma.cri],
+            [self.rtio_core.cri])
         self.submodules.rtio_moninj = rtio.MonInj(rtio_channels)
         self.csr_devices.append("rtio_moninj")
 
@@ -153,8 +160,8 @@ class _NIST_Ions(MiniSoC, AMPSoC):
             self.crg.cd_sys.clk,
             self.rtio_crg.cd_rtio.clk)
 
-        self.submodules.rtio_analyzer = rtio.Analyzer(self.rtio,
+        self.submodules.rtio_analyzer = rtio.Analyzer(
-            self.get_native_sdram_if())
+            self.rtio, self.rtio_core.cri.counter, self.get_native_sdram_if())
         self.csr_devices.append("rtio_analyzer")
 
 

artiq/gateware/targets/kc705_drtio_master.py

@@ -0,0 +1,126 @@
+#!/usr/bin/env python3.5
+
+import argparse
+
+from migen import *
+from migen.build.generic_platform import *
+
+from misoc.targets.kc705 import MiniSoC, soc_kc705_args, soc_kc705_argdict
+from misoc.integration.builder import builder_args, builder_argdict
+
+from artiq.gateware.soc import AMPSoC, build_artiq_soc
+from artiq.gateware import rtio
+from artiq.gateware.rtio.phy import ttl_simple
+from artiq.gateware.drtio.transceiver import gtx_7series
+from artiq.gateware.drtio import DRTIOMaster
+from artiq import __version__ as artiq_version
+
+
+fmc_clock_io = [
+    ("ad9154_refclk", 0,
+        Subsignal("p", Pins("HPC:GBTCLK0_M2C_P")),
+        Subsignal("n", Pins("HPC:GBTCLK0_M2C_N")),
+    )
+]
+
+
+class Master(MiniSoC, AMPSoC):
+    mem_map = {
+        "timer_kernel":  0x10000000,
+        "rtio":          0x20000000,
+        "rtio_dma":      0x30000000,
+        "mailbox":       0x70000000
+    }
+    mem_map.update(MiniSoC.mem_map)
+
+    def __init__(self, cfg, medium, **kwargs):
+        MiniSoC.__init__(self,
+                         cpu_type="or1k",
+                         sdram_controller_type="minicon",
+                         l2_size=128*1024,
+                         with_timer=False,
+                         ident=artiq_version,
+                         **kwargs)
+        AMPSoC.__init__(self)
+
+        platform = self.platform
+
+        if medium == "sfp":
+            self.comb += platform.request("sfp_tx_disable_n").eq(1)
+            tx_pads = platform.request("sfp_tx")
+            rx_pads = platform.request("sfp_rx")
+        elif medium == "sma":
+            tx_pads = platform.request("user_sma_mgt_tx")
+            rx_pads = platform.request("user_sma_mgt_rx")
+        else:
+            raise ValueError
+
+        if cfg == "simple_gbe":
+            # GTX_1000BASE_BX10 Ethernet compatible, 62.5MHz RTIO clock
+            # simple TTLs
+            self.submodules.transceiver = gtx_7series.GTX_1000BASE_BX10(
+                clock_pads=platform.request("sgmii_clock"),
+                tx_pads=tx_pads,
+                rx_pads=rx_pads,
+                sys_clk_freq=self.clk_freq,
+                clock_div2=True)
+        elif cfg == "sawg_3g":
+            # 3Gb link, 150MHz RTIO clock
+            # with SAWG on local RTIO and AD9154-FMC-EBZ
+            platform.register_extension(fmc_clock_io)
+            self.submodules.transceiver = gtx_7series.GTX_3G(
+                clock_pads=platform.request("ad9154_refclk"),
+                tx_pads=tx_pads,
+                rx_pads=rx_pads,
+                sys_clk_freq=self.clk_freq)
+        else:
+            raise ValueError
+        self.submodules.drtio = DRTIOMaster(self.transceiver)
+        self.csr_devices.append("drtio")
+
+        rtio_clk_period = 1e9/self.transceiver.rtio_clk_freq
+        platform.add_period_constraint(self.transceiver.txoutclk, rtio_clk_period)
+        platform.add_period_constraint(self.transceiver.rxoutclk, rtio_clk_period)
+        platform.add_false_path_constraints(
+            self.crg.cd_sys.clk,
+            self.transceiver.txoutclk, self.transceiver.rxoutclk)
+
+        rtio_channels = []
+        for i in range(8):
+            phy = ttl_simple.Output(platform.request("user_led", i))
+            self.submodules += phy
+            rtio_channels.append(rtio.Channel.from_phy(phy))
+        for sma in "user_sma_gpio_p", "user_sma_gpio_n":
+            phy = ttl_simple.Inout(platform.request(sma))
+            self.submodules += phy
+            rtio_channels.append(rtio.Channel.from_phy(phy))
+        self.submodules.rtio_core = rtio.Core(rtio_channels, 3)
+
+        self.submodules.rtio = rtio.KernelInitiator()
+        self.submodules.rtio_dma = rtio.DMA(self.get_native_sdram_if())
+        self.register_kernel_cpu_csrdevice("rtio")
+        self.register_kernel_cpu_csrdevice("rtio_dma")
+        self.submodules.cri_con = rtio.CRIInterconnectShared(
+            [self.rtio.cri, self.rtio_dma.cri],
+            [self.drtio.cri, self.rtio_core.cri])
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="ARTIQ with DRTIO on KC705 - Master")
+    builder_args(parser)
+    soc_kc705_args(parser)
+    parser.add_argument("-c", "--config", default="simple_gbe",
+                        help="configuration: simple_gbe/sawg_3g "
+                             "(default: %(default)s)")
+    parser.add_argument("--medium", default="sfp",
+                        help="medium to use for transceiver link: sfp/sma "
+                             "(default: %(default)s)")
+    args = parser.parse_args()
+
+    soc = Master(args.config, args.medium, **soc_kc705_argdict(args))
+    build_artiq_soc(soc, builder_argdict(args))
+
+
+if __name__ == "__main__":
+    main()

artiq/gateware/targets/kc705_drtio_satellite.py

@@ -0,0 +1,220 @@
+import argparse
+
+from migen import *
+from migen.build.generic_platform import *
+from migen.build.platforms import kc705
+
+from misoc.cores.i2c import *
+from misoc.cores.sequencer import *
+
+from artiq.gateware import rtio
+from artiq.gateware.rtio.phy import ttl_simple
+from artiq.gateware.drtio.transceiver import gtx_7series
+from artiq.gateware.drtio import DRTIOSatellite
+
+
+# TODO: parameters for sawg_3g
+def get_i2c_program(sys_clk_freq):
+    # NOTE: the logical parameters DO NOT MAP to physical values written
+    # into registers. They have to be mapped; see the datasheet.
+    # DSPLLsim reports the logical parameters in the design summary, not
+    # the physical register values (but those are present separately).
+    N1_HS  = 6     # 10
+    NC1_LS = 7     # 8
+    N2_HS  = 6     # 10
+    N2_LS  = 20111 # 20112
+    N31    = 2513  # 2514
+    N32    = 4596  # 4597
+
+    i2c_sequence = [
+        # PCA9548: select channel 7
+        [(0x74 << 1), 1 << 7],
+        # Si5324: configure
+        [(0x68 << 1), 0,   0b01010000],        # FREE_RUN=1
+        [(0x68 << 1), 1,   0b11100100],        # CK_PRIOR2=1 CK_PRIOR1=0
+        [(0x68 << 1), 2,   0b0010 | (4 << 4)], # BWSEL=4
+        [(0x68 << 1), 3,   0b0101 | 0x10],     # SQ_ICAL=1
+        [(0x68 << 1), 4,   0b10010010],        # AUTOSEL_REG=b10
+        [(0x68 << 1), 6,            0x07],     # SFOUT1_REG=b111
+        [(0x68 << 1), 25,  (N1_HS  << 5 ) & 0xff],
+        [(0x68 << 1), 31,  (NC1_LS >> 16) & 0xff],
+        [(0x68 << 1), 32,  (NC1_LS >> 8 ) & 0xff],
+        [(0x68 << 1), 33,  (NC1_LS)       & 0xff],
+        [(0x68 << 1), 40,  (N2_HS  << 5 ) & 0xff |
+                           (N2_LS  >> 16) & 0xff],
+        [(0x68 << 1), 41,  (N2_LS  >> 8 ) & 0xff],
+        [(0x68 << 1), 42,  (N2_LS)        & 0xff],
+        [(0x68 << 1), 43,  (N31    >> 16) & 0xff],
+        [(0x68 << 1), 44,  (N31    >> 8)  & 0xff],
+        [(0x68 << 1), 45,  (N31)          & 0xff],
+        [(0x68 << 1), 46,  (N32    >> 16) & 0xff],
+        [(0x68 << 1), 47,  (N32    >> 8)  & 0xff],
+        [(0x68 << 1), 48,  (N32)          & 0xff],
+        [(0x68 << 1), 137,          0x01],     # FASTLOCK=1
+        [(0x68 << 1), 136,          0x40],     # ICAL=1
+    ]
+
+    program = [
+        InstWrite(I2C_CONFIG_ADDR, int(sys_clk_freq/1e3)),
+    ]
+    for subseq in i2c_sequence:
+        program += [
+            InstWrite(I2C_XFER_ADDR, I2C_START),
+            InstWait(I2C_XFER_ADDR, I2C_IDLE),
+        ]
+        for octet in subseq:
+            program += [
+                InstWrite(I2C_XFER_ADDR, I2C_WRITE | octet),
+                InstWait(I2C_XFER_ADDR, I2C_IDLE),
+            ]
+        program += [
+            InstWrite(I2C_XFER_ADDR, I2C_STOP),
+            InstWait(I2C_XFER_ADDR, I2C_IDLE),
+        ]
+    program += [
+        InstEnd(),
+    ]
+    return program
+
+
+class Si5324ResetClock(Module):
+    def __init__(self, platform, sys_clk_freq):
+        self.si5324_not_ready = Signal(reset=1)
+
+        # minimum reset pulse 1us
+        reset_done = Signal()
+        si5324_rst_n = platform.request("si5324").rst_n
+        reset_val = int(sys_clk_freq*1.1e-6)
+        reset_ctr = Signal(max=reset_val+1, reset=reset_val)
+        self.sync += \
+            If(reset_ctr != 0,
+                reset_ctr.eq(reset_ctr - 1)
+            ).Else(
+                si5324_rst_n.eq(1),
+                reset_done.eq(1)
+            )
+        # 10ms after reset to microprocessor access ready
+        ready_val = int(sys_clk_freq*11e-3)
+        ready_ctr = Signal(max=ready_val+1, reset=ready_val)
+        self.sync += \
+            If(reset_done,
+                If(ready_ctr != 0,
+                    ready_ctr.eq(ready_ctr - 1)
+                ).Else(
+                    self.si5324_not_ready.eq(0)
+                )
+            )
+
+        si5324_clkin = platform.request("si5324_clkin")
+        self.specials += \
+            Instance("OBUFDS",
+                i_I=ClockSignal("rtio_rx"),
+                o_O=si5324_clkin.p, o_OB=si5324_clkin.n
+            )
+
+
+fmc_clock_io = [
+    ("ad9154_refclk", 0,
+        Subsignal("p", Pins("HPC:GBTCLK0_M2C_P")),
+        Subsignal("n", Pins("HPC:GBTCLK0_M2C_N")),
+    )
+]
+
+
+class Satellite(Module):
+    def __init__(self, cfg, medium, toolchain):
+        self.platform = platform = kc705.Platform(toolchain=toolchain)
+
+        rtio_channels = []
+        for i in range(8):
+            phy = ttl_simple.Output(platform.request("user_led", i))
+            self.submodules += phy
+            rtio_channels.append(rtio.Channel.from_phy(phy))
+        for sma in "user_sma_gpio_p", "user_sma_gpio_n":
+            phy = ttl_simple.Inout(platform.request(sma))
+            self.submodules += phy
+            rtio_channels.append(rtio.Channel.from_phy(phy))
+
+        sys_clock_pads = platform.request("clk156")
+        self.clock_domains.cd_sys = ClockDomain(reset_less=True)
+        self.specials += Instance("IBUFGDS",
+            i_I=sys_clock_pads.p, i_IB=sys_clock_pads.n,
+            o_O=self.cd_sys.clk)
+        sys_clk_freq = 156000000
+
+        i2c_master = I2CMaster(platform.request("i2c"))
+        sequencer = ResetInserter()(Sequencer(get_i2c_program(sys_clk_freq)))
+        si5324_reset_clock = Si5324ResetClock(platform, sys_clk_freq)
+        self.submodules += i2c_master, sequencer, si5324_reset_clock
+        self.comb += [
+            sequencer.bus.connect(i2c_master.bus),
+            sequencer.reset.eq(si5324_reset_clock.si5324_not_ready)
+        ]
+
+        if medium == "sfp":
+            self.comb += platform.request("sfp_tx_disable_n").eq(1)
+            tx_pads = platform.request("sfp_tx")
+            rx_pads = platform.request("sfp_rx")
+        elif medium == "sma":
+            tx_pads = platform.request("user_sma_mgt_tx")
+            rx_pads = platform.request("user_sma_mgt_rx")
+        else:
+            raise ValueError
+
+        if cfg == "simple_gbe":
+            # GTX_1000BASE_BX10 Ethernet compatible, 62.5MHz RTIO clock
+            # simple TTLs
+            self.submodules.transceiver = gtx_7series.GTX_1000BASE_BX10(
+                clock_pads=platform.request("sgmii_clock"),
+                tx_pads=tx_pads,
+                rx_pads=rx_pads,
+                sys_clk_freq=sys_clk_freq,
+                clock_div2=True)
+        elif cfg == "sawg_3g":
+            # 3Gb link, 150MHz RTIO clock
+            # with SAWG on local RTIO and AD9154-FMC-EBZ
+            platform.register_extension(fmc_clock_io)
+            self.submodules.transceiver = gtx_7series.GTX_3G(
+                clock_pads=platform.request("ad9154_refclk"),
+                tx_pads=tx_pads,
+                rx_pads=rx_pads,
+                sys_clk_freq=sys_clk_freq)
+        else:
+            raise ValueError
+        self.submodules.rx_synchronizer = gtx_7series.RXSynchronizer(
+            self.transceiver.rtio_clk_freq)
+        self.submodules.drtio = DRTIOSatellite(
+            self.transceiver, self.rx_synchronizer, rtio_channels)
+
+        rtio_clk_period = 1e9/self.transceiver.rtio_clk_freq
+        platform.add_period_constraint(self.transceiver.txoutclk, rtio_clk_period)
+        platform.add_period_constraint(self.transceiver.rxoutclk, rtio_clk_period)
+        platform.add_false_path_constraints(
+            sys_clock_pads,
+            self.transceiver.txoutclk, self.transceiver.rxoutclk)
+
+
+    def build(self, *args, **kwargs):
+        self.platform.build(self, *args, **kwargs)
+
+
+def main():
+    parser = argparse.ArgumentParser(description="KC705 DRTIO satellite")
+    parser.add_argument("--toolchain", default="vivado",
+                        help="FPGA toolchain to use: ise, vivado")
+    parser.add_argument("--output-dir", default="drtiosat_kc705",
+                        help="output directory for generated "
+                             "source files and binaries")
+    parser.add_argument("-c", "--config", default="simple_gbe",
+                        help="configuration: simple_gbe/sawg_3g "
+                             "(default: %(default)s)")
+    parser.add_argument("--medium", default="sfp",
+                        help="medium to use for transceiver link: sfp/sma "
+                             "(default: %(default)s)")
+    args = parser.parse_args()
+
+    top = Satellite(args.config, args.medium, args.toolchain)
+    top.build(build_dir=args.output_dir)
+
+if __name__ == "__main__":
+    main()

artiq/gateware/targets/pipistrello.py

@@ -217,12 +217,13 @@ trce -v 12 -fastpaths -tsi {build_name}.tsi -o {build_name}.twr {build_name}.ncd
         rtio_channels.append(rtio.LogChannel())
 
         # RTIO logic
-        self.submodules.rtio = rtio.RTIO(rtio_channels)
+        self.submodules.rtio_core = rtio.Core(rtio_channels)
+        self.submodules.rtio = rtio.KernelInitiator(self.rtio_core.cri)
         self.register_kernel_cpu_csrdevice("rtio")
         self.submodules.rtio_moninj = rtio.MonInj(rtio_channels)
         self.csr_devices.append("rtio_moninj")
         self.submodules.rtio_analyzer = rtio.Analyzer(
-            self.rtio, self.get_native_sdram_if())
+            self.rtio, self.rtio_core.cri.counter, self.get_native_sdram_if())
         self.csr_devices.append("rtio_analyzer")
 
 

artiq/protocols/analyzer.py

@@ -9,10 +9,10 @@ class MessageType(Enum):
 
 
 class ExceptionType(Enum):
-    reset_rising = 0b000000
+    reset = 0b000000
-    reset_falling = 0b000001
+    legacy_reset_falling = 0b000001
-    reset_phy_rising = 0b000010
+    reset_phy = 0b000010
-    reset_phy_falling = 0b000011
+    legacy_reset_phy_falling = 0b000011
 
     o_underflow_reset = 0b010000
     o_sequence_error_reset = 0b010001

artiq/runtime.rs/libksupport/lib.rs

@@ -322,10 +322,7 @@ pub unsafe fn main() {
     (mem::transmute::<usize, fn()>(library.lookup("__modinit__")))();
     send(&NowSave(NOW));
 
-    let typeinfo = library.lookup("typeinfo");
+    attribute_writeback(library.lookup("typeinfo") as *const ());
-    if typeinfo != 0 {
-        attribute_writeback(typeinfo as *const ())
-    }
 
     send(&RunFinished);
 

artiq/runtime.rs/libksupport/rtio.rs

@@ -13,8 +13,6 @@ const RTIO_I_STATUS_OVERFLOW:       u32 = 2;
 pub extern fn init() {
     unsafe {
         csr::rtio::reset_write(1);
-        csr::rtio::reset_write(0);
-        csr::rtio::reset_phy_write(0);
     }
 }
 

artiq/runtime.rs/src/drtio.rs

@@ -0,0 +1,70 @@
+use board::csr;
+use sched::{Waiter, Spawner};
+
+fn drtio_link_is_up() -> bool {
+    unsafe {
+        csr::drtio::link_status_read() == 1
+    }
+}
+
+fn drtio_sync_tsc() {
+    unsafe {
+        csr::drtio::set_time_write(1);
+        while csr::drtio::set_time_read() == 1 {}
+    }
+}
+
+fn drtio_init_channel(channel: u16) {
+    unsafe {
+        csr::drtio::chan_sel_override_write(channel);
+        csr::drtio::chan_sel_override_en_write(1);
+        
+        csr::drtio::o_reset_channel_status_write(1);
+        csr::drtio::o_get_fifo_space_write(1);
+        while csr::drtio::o_wait_read() == 1 {}
+        info!("FIFO space on channel {} is {}", channel, csr::drtio::o_dbg_fifo_space_read());
+
+        csr::drtio::chan_sel_override_en_write(0);
+    }
+}
+
+pub fn link_thread(waiter: Waiter, _spawner: Spawner) {
+    loop {
+        waiter.until(drtio_link_is_up).unwrap();
+        info!("link RX is up");
+
+        waiter.sleep(600).unwrap();
+        info!("wait for remote side done");
+
+        drtio_sync_tsc();
+        info!("TSC synced");
+        for channel in 0..16 {
+            drtio_init_channel(channel);
+        }
+        info!("link initialization completed");
+
+        waiter.until(|| !drtio_link_is_up()).unwrap();
+        info!("link is down");
+    }
+}
+
+fn drtio_packet_error_present() -> bool {
+    unsafe {
+        csr::drtio::packet_err_present_read() != 0
+    }
+}
+
+fn drtio_get_packet_error() -> u8 {
+    unsafe {
+        let err = csr::drtio::packet_err_code_read();
+        csr::drtio::packet_err_present_write(1);
+        err
+    }
+}
+
+pub fn error_thread(waiter: Waiter, _spawner: Spawner) {
+    loop {
+        waiter.until(drtio_packet_error_present).unwrap();
+        error!("DRTIO packet error {}", drtio_get_packet_error());
+    }
+}

artiq/runtime.rs/src/lib.rs

@@ -80,6 +80,8 @@ mod session;
 mod moninj;
 #[cfg(has_rtio_analyzer)]
 mod analyzer;
+#[cfg(has_drtio)]
+mod drtio;
 
 extern {
     fn network_init();
@@ -127,6 +129,11 @@ pub unsafe extern fn rust_main() {
         scheduler.spawner().spawn(4096, moninj::thread);
         #[cfg(has_rtio_analyzer)]
         scheduler.spawner().spawn(4096, analyzer::thread);
+        #[cfg(has_drtio)]
+        {
+            scheduler.spawner().spawn(4096, drtio::link_thread);
+            scheduler.spawner().spawn(4096, drtio::error_thread);
+        }
 
         loop {
             scheduler.run();

artiq/runtime.rs/src/logger.rs

@@ -1,12 +1,11 @@
 use core::{mem, ptr};
-use core::cell::{Cell, RefCell};
+use core::cell::RefCell;
 use log::{self, Log, LogLevel, LogMetadata, LogRecord, LogLevelFilter};
 use log_buffer::LogBuffer;
 use clock;
 
 pub struct BufferLogger {
-    buffer: RefCell<LogBuffer<&'static mut [u8]>>,
+    buffer: RefCell<LogBuffer<&'static mut [u8]>>
-    trace_to_uart: Cell<bool>
 }
 
 unsafe impl Sync for BufferLogger {}
@@ -16,8 +15,7 @@ static mut LOGGER: *const BufferLogger = ptr::null();
 impl BufferLogger {
     pub fn new(buffer: &'static mut [u8]) -> BufferLogger {
         BufferLogger {
-            buffer: RefCell::new(LogBuffer::new(buffer)),
+            buffer: RefCell::new(LogBuffer::new(buffer))
-            trace_to_uart: Cell::new(true)
         }
     }
 
@@ -50,14 +48,6 @@ impl BufferLogger {
     pub fn extract<R, F: FnOnce(&str) -> R>(&self, f: F) -> R {
         f(self.buffer.borrow_mut().extract())
     }
-
-    pub fn disable_trace_to_uart(&self) {
-        if self.trace_to_uart.get() {
-            trace!("disabling tracing to UART; all further trace messages \
-                    are sent to core log only");
-            self.trace_to_uart.set(false)
-        }
-    }
 }
 
 impl Log for BufferLogger {
@@ -71,10 +61,7 @@ impl Log for BufferLogger {
             writeln!(self.buffer.borrow_mut(),
                      "[{:12}us] {:>5}({}): {}",
                      clock::get_us(), record.level(), record.target(), record.args()).unwrap();
-
+            if record.level() <= LogLevel::Info {
-            // Printing to UART is really slow, so avoid doing that when we have an alternative
-            // route to retrieve the debug messages.
-            if self.trace_to_uart.get() || record.level() <= LogLevel::Info {
                 println!("[{:12}us] {:>5}({}): {}",
                          clock::get_us(), record.level(), record.target(), record.args());
             }

artiq/runtime.rs/src/session.rs

@@ -583,8 +583,6 @@ pub fn thread(waiter: Waiter, spawner: Spawner) {
         }
     }
 
-    BufferLogger::with_instance(|logger| logger.disable_trace_to_uart());
-
     let addr = SocketAddr::new(IP_ANY, 1381);
     let listener = TcpListener::bind(waiter, addr).expect("cannot bind socket");
     listener.set_keepalive(true);

artiq/test/gateware/drtio/test_aux_controller.py

@@ -0,0 +1,95 @@
+import unittest
+import random
+from types import SimpleNamespace
+
+from migen import *
+
+from artiq.gateware.drtio.link_layer import *
+from artiq.gateware.drtio.aux_controller import *
+
+
+class Loopback(Module):
+    def __init__(self, nwords):
+        ks = [Signal() for k in range(nwords)]
+        ds = [Signal(8) for d in range(nwords)]
+        encoder = SimpleNamespace(k=ks, d=ds)
+        decoders = [SimpleNamespace(k=k, d=d) for k, d in zip(ks, ds)]
+        self.submodules.tx = LinkLayerTX(encoder)
+        self.submodules.rx = LinkLayerRX(decoders)
+
+        self.ready = Signal()
+
+        self.tx_aux_frame = self.tx.aux_frame
+        self.tx_aux_data = self.tx.aux_data
+        self.tx_aux_ack = self.tx.aux_ack
+        self.tx_rt_frame = self.tx.rt_frame
+        self.tx_rt_data = self.tx.rt_data
+
+        self.rx_aux_stb = self.rx.aux_stb
+        self.rx_aux_frame = self.rx.aux_frame & self.ready
+        self.rx_aux_data = self.rx.aux_data
+        self.rx_rt_frame = self.rx.rt_frame & self.ready
+        self.rx_rt_data = self.rx.rt_data
+
+
+class TB(Module):
+    def __init__(self, nwords):
+        self.submodules.link_layer = Loopback(nwords)
+        self.submodules.aux_controller = ClockDomainsRenamer(
+            {"rtio": "sys", "rtio_rx": "sys"})(AuxController(self.link_layer))
+
+
+class TestAuxController(unittest.TestCase):
+    def test_aux_controller(self):
+        dut = TB(4)
+
+        def link_init():
+            for i in range(8):
+                yield
+            yield dut.link_layer.ready.eq(1)
+
+        def send_packet(packet):
+            for i, d in enumerate(packet):
+                yield from dut.aux_controller.bus.write(i, d)
+            yield from dut.aux_controller.transmitter.aux_tx_length.write(len(packet)*4)
+            yield from dut.aux_controller.transmitter.aux_tx.write(1)
+            yield
+            while (yield from dut.aux_controller.transmitter.aux_tx.read()):
+                yield
+
+        def receive_packet():
+            while not (yield from dut.aux_controller.receiver.aux_rx_present.read()):
+                yield
+            length = yield from dut.aux_controller.receiver.aux_rx_length.read()
+            r = []
+            for i in range(length//4):
+                r.append((yield from dut.aux_controller.bus.read(256+i)))
+            yield from dut.aux_controller.receiver.aux_rx_present.write(1)
+            return r
+
+        prng = random.Random(0)
+
+        def send_and_check_packet():
+            data = [prng.randrange(2**32-1) for _ in range(prng.randrange(1, 16))]
+            yield from send_packet(data)
+            received = yield from receive_packet()
+            self.assertEqual(data, received)
+
+        def sim():
+            yield from link_init()
+            for i in range(8):
+                yield from send_and_check_packet()
+
+        @passive
+        def rt_traffic():
+            while True:
+                while prng.randrange(4):
+                    yield
+                yield dut.link_layer.tx_rt_frame.eq(1)
+                yield
+                while prng.randrange(4):
+                    yield
+                yield dut.link_layer.tx_rt_frame.eq(0)
+                yield
+
+        run_simulation(dut, [sim(), rt_traffic()])

artiq/test/gateware/drtio/test_full_stack.py

@@ -0,0 +1,258 @@
+import unittest
+from types import SimpleNamespace
+import random
+
+from migen import *
+
+from artiq.gateware.drtio import *
+from artiq.gateware import rtio
+from artiq.gateware.rtio import rtlink
+from artiq.gateware.rtio.phy import ttl_simple
+from artiq.coredevice.exceptions import *
+
+
+class DummyTransceiverPair:
+    def __init__(self, nwords):
+        a2b_k = [Signal() for _ in range(nwords)]
+        a2b_d = [Signal(8) for _ in range(nwords)]
+        b2a_k = [Signal() for _ in range(nwords)]
+        b2a_d = [Signal(8) for _ in range(nwords)]
+
+        self.alice = SimpleNamespace(
+            encoder=SimpleNamespace(k=a2b_k, d=a2b_d),
+            decoders=[SimpleNamespace(k=k, d=d) for k, d in zip(b2a_k, b2a_d)],
+            rx_ready=1
+        )
+        self.bob = SimpleNamespace(
+            encoder=SimpleNamespace(k=b2a_k, d=b2a_d),
+            decoders=[SimpleNamespace(k=k, d=d) for k, d in zip(a2b_k, a2b_d)],
+            rx_ready=1
+        )
+
+
+class DummyRXSynchronizer:
+    def resync(self, signal):
+        return signal
+
+
+class LargeDataReceiver(Module):
+    def __init__(self, width):
+        self.rtlink = rtlink.Interface(rtlink.OInterface(width))
+        self.received_data = Signal(width)
+        self.sync.rio_phy += If(self.rtlink.o.stb,
+            self.received_data.eq(self.rtlink.o.data))
+
+
+class DUT(Module):
+    def __init__(self, nwords):
+        self.ttl0 = Signal()
+        self.ttl1 = Signal()
+        self.transceivers = DummyTransceiverPair(nwords)
+
+        self.submodules.master = DRTIOMaster(self.transceivers.alice)
+        self.submodules.master_ki = rtio.KernelInitiator(self.master.cri)
+
+        rx_synchronizer = DummyRXSynchronizer()
+        self.submodules.phy0 = ttl_simple.Output(self.ttl0)
+        self.submodules.phy1 = ttl_simple.Output(self.ttl1)
+        self.submodules.phy2 = LargeDataReceiver(512)
+        rtio_channels = [
+            rtio.Channel.from_phy(self.phy0, ofifo_depth=4),
+            rtio.Channel.from_phy(self.phy1, ofifo_depth=4),
+            rtio.Channel.from_phy(self.phy2, ofifo_depth=4),
+        ]
+        self.submodules.satellite = DRTIOSatellite(
+            self.transceivers.bob, rx_synchronizer, rtio_channels)
+        
+
+class TestFullStack(unittest.TestCase):
+    clocks = {"sys": 8, "rtio": 5, "rtio_rx": 5, "rio": 5, "rio_phy": 5}
+
+    def test_controller(self):
+        dut = DUT(2)
+        kcsrs = dut.master_ki
+        csrs = dut.master.rt_controller.csrs
+        mgr = dut.master.rt_manager
+
+        ttl_changes = []
+        correct_ttl_changes = [
+            # from test_pulses
+            (203, 0),
+            (208, 0),
+            (208, 1),
+            (214, 1),
+
+            # from test_fifo_space
+            (414, 0),
+            (454, 0),
+            (494, 0),
+            (534, 0),
+            (574, 0),
+            (614, 0)
+        ]
+
+        now = 0
+        def delay(dt):
+            nonlocal now
+            now += dt
+
+        def get_fifo_space(channel):
+            yield from csrs.chan_sel_override_en.write(1)
+            yield from csrs.chan_sel_override.write(channel)
+            yield from csrs.o_get_fifo_space.write(1)
+            yield
+            while (yield from csrs.o_wait.read()):
+                yield
+            r = (yield from csrs.o_dbg_fifo_space.read())
+            yield from csrs.chan_sel_override_en.write(0)
+            return r
+
+        def write(channel, data):
+            yield from kcsrs.chan_sel.write(channel)
+            yield from kcsrs.o_timestamp.write(now)
+            yield from kcsrs.o_data.write(data)
+            yield from kcsrs.o_we.write(1)
+            yield
+            status = 1
+            wlen = 0
+            while status:
+                status = yield from kcsrs.o_status.read()
+                if status & 2:
+                    yield from kcsrs.o_underflow_reset.write(1)
+                    raise RTIOUnderflow
+                if status & 4:
+                    yield from kcsrs.o_sequence_error_reset.write(1)
+                    raise RTIOSequenceError
+                yield
+                wlen += 1
+            return wlen
+
+        def test_init():
+            yield from get_fifo_space(0)
+            yield from get_fifo_space(1)
+
+        def test_underflow():
+            with self.assertRaises(RTIOUnderflow):
+                yield from write(0, 0)
+
+        def test_pulses():
+            delay(200*8)
+            yield from write(0, 1)
+            delay(5*8)
+            yield from write(0, 0)
+            yield from write(1, 1)
+            delay(6*8)
+            yield from write(1, 0)
+
+        def test_sequence_error():
+            delay(-200*8)
+            with self.assertRaises(RTIOSequenceError):
+                yield from write(0, 1)
+            delay(200*8)
+
+        def test_large_data():
+            correct_large_data = random.Random(0).randrange(2**512-1)
+            self.assertNotEqual((yield dut.phy2.received_data), correct_large_data)
+            delay(10*8)
+            yield from write(2, correct_large_data)
+            for i in range(40):
+                yield
+            self.assertEqual((yield dut.phy2.received_data), correct_large_data)
+
+        def test_fifo_space():
+            delay(200*8)
+            max_wlen = 0
+            for _ in range(3):
+                wlen = yield from write(0, 1)
+                max_wlen = max(max_wlen, wlen)
+                delay(40*8)
+                wlen = yield from write(0, 0)
+                max_wlen = max(max_wlen, wlen)
+                delay(40*8)
+            # check that some writes caused FIFO space requests
+            self.assertGreater(max_wlen, 5)
+
+        def test_fifo_emptied():
+            # wait for all TTL events to execute
+            while len(ttl_changes) < len(correct_ttl_changes):
+                yield
+            # check "last timestamp passed" FIFO empty condition
+            delay(1000*8)
+            wlen = yield from write(0, 1)
+            self.assertEqual(wlen, 2)
+
+        def test_tsc_error():
+            err_present = yield from mgr.packet_err_present.read()
+            self.assertEqual(err_present, 0)
+            yield from csrs.tsc_correction.write(100000000)
+            yield from csrs.set_time.write(1)
+            for i in range(15):
+               yield
+            delay(10000*8)
+            yield from write(0, 1)
+            for i in range(10):
+               yield
+            err_present = yield from mgr.packet_err_present.read()
+            err_code = yield from mgr.packet_err_code.read()
+            self.assertEqual(err_present, 1)
+            self.assertEqual(err_code, 3)
+            yield from mgr.packet_err_present.write(1)
+            yield
+            err_present = yield from mgr.packet_err_present.read()
+            self.assertEqual(err_present, 0)
+
+        def test():
+            while not (yield from dut.master.link_layer.link_status.read()):
+                yield
+
+            yield from test_init()
+            yield from test_underflow()
+            yield from test_pulses()
+            yield from test_sequence_error()
+            yield from test_fifo_space()
+            yield from test_large_data()
+            yield from test_fifo_emptied()
+            yield from test_tsc_error()
+
+        @passive
+        def check_ttls():
+            cycle = 0
+            old_ttls = [0, 0]
+            while True:
+                ttls = [(yield dut.ttl0), (yield dut.ttl1)]
+                for n, (old_ttl, ttl) in enumerate(zip(old_ttls, ttls)):
+                    if ttl != old_ttl:
+                        ttl_changes.append((cycle, n))
+                old_ttls = ttls
+                yield
+                cycle += 1
+
+        run_simulation(dut,
+            {"sys": test(), "rtio": check_ttls()}, self.clocks)
+        self.assertEqual(ttl_changes, correct_ttl_changes)
+
+    def test_echo(self):
+        dut = DUT(2)
+        csrs = dut.master.rt_controller.csrs
+        mgr = dut.master.rt_manager
+
+        def test():
+            while not (yield from dut.master.link_layer.link_status.read()):
+                yield
+
+            yield from mgr.update_packet_cnt.write(1)
+            yield
+            self.assertEqual((yield from mgr.packet_cnt_tx.read()), 0)
+            self.assertEqual((yield from mgr.packet_cnt_rx.read()), 0)
+
+            yield from mgr.request_echo.write(1)
+
+            for i in range(15):
+                yield
+
+            yield from mgr.update_packet_cnt.write(1)
+            yield
+            self.assertEqual((yield from mgr.packet_cnt_tx.read()), 1)
+            self.assertEqual((yield from mgr.packet_cnt_rx.read()), 1)
+
+        run_simulation(dut, test(), self.clocks)

artiq/test/gateware/drtio/test_link_layer.py

@@ -0,0 +1,129 @@
+import unittest
+from types import SimpleNamespace
+
+from migen import *
+
+from artiq.gateware.drtio.link_layer import *
+
+
+def process(seq):
+    dut = Scrambler(8)
+    rseq = []
+    def pump():
+        yield dut.i.eq(seq[0])
+        yield
+        for w in seq[1:]:
+            yield dut.i.eq(w)
+            yield
+            rseq.append((yield dut.o))
+        yield
+        rseq.append((yield dut.o))
+    run_simulation(dut, pump())
+    return rseq
+
+
+class Loopback(Module):
+    def __init__(self, nwords):
+        ks = [Signal() for k in range(nwords)]
+        ds = [Signal(8) for d in range(nwords)]
+        encoder = SimpleNamespace(k=ks, d=ds)
+        decoders = [SimpleNamespace(k=k, d=d) for k, d in zip(ks, ds)]
+        self.submodules.tx = LinkLayerTX(encoder)
+        self.submodules.rx = LinkLayerRX(decoders)
+
+
+class TestLinkLayer(unittest.TestCase):
+    def test_packets(self):
+        dut = Loopback(4)
+
+        def scrambler_sync():
+            for i in range(8):
+                yield
+
+        rt_packets = [
+            [0x12459970, 0x9938cdef, 0x12340000],
+            [0xabcdef00, 0x12345678],
+            [0xeeeeeeee, 0xffffffff, 0x01020304, 0x11223344],
+            [0x88277475, 0x19883332, 0x19837662, 0x81726668, 0x81876261]
+        ]
+        def transmit_rt_packets():
+            yield from scrambler_sync()
+
+            for packet in rt_packets:
+                yield dut.tx.rt_frame.eq(1)
+                for data in packet:
+                    yield dut.tx.rt_data.eq(data)
+                    yield
+                yield dut.tx.rt_frame.eq(0)
+                yield
+            # flush
+            for i in range(20):
+                yield
+
+        rx_rt_packets = []
+        @passive
+        def receive_rt_packets():
+            yield from scrambler_sync()
+
+            previous_frame = 0
+            while True:
+                frame = yield dut.rx.rt_frame
+                if frame and not previous_frame:
+                    packet = []
+                    rx_rt_packets.append(packet)
+                previous_frame = frame
+                if frame:
+                    packet.append((yield dut.rx.rt_data))
+                yield
+
+        aux_packets = [
+            [0x12, 0x34],
+            [0x44, 0x11, 0x98, 0x78],
+            [0xbb, 0xaa, 0xdd, 0xcc, 0x00, 0xff, 0xee]
+        ]
+        def transmit_aux_packets():
+            yield from scrambler_sync()
+
+            for packet in aux_packets:
+                yield dut.tx.aux_frame.eq(1)
+                for data in packet:
+                    yield dut.tx.aux_data.eq(data)
+                    yield
+                    while not (yield dut.tx.aux_ack):
+                        yield
+                yield dut.tx.aux_frame.eq(0)
+                yield
+                while not (yield dut.tx.aux_ack):
+                    yield
+            # flush
+            for i in range(20):
+                yield
+
+        rx_aux_packets = []
+        @passive
+        def receive_aux_packets():
+            yield from scrambler_sync()
+
+            previous_frame = 0
+            while True:
+                if (yield dut.rx.aux_stb):
+                    frame = yield dut.rx.aux_frame
+                    if frame and not previous_frame:
+                        packet = []
+                        rx_aux_packets.append(packet)
+                    previous_frame = frame
+                    if frame:
+                        packet.append((yield dut.rx.aux_data))
+                yield
+
+        run_simulation(dut, [transmit_rt_packets(), receive_rt_packets(),
+                             transmit_aux_packets(), receive_aux_packets()])
+
+        # print("RT:")
+        # for packet in rx_rt_packets:
+        #     print(" ".join("{:08x}".format(x) for x in packet))
+        # print("AUX:")
+        # for packet in rx_aux_packets:
+        #     print(" ".join("{:02x}".format(x) for x in packet))
+        self.assertEqual(rt_packets, rx_rt_packets)
+        self.assertEqual(aux_packets, rx_aux_packets)

artiq/test/gateware/drtio/test_rt_packets.py

@@ -0,0 +1,212 @@
+import unittest
+from types import SimpleNamespace
+import random
+
+from migen import *
+
+from artiq.gateware.drtio.rt_packets import *
+from artiq.gateware.drtio.rt_packets import (_CrossDomainRequest,
+                                             _CrossDomainNotification)
+
+
+class PacketInterface:
+    def __init__(self, direction, ws):
+        if direction == "m2s":
+            self.plm = get_m2s_layouts(ws)
+        elif direction == "s2m":
+            self.plm = get_s2m_layouts(ws)
+        else:
+            raise ValueError
+        self.frame = Signal()
+        self.data = Signal(ws)
+
+    def send(self, ty, **kwargs):
+        idx = 8
+        value = self.plm.types[ty]
+        for field_name, field_size in self.plm.layouts[ty][1:]:
+            try:
+                fvalue = kwargs[field_name]
+                del kwargs[field_name]
+            except KeyError:
+                fvalue = 0
+            value = value | (fvalue << idx)
+            idx += field_size
+        if kwargs:
+            raise ValueError
+
+        ws = len(self.data)
+        yield self.frame.eq(1)
+        for i in range(idx//ws):
+            yield self.data.eq(value)
+            value >>= ws
+            yield
+        yield self.frame.eq(0)
+        yield
+
+    @passive
+    def receive(self, callback):
+        previous_frame = 0
+        frame_words = []
+        while True:
+            frame = yield self.frame
+            if frame:
+                frame_words.append((yield self.data))
+            if previous_frame and not frame:
+                packet_type = self.plm.type_names[frame_words[0] & 0xff]
+                packet_nwords = layout_len(self.plm.layouts[packet_type]) \
+                                //len(self.data)
+                packet, trailer = frame_words[:packet_nwords], \
+                                  frame_words[packet_nwords:]
+
+                n = 0
+                packet_int = 0
+                for w in packet:
+                    packet_int |= (w << n)
+                    n += len(self.data)
+
+                field_dict = dict()
+                idx = 0
+                for field_name, field_size in self.plm.layouts[packet_type]:
+                    v = (packet_int >> idx) & (2**field_size - 1)
+                    field_dict[field_name] = v
+                    idx += field_size
+
+                callback(packet_type, field_dict, trailer)
+
+                frame_words = []
+            previous_frame = frame
+            yield
+
+
+class TestSatellite(unittest.TestCase):
+    def create_dut(self, nwords):
+        pt = PacketInterface("m2s", nwords*8)
+        pr = PacketInterface("s2m", nwords*8)
+        dut = RTPacketSatellite(SimpleNamespace(
+            rx_rt_frame=pt.frame, rx_rt_data=pt.data,
+            tx_rt_frame=pr.frame, tx_rt_data=pr.data))
+        return pt, pr, dut
+
+    def test_echo(self):
+        for nwords in range(1, 8):
+            pt, pr, dut = self.create_dut(nwords)
+            completed = False
+            def send():
+                yield from pt.send("echo_request")
+                while not completed:
+                    yield
+            def receive(packet_type, field_dict, trailer):
+                nonlocal completed
+                self.assertEqual(packet_type, "echo_reply")
+                self.assertEqual(trailer, [])
+                completed = True
+            run_simulation(dut, [send(), pr.receive(receive)])
+
+    def test_set_time(self):
+        for nwords in range(1, 8):
+            pt, _, dut = self.create_dut(nwords)
+            tx_times = [0x12345678aabbccdd, 0x0102030405060708,
+                        0xaabbccddeeff1122]
+            def send():
+                for t in tx_times:
+                    yield from pt.send("set_time", timestamp=t)
+                # flush
+                for i in range(10):
+                    yield
+            rx_times = []
+            @passive
+            def receive():
+                while True:
+                    if (yield dut.tsc_load):
+                        rx_times.append((yield dut.tsc_value))
+                    yield
+            run_simulation(dut, [send(), receive()])
+            self.assertEqual(tx_times, rx_times)
+
+
+class TestCDC(unittest.TestCase):
+    def test_cross_domain_request(self):
+        prng = random.Random(1)
+        for sys_freq in 3, 6, 11:
+            for srv_freq in 3, 6, 11:
+                req_stb = Signal()
+                req_ack = Signal()
+                req_data = Signal(8)
+                srv_stb = Signal()
+                srv_ack = Signal()
+                srv_data = Signal(8)
+                test_seq = [93, 92, 19, 39, 91, 30, 12, 91, 38, 42]
+                received_seq = []
+
+                def requester():
+                    for data in test_seq:
+                        yield req_data.eq(data)
+                        yield req_stb.eq(1)
+                        yield
+                        while not (yield req_ack):
+                            yield
+                        yield req_stb.eq(0)
+                        for j in range(prng.randrange(0, 10)):
+                            yield
+
+                def server():
+                    for i in range(len(test_seq)):
+                        while not (yield srv_stb):
+                            yield
+                        received_seq.append((yield srv_data))
+                        for j in range(prng.randrange(0, 10)):
+                            yield
+                        yield srv_ack.eq(1)
+                        yield
+                        yield srv_ack.eq(0)
+                        yield
+
+                dut = _CrossDomainRequest("srv",
+                         req_stb, req_ack, req_data,
+                         srv_stb, srv_ack, srv_data)
+                run_simulation(dut,
+                    {"sys": requester(), "srv": server()},
+                    {"sys": sys_freq, "srv": srv_freq})
+                self.assertEqual(test_seq, received_seq)
+
+    def test_cross_domain_notification(self):
+        prng = random.Random(1)
+
+        emi_stb = Signal()
+        emi_data = Signal(8)
+        rec_stb = Signal()
+        rec_ack = Signal()
+        rec_data = Signal(8)
+
+        test_seq = [23, 12, 8, 3, 28]
+        received_seq = []
+
+        def emitter():
+            for data in test_seq:
+                yield emi_stb.eq(1)
+                yield emi_data.eq(data)
+                yield
+                yield emi_stb.eq(0)
+                yield
+                for j in range(prng.randrange(0, 3)):
+                    yield
+
+        def receiver():
+            for i in range(len(test_seq)):
+                while not (yield rec_stb):
+                    yield
+                received_seq.append((yield rec_data))
+                yield rec_ack.eq(1)
+                yield
+                yield rec_ack.eq(0)
+                yield
+                for j in range(prng.randrange(0, 3)):
+                    yield
+
+        dut = _CrossDomainNotification("emi",
+            emi_stb, emi_data,
+            rec_stb, rec_ack, rec_data)
+        run_simulation(dut,
+            {"emi": emitter(), "sys": receiver()},
+            {"emi": 13, "sys": 3})
+        self.assertEqual(test_seq, received_seq)