rtio: refactor TSC to allow sharing between cores

artiq/gateware/drtio/core.py

@@ -5,7 +5,7 @@ from migen.genlib.resetsync import AsyncResetSynchronizer
 from migen.genlib.cdc import PulseSynchronizer
 from misoc.interconnect.csr import *
 
-from artiq.gateware.rtio import cri
+from artiq.gateware.rtio import cri, rtlink
 from artiq.gateware.rtio.sed.core import *
 from artiq.gateware.rtio.input_collector import *
 from artiq.gateware.drtio import (link_layer, aux_controller,
@@ -47,32 +47,33 @@ async_errors_layout = [
 
 
 class SyncRTIO(Module):
-    def __init__(self, channels, fine_ts_width=3, lane_count=8, fifo_depth=128):
+    def __init__(self, tsc, channels, lane_count=8, fifo_depth=128):
         self.cri = cri.Interface()
         self.async_errors = Record(async_errors_layout)
-        self.coarse_ts = Signal(64 - fine_ts_width)
 
-        self.comb += self.cri.counter.eq(self.coarse_ts << fine_ts_width)
+        chan_fine_ts_width = max(max(rtlink.get_fine_ts_width(channel.interface.o)
+                                     for channel in channels),
+                                 max(rtlink.get_fine_ts_width(channel.interface.i)
+                                     for channel in channels))
+        assert tsc.glbl_fine_ts_width >= chan_fine_ts_width
 
         self.submodules.outputs = ClockDomainsRenamer("rio")(
-            SED(channels, fine_ts_width, "sync",
+            SED(channels, tsc.glbl_fine_ts_width, "sync",
                 lane_count=lane_count, fifo_depth=fifo_depth,
                 enable_spread=False, report_buffer_space=True,
                 interface=self.cri))
-        self.comb += self.outputs.coarse_timestamp.eq(self.coarse_ts)
+        self.comb += self.outputs.coarse_timestamp.eq(tsc.coarse_ts)
-        self.sync.rtio += self.outputs.minimum_coarse_timestamp.eq(self.coarse_ts + 16)
+        self.sync.rtio += self.outputs.minimum_coarse_timestamp.eq(tsc.coarse_ts + 16)
 
         self.submodules.inputs = ClockDomainsRenamer("rio")(
-            InputCollector(channels, fine_ts_width, "sync",
+            InputCollector(tsc, channels, "sync", interface=self.cri))
-                           interface=self.cri))
-        self.comb += self.inputs.coarse_timestamp.eq(self.coarse_ts)
 
         for attr, _ in async_errors_layout:
             self.comb += getattr(self.async_errors, attr).eq(getattr(self.outputs, attr))
 
 
 class DRTIOSatellite(Module):
-    def __init__(self, chanif, rx_synchronizer=None, fine_ts_width=3):
+    def __init__(self, tsc, chanif, rx_synchronizer=None):
         self.reset = CSRStorage(reset=1)
         self.reset_phy = CSRStorage(reset=1)
         self.tsc_loaded = CSR()
@@ -127,13 +128,10 @@ class DRTIOSatellite(Module):
             rt_packet_satellite.RTPacketSatellite(link_layer_sync, interface=self.cri))
         self.comb += self.rt_packet.reset.eq(self.cd_rio.rst)
 
-        self.coarse_ts = Signal(64 - fine_ts_width)
+        self.comb += [
-        self.sync.rtio += \
+            tsc.load.eq(self.rt_packet.tsc_load),
-            If(self.rt_packet.tsc_load,
+            tsc.load_value.eq(self.rt_packet.tsc_load_value)
-                self.coarse_ts.eq(self.rt_packet.tsc_load_value)
+        ]
-            ).Else(
-                self.coarse_ts.eq(self.coarse_ts + 1)
-            )
 
         ps_tsc_load = PulseSynchronizer("rtio", "sys")
         self.submodules += ps_tsc_load
@@ -144,7 +142,7 @@ class DRTIOSatellite(Module):
         ]
 
         self.submodules.rt_errors = rt_errors_satellite.RTErrorsSatellite(
-            self.rt_packet, self.cri, self.async_errors)
+            self.rt_packet, tsc, self.cri, self.async_errors)
 
         self.submodules.aux_controller = aux_controller.AuxController(
             self.link_layer)
@@ -156,7 +154,7 @@ class DRTIOSatellite(Module):
 
 
 class DRTIOMaster(Module):
-    def __init__(self, chanif, fine_ts_width=3):
+    def __init__(self, tsc, chanif):
         self.submodules.link_layer = link_layer.LinkLayer(
             chanif.encoder, chanif.decoders)
         self.comb += self.link_layer.rx_ready.eq(chanif.rx_ready)
@@ -164,7 +162,7 @@ class DRTIOMaster(Module):
         self.submodules.link_stats = link_layer.LinkLayerStats(self.link_layer, "rtio_rx")
         self.submodules.rt_packet = rt_packet_master.RTPacketMaster(self.link_layer)
         self.submodules.rt_controller = rt_controller_master.RTController(
-            self.rt_packet, fine_ts_width)
+            tsc, self.rt_packet)
         self.submodules.rt_manager = rt_controller_master.RTManager(self.rt_packet)
 
         self.submodules.aux_controller = aux_controller.AuxController(

artiq/gateware/drtio/rt_controller_master.py

@@ -7,7 +7,6 @@ from migen.genlib.resetsync import AsyncResetSynchronizer
 
 from misoc.interconnect.csr import *
 
-from artiq.gateware.rtio.cdc import GrayCodeTransfer
 from artiq.gateware.rtio import cri
 
 
@@ -26,26 +25,8 @@ class _CSRs(AutoCSR):
         self.o_wait = CSRStatus()
 
 
-class RTIOCounter(Module):
-    def __init__(self, width):
-        self.width = width
-        # Timestamp counter in RTIO domain
-        self.value_rtio = Signal(width)
-        # Timestamp counter resynchronized to sys domain
-        # Lags behind value_rtio, monotonic and glitch-free
-        self.value_sys = Signal(width)
-
-        # # #
-
-        # note: counter is in rtio domain and never affected by the reset CSRs
-        self.sync.rtio += self.value_rtio.eq(self.value_rtio + 1)
-        gt = GrayCodeTransfer(width)
-        self.submodules += gt
-        self.comb += gt.i.eq(self.value_rtio), self.value_sys.eq(gt.o)
-
-
 class RTController(Module):
-    def __init__(self, rt_packet, fine_ts_width):
+    def __init__(self, tsc, rt_packet):
         self.csrs = _CSRs()
         self.cri = cri.Interface()
 
@@ -80,11 +61,9 @@ class RTController(Module):
         self.comb += self.csrs.protocol_error.w.eq(
             Cat(err_unknown_packet_type, err_packet_truncated, err_buffer_space_timeout))
 
-        # master RTIO counter and counter synchronization
+        # TSC synchronization
-        self.submodules.counter = RTIOCounter(64-fine_ts_width)
         self.comb += [
-            self.cri.counter.eq(self.counter.value_sys << fine_ts_width),
+            rt_packet.tsc_value.eq(tsc.coarse_ts),
-            rt_packet.tsc_value.eq(self.counter.value_rtio),
             self.csrs.set_time.w.eq(rt_packet.set_time_stb)
         ]
         self.sync += [
@@ -130,8 +109,8 @@ class RTController(Module):
         self.submodules += timeout_counter
 
         cond_underflow = Signal()
-        self.comb += cond_underflow.eq((self.cri.timestamp[fine_ts_width:]
+        self.comb += cond_underflow.eq((self.cri.timestamp[tsc.glbl_fine_ts_width:]
-                           - self.csrs.underflow_margin.storage[fine_ts_width:]) < self.counter.value_sys)
+                           - self.csrs.underflow_margin.storage[tsc.glbl_fine_ts_width:]) < tsc.coarse_ts_sys)
 
         buffer_space = Signal(16)
 

artiq/gateware/drtio/rt_errors_satellite.py

@@ -7,7 +7,7 @@ from artiq.gateware.rtio.cdc import BlindTransfer
 
 
 class RTErrorsSatellite(Module, AutoCSR):
-    def __init__(self, rt_packet, cri, async_errors):
+    def __init__(self, rt_packet, tsc, cri, async_errors):
         self.protocol_error = CSR(4)
         self.underflow_channel = CSRStatus(16)
         self.underflow_timestamp_event = CSRStatus(64)
@@ -60,7 +60,7 @@ class RTErrorsSatellite(Module, AutoCSR):
             overflow.eq(cri.o_status[0]),
             underflow_error_cri.eq(Cat(cri.chan_sel[:16],
                                        cri.timestamp,
-                                       cri.counter)),
+                                       tsc.full_ts_cri)),
             Cat(self.underflow_channel.status,
                 self.underflow_timestamp_event.status,
                 self.underflow_timestamp_counter.status).eq(underflow_error_csr)

artiq/gateware/rtio/__init__.py

@@ -1,3 +1,4 @@
+from artiq.gateware.rtio.tsc import TSC
 from artiq.gateware.rtio.cri import KernelInitiator, CRIInterconnectShared
 from artiq.gateware.rtio.channel import Channel, LogChannel
 from artiq.gateware.rtio.core import Core

artiq/gateware/rtio/core.py

@@ -14,8 +14,7 @@ from artiq.gateware.rtio.input_collector import *
 
 
 class Core(Module, AutoCSR):
-    def __init__(self, channels, lane_count=8, fifo_depth=128,
+    def __init__(self, tsc, channels, lane_count=8, fifo_depth=128):
-                 glbl_fine_ts_width=None):
         self.cri = cri.Interface()
         self.reset = CSR()
         self.reset_phy = CSR()
@@ -61,36 +60,23 @@ class Core(Module, AutoCSR):
                                      for channel in channels),
                                  max(rtlink.get_fine_ts_width(channel.interface.i)
                                      for channel in channels))
-        if glbl_fine_ts_width is None:
+        assert tsc.glbl_fine_ts_width >= chan_fine_ts_width
-            glbl_fine_ts_width = chan_fine_ts_width
-        assert glbl_fine_ts_width >= chan_fine_ts_width
-
-        coarse_ts = Signal(64-glbl_fine_ts_width)
-        self.sync.rtio += coarse_ts.eq(coarse_ts + 1)
-        coarse_ts_cdc = GrayCodeTransfer(len(coarse_ts))  # from rtio to sys
-        self.submodules += coarse_ts_cdc
-        self.comb += [
-            coarse_ts_cdc.i.eq(coarse_ts),
-            self.cri.counter.eq(coarse_ts_cdc.o << glbl_fine_ts_width)
-        ]
-        self.coarse_ts = coarse_ts
 
         # Outputs/Inputs
         quash_channels = [n for n, c in enumerate(channels) if isinstance(c, LogChannel)]
 
-        outputs = SED(channels, glbl_fine_ts_width, "async",
+        outputs = SED(channels, tsc.glbl_fine_ts_width, "async",
             quash_channels=quash_channels,
             lane_count=lane_count, fifo_depth=fifo_depth,
             interface=self.cri)
         self.submodules += outputs
-        self.comb += outputs.coarse_timestamp.eq(coarse_ts)
+        self.comb += outputs.coarse_timestamp.eq(tsc.coarse_ts)
-        self.sync += outputs.minimum_coarse_timestamp.eq(coarse_ts_cdc.o + 16)
+        self.sync += outputs.minimum_coarse_timestamp.eq(tsc.coarse_ts_sys + 16)
 
-        inputs = InputCollector(channels, glbl_fine_ts_width, "async",
+        inputs = InputCollector(tsc, channels, "async",
             quash_channels=quash_channels,
             interface=self.cri)
         self.submodules += inputs
-        self.comb += inputs.coarse_timestamp.eq(coarse_ts)
 
         # Asychronous output errors
         o_collision_sync = BlindTransfer(data_width=16)

artiq/gateware/rtio/cri.py

@@ -49,11 +49,6 @@ layout = [
     # <3:link error>
     # <0> and <1> are mutually exclusive. <1> has higher priority.
     ("i_status", 4, DIR_S_TO_M),
-
-    # value of the timestamp counter transferred into the CRI clock domain.
-    # monotonic, may lag behind the counter in the IO clock domain, but
-    # not be ahead of it.
-    ("counter", 64, DIR_S_TO_M)
 ]
 
 
@@ -63,8 +58,10 @@ class Interface(Record):
 
 
 class KernelInitiator(Module, AutoCSR):
-    def __init__(self, cri=None):
+    def __init__(self, tsc, cri=None):
         self.chan_sel = CSRStorage(24)
+        # monotonic, may lag behind the counter in the IO clock domain, but
+        # not be ahead of it.
         self.timestamp = CSRStorage(64)
 
         # Writing timestamp clears o_data. This implements automatic
@@ -109,7 +106,7 @@ class KernelInitiator(Module, AutoCSR):
             self.o_data.dat_w.eq(0),
             self.o_data.we.eq(self.timestamp.re),
         ]
-        self.sync += If(self.counter_update.re, self.counter.status.eq(self.cri.counter))
+        self.sync += If(self.counter_update.re, self.counter.status.eq(tsc.full_ts_cri))
 
 
 class CRIDecoder(Module):

artiq/gateware/rtio/input_collector.py

@@ -24,11 +24,10 @@ def get_channel_layout(coarse_ts_width, interface):
 
 
 class InputCollector(Module):
-    def __init__(self, channels, glbl_fine_ts_width, mode, quash_channels=[], interface=None):
+    def __init__(self, tsc, channels, mode, quash_channels=[], interface=None):
         if interface is None:
             interface = cri.Interface()
         self.cri = interface
-        self.coarse_timestamp = Signal(64 - glbl_fine_ts_width)
 
         # # #
 
@@ -55,7 +54,7 @@ class InputCollector(Module):
                 continue
 
             # FIFO
-            layout = get_channel_layout(len(self.coarse_timestamp), iif)
+            layout = get_channel_layout(len(tsc.coarse_ts), iif)
             fifo = fifo_factory(layout_len(layout), channel.ififo_depth)
             self.submodules += fifo
             fifo_in = Record(layout)
@@ -67,10 +66,10 @@ class InputCollector(Module):
 
             # FIFO write
             if iif.delay:
-                counter_rtio = Signal.like(self.coarse_timestamp, reset_less=True)
+                counter_rtio = Signal.like(tsc.coarse_ts, reset_less=True)
-                sync_io += counter_rtio.eq(self.coarse_timestamp - (iif.delay + 1))
+                sync_io += counter_rtio.eq(tsc.coarse_ts - (iif.delay + 1))
             else:
-                counter_rtio = self.coarse_timestamp
+                counter_rtio = tsc.coarse_ts
             if hasattr(fifo_in, "data"):
                 self.comb += fifo_in.data.eq(iif.data)
             if hasattr(fifo_in, "timestamp"):
@@ -130,7 +129,7 @@ class InputCollector(Module):
                 self.cri.i_data.eq(Array(i_datas)[sel]),
                 self.cri.i_timestamp.eq(Array(i_timestamps)[sel]),
             ),
-            If((self.cri.counter >= input_timeout) | (i_status_raw != 0),
+            If((tsc.full_ts_cri >= input_timeout) | (i_status_raw != 0),
                 If(input_pending, i_ack.eq(1)),
                 input_pending.eq(0)
             ),

artiq/gateware/test/drtio/test_full_stack.py

@@ -52,9 +52,11 @@ class DUT(Module):
         self.ttl1 = Signal()
         self.transceivers = DummyTransceiverPair(nwords)
 
-        self.submodules.master = DRTIOMaster(self.transceivers.alice,
+        self.submodules.tsc_master = rtio.TSC("async")
-                                             fine_ts_width=0)
+        self.submodules.master = DRTIOMaster(self.tsc_master,
-        self.submodules.master_ki = rtio.KernelInitiator(self.master.cri)
+                                             self.transceivers.alice)
+        self.submodules.master_ki = rtio.KernelInitiator(self.tsc_master,
+            self.master.cri)
         self.master.rt_controller.csrs.link_up.storage.reset = 1
 
         rx_synchronizer = DummyRXSynchronizer()
@@ -66,16 +68,16 @@ class DUT(Module):
             rtio.Channel.from_phy(self.phy1),
             rtio.Channel.from_phy(self.phy2),
         ]
+        self.submodules.tsc_satellite = rtio.TSC("sync")
         self.submodules.satellite = DRTIOSatellite(
-            self.transceivers.bob, rx_synchronizer, fine_ts_width=0)
+            self.tsc_satellite, self.transceivers.bob, rx_synchronizer)
         self.satellite.reset.storage.reset = 0
         self.satellite.reset.storage_full.reset = 0
         self.satellite.reset_phy.storage.reset = 0
         self.satellite.reset_phy.storage_full.reset = 0
         self.submodules.satellite_rtio = SyncRTIO(
-            rtio_channels, fine_ts_width=0, lane_count=4, fifo_depth=8)
+            self.tsc_satellite, rtio_channels, lane_count=4, fifo_depth=8)
         self.comb += [
-        self.satellite_rtio.coarse_ts.eq(self.satellite.coarse_ts),
             self.satellite.cri.connect(self.satellite_rtio.cri),
             self.satellite.async_errors.eq(self.satellite_rtio.async_errors),
         ]
@@ -106,7 +108,7 @@ class OutputsTestbench:
 
     def sync(self):
         t = self.now + 15
-        while (yield self.dut.master.cri.counter) < t:
+        while (yield self.dut.tsc_master.full_ts_cri) < t:
             yield
 
     def write(self, channel, data):

artiq/gateware/test/rtio/test_dma.py

@@ -127,7 +127,8 @@ class FullStackTB(Module):
         self.submodules.memory = wishbone.SRAM(
             256, init=sequence, bus=bus)
         self.submodules.dut = dma.DMA(bus)
-        self.submodules.rtio = rtio.Core(rtio_channels)
+        self.submodules.tsc = rtio.TSC("async")
+        self.submodules.rtio = rtio.Core(self.tsc, rtio_channels)
         self.comb += self.dut.cri.connect(self.rtio.cri)
 
 

artiq/gateware/test/rtio/test_input_collector.py

@@ -38,9 +38,8 @@ class DUT(Module):
             rtio.Channel.from_phy(self.phy0, ififo_depth=4),
             rtio.Channel.from_phy(self.phy1, ififo_depth=4)
         ]
-        self.submodules.input_collector = InputCollector(rtio_channels, 0, "sync")
+        self.submodules.tsc = ClockDomainsRenamer({"rtio": "sys"})(rtio.TSC("sync"))
-        self.sync += self.input_collector.coarse_timestamp.eq(self.input_collector.coarse_timestamp + 1)
+        self.submodules.input_collector = InputCollector(self.tsc, rtio_channels, "sync")
-        self.comb += self.input_collector.cri.counter.eq(self.input_collector.coarse_timestamp)
 
     @property
     def cri(self):