From fa83ad0d9ce77174a14205269bbf01474901e9b8 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Mon, 26 Sep 2016 12:53:10 +0800
Subject: [PATCH 001/134] drtio: add TX link layer

---
 artiq/gateware/drtio/__init__.py   |  0
 artiq/gateware/drtio/link_layer.py | 97 ++++++++++++++++++++++++++++++
 2 files changed, 97 insertions(+)
 create mode 100644 artiq/gateware/drtio/__init__.py
 create mode 100644 artiq/gateware/drtio/link_layer.py

diff --git a/artiq/gateware/drtio/__init__.py b/artiq/gateware/drtio/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/artiq/gateware/drtio/link_layer.py b/artiq/gateware/drtio/link_layer.py
new file mode 100644
index 000000000..76acbed4f
--- /dev/null
+++ b/artiq/gateware/drtio/link_layer.py
@@ -0,0 +1,97 @@
+from migen import *
+
+
+def K(x, y):
+    return (y << 5) | x
+
+
+class LinkLayerTX(Module):
+    def __init__(self, encoder):
+        nwords = len(encoder.k)
+        # nwords must be a power of 2
+        assert nwords & (nwords - 1) == 0
+
+        self.link_init = Signal()
+
+        self.aux_frame = Signal()
+        self.aux_data = Signal(2*nwords)
+        self.aux_ack = Signal()
+
+        self.rt_frame = Signal()
+        self.rt_data = Signal(8*nwords)
+
+        self.transceiver_data = Signal(10*nwords)
+
+        # # #
+
+        # Idle and auxiliary traffic use special characters excluding
+        # K.28.7 and K.29.7 in order to easily separate the link initialization
+        # phase (K.28.7 is additionally excluded as we cannot guarantee its
+        # non-repetition here).
+        # A set of 8 special characters is chosen using a 3-bit control word.
+        # This control word is scrambled to reduce EMI. The control words have
+        # the following meanings:
+        #   100 idle/auxiliary framing
+        #   0AB 2 bits of auxiliary data
+        aux_scrambler = Scrambler(3*nwords)
+        self.submodules += aux_scrambler
+        aux_data_ctl = []
+        for i in range(nwords):
+            aux_data_ctl.append(self.aux_data[i*2:i*2+1])
+            aux_data_ctl.append(0)
+        self.comb += [
+            If(self.aux_frame,
+                aux_scrambler.i.eq(Cat(*aux_data_ctl))
+            ).Else(
+                aux_scrambler.i.eq(Replicate(0b100, nwords))
+            ),
+            aux_scrambler.ce.eq(~self.rt_frame),
+            self.aux_ack.eq(~self.rt_frame)
+        ]
+        for i in range(nwords):
+            scrambled_ctl = scrambler.o[i*3:i*3+3]
+            self.sync += [
+                encoder.k[i].eq(1),
+                If(scrambled_ctl == 7,
+                    encoder.d[i].eq(K(23, 7))
+                ).Else(
+                    encoder.d[i].eq(K(28, scrambled_ctl))
+                )
+            ]
+
+        # Real-time traffic uses data characters and is framed by the special
+        # characters of auxiliary traffic. RT traffic is also scrambled.
+        rt_scrambler = Scrambler(8*nwords)
+        self.submodules += rt_scrambler
+        self.comb += rt_scrambler.i.eq(self.rt_data)
+        rt_frame_r = Signal()
+        self.sync += [
+            rt_frame_r.eq(self.rt_frame),
+            If(rt_frame_r,
+                [k.eq(0) for k in encoder.k],
+                [d.eq(self.rt_data[i*8:i*8+8]) for i, d in enumerate(encoder.d)]
+            )
+        ]
+
+        # During link init, send a series of 1*K.28.7 (comma) + 31*K.29.7
+        # The receiving end configures its transceiver to also place the comma
+        # on its LSB, achieving fixed (or known) latency and alignment of
+        # packet starts.
+        # K.29.7 is chosen to avoid comma alignment issues arising from K.28.7.
+        link_init_r = Signal()
+        link_init_counter = Signal(max=32//nwords)
+        self.sync += [
+            link_init_r.eq(self.link_init),
+            If(link_init_r,
+                link_init_counter.eq(link_init_counter + 1),
+                [k.eq(1) for k in encoder.k],
+                [d.eq(K(29, 7)) for d in encoder.d[1:]],
+                If(link_init_counter == 0,
+                    encoder.d[0].eq(K(28, 7)),
+                ).Else(
+                    encoder.d[0].eq(K(29, 7)),
+                )
+            ).Else(
+                link_init_counter.eq(0)
+            )
+        ]

From 4e47decdbc276d97aa32fdb255ac072283c3874c Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Mon, 26 Sep 2016 14:14:14 +0800
Subject: [PATCH 002/134] drtio: add scrambler/descrambler and test

---
 artiq/gateware/drtio/link_layer.py          | 41 ++++++++++++++++++++-
 artiq/test/gateware/drtio/test_scrambler.py | 35 ++++++++++++++++++
 2 files changed, 75 insertions(+), 1 deletion(-)
 create mode 100644 artiq/test/gateware/drtio/test_scrambler.py

diff --git a/artiq/gateware/drtio/link_layer.py b/artiq/gateware/drtio/link_layer.py
index 76acbed4f..0274a0725 100644
--- a/artiq/gateware/drtio/link_layer.py
+++ b/artiq/gateware/drtio/link_layer.py
@@ -1,6 +1,45 @@
+from functools import reduce
+from operator import xor
+
 from migen import *
 
 
+class Scrambler(Module):
+    def __init__(self, n_io, n_state=23, taps=[17, 22]):
+        self.i = Signal(n_io)
+        self.o = Signal(n_io)
+
+        # # #
+
+        state = Signal(n_state, reset=1)
+        curval = [state[i] for i in range(n_state)]
+        for i in reversed(range(n_io)):
+            out = self.i[i] ^ reduce(xor, [curval[tap] for tap in taps])
+            self.sync += self.o[i].eq(out)
+            curval.insert(0, out)
+            curval.pop()
+
+        self.sync += state.eq(Cat(*curval[:n_state]))
+
+
+class Descrambler(Module):
+    def __init__(self, n_io, n_state=23, taps=[17, 22]):
+        self.i = Signal(n_io)
+        self.o = Signal(n_io)
+
+        # # #
+
+        state = Signal(n_state, reset=1)
+        curval = [state[i] for i in range(n_state)]
+        for i in reversed(range(n_io)):
+            flip = reduce(xor, [curval[tap] for tap in taps])
+            self.sync += self.o[i].eq(self.i[i] ^ flip)
+            curval.insert(0, self.i[i])
+            curval.pop()
+
+        self.sync += state.eq(Cat(*curval[:n_state]))
+
+
 def K(x, y):
     return (y << 5) | x
 
@@ -33,7 +72,7 @@ class LinkLayerTX(Module):
         # the following meanings:
         #   100 idle/auxiliary framing
         #   0AB 2 bits of auxiliary data
-        aux_scrambler = Scrambler(3*nwords)
+        aux_scrambler = CEInserter()(Scrambler(3*nwords))
         self.submodules += aux_scrambler
         aux_data_ctl = []
         for i in range(nwords):
diff --git a/artiq/test/gateware/drtio/test_scrambler.py b/artiq/test/gateware/drtio/test_scrambler.py
new file mode 100644
index 000000000..5c57a4454
--- /dev/null
+++ b/artiq/test/gateware/drtio/test_scrambler.py
@@ -0,0 +1,35 @@
+import unittest
+
+from migen import *
+
+from artiq.gateware.drtio.link_layer import Scrambler, Descrambler
+
+
+def process(dut, seq):
+    rseq = []
+    def pump():
+        yield dut.i.eq(seq[0])
+        yield
+        for w in seq[1:]:
+            yield dut.i.eq(w)
+            yield
+            rseq.append((yield dut.o))
+        yield
+        rseq.append((yield dut.o))
+    run_simulation(dut, pump())
+    return rseq
+
+
+class TestScrambler(unittest.TestCase):
+    def test_roundtrip(self):
+        seq = list(range(256))*3
+        scrambled_seq = process(Scrambler(8), seq)
+        descrambled_seq = process(Descrambler(8), scrambled_seq)
+        self.assertNotEqual(seq, scrambled_seq)
+        self.assertEqual(seq, descrambled_seq)
+
+    def test_resync(self):
+        seq = list(range(256))
+        scrambled_seq = process(Scrambler(8), seq)
+        descrambled_seq = process(Descrambler(8), scrambled_seq[20:])
+        self.assertEqual(seq[100:], descrambled_seq[80:])

From 8a92c2c7e509b22fe0af5521df092fd40f1dd628 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Tue, 27 Sep 2016 11:23:29 +0800
Subject: [PATCH 003/134] drtio: add RX link layer, fixes, simple loopback demo

---
 artiq/gateware/drtio/link_layer.py           | 63 +++++++++++++--
 artiq/test/gateware/drtio/test_link_layer.py | 84 ++++++++++++++++++++
 artiq/test/gateware/drtio/test_scrambler.py  | 35 --------
 3 files changed, 140 insertions(+), 42 deletions(-)
 create mode 100644 artiq/test/gateware/drtio/test_link_layer.py
 delete mode 100644 artiq/test/gateware/drtio/test_scrambler.py

diff --git a/artiq/gateware/drtio/link_layer.py b/artiq/gateware/drtio/link_layer.py
index 0274a0725..0b6eba04f 100644
--- a/artiq/gateware/drtio/link_layer.py
+++ b/artiq/gateware/drtio/link_layer.py
@@ -1,5 +1,5 @@
 from functools import reduce
-from operator import xor
+from operator import xor, or_
 
 from migen import *
 
@@ -59,8 +59,6 @@ class LinkLayerTX(Module):
         self.rt_frame = Signal()
         self.rt_data = Signal(8*nwords)
 
-        self.transceiver_data = Signal(10*nwords)
-
         # # #
 
         # Idle and auxiliary traffic use special characters excluding
@@ -88,7 +86,7 @@ class LinkLayerTX(Module):
             self.aux_ack.eq(~self.rt_frame)
         ]
         for i in range(nwords):
-            scrambled_ctl = scrambler.o[i*3:i*3+3]
+            scrambled_ctl = aux_scrambler.o[i*3:i*3+3]
             self.sync += [
                 encoder.k[i].eq(1),
                 If(scrambled_ctl == 7,
@@ -100,15 +98,18 @@ class LinkLayerTX(Module):
 
         # Real-time traffic uses data characters and is framed by the special
         # characters of auxiliary traffic. RT traffic is also scrambled.
-        rt_scrambler = Scrambler(8*nwords)
+        rt_scrambler = CEInserter()(Scrambler(8*nwords))
         self.submodules += rt_scrambler
-        self.comb += rt_scrambler.i.eq(self.rt_data)
+        self.comb += [
+            rt_scrambler.i.eq(self.rt_data),
+            rt_scrambler.ce.eq(self.rt_frame)
+        ]
         rt_frame_r = Signal()
         self.sync += [
             rt_frame_r.eq(self.rt_frame),
             If(rt_frame_r,
                 [k.eq(0) for k in encoder.k],
-                [d.eq(self.rt_data[i*8:i*8+8]) for i, d in enumerate(encoder.d)]
+                [d.eq(rt_scrambler.o[i*8:i*8+8]) for i, d in enumerate(encoder.d)]
             )
         ]
 
@@ -134,3 +135,51 @@ class LinkLayerTX(Module):
                 link_init_counter.eq(0)
             )
         ]
+
+
+class LinkLayerRX(Module):
+    def __init__(self, decoders):
+        nwords = len(decoders)
+        # nwords must be a power of 2
+        assert nwords & (nwords - 1) == 0
+
+        self.link_init = Signal()
+
+        self.aux_frame = Signal()
+        self.aux_data = Signal(2*nwords)
+
+        self.rt_frame = Signal()
+        self.rt_data = Signal(8*nwords)
+
+        # # #
+
+        aux_descrambler = CEInserter()(Descrambler(2*nwords))
+        rt_descrambler = CEInserter()(Descrambler(8*nwords))
+        self.submodules += aux_descrambler, rt_descrambler
+        self.comb += [
+            self.aux_frame.eq(~aux_descrambler.o[2]),
+            self.aux_data.eq(
+                Cat(*[aux_descrambler.o[3*i:3*i+2] for i in range(nwords)])),
+            self.rt_data.eq(rt_descrambler.o),
+        ]
+
+        link_init_d = Signal()
+        rt_frame_d = Signal()
+        self.sync += [
+            self.link_init.eq(link_init_d),
+            self.rt_frame.eq(rt_frame_d)
+        ]
+
+        self.comb += [
+            If(decoders[0].k,
+                If((decoders[0].d == K(28, 7)) | (decoders[0].d == K(29, 7)),
+                    link_init_d.eq(1)
+                ),
+                aux_descrambler.ce.eq(1)
+            ).Else(
+                rt_frame_d.eq(1),
+                rt_descrambler.ce.eq(1)
+            ),
+            aux_descrambler.i.eq(Cat(*[d.d >> 5 for d in decoders])),
+            rt_descrambler.i.eq(Cat(*[d.d for d in decoders]))
+        ]
diff --git a/artiq/test/gateware/drtio/test_link_layer.py b/artiq/test/gateware/drtio/test_link_layer.py
new file mode 100644
index 000000000..4e1906f52
--- /dev/null
+++ b/artiq/test/gateware/drtio/test_link_layer.py
@@ -0,0 +1,84 @@
+import unittest
+from types import SimpleNamespace
+
+from migen import *
+
+from artiq.gateware.drtio.link_layer import *
+
+
+def process(dut, seq):
+    rseq = []
+    def pump():
+        yield dut.i.eq(seq[0])
+        yield
+        for w in seq[1:]:
+            yield dut.i.eq(w)
+            yield
+            rseq.append((yield dut.o))
+        yield
+        rseq.append((yield dut.o))
+    run_simulation(dut, pump())
+    return rseq
+
+
+class TestScrambler(unittest.TestCase):
+    def test_roundtrip(self):
+        seq = list(range(256))*3
+        scrambled_seq = process(Scrambler(8), seq)
+        descrambled_seq = process(Descrambler(8), scrambled_seq)
+        self.assertNotEqual(seq, scrambled_seq)
+        self.assertEqual(seq, descrambled_seq)
+
+    def test_resync(self):
+        seq = list(range(256))
+        scrambled_seq = process(Scrambler(8), seq)
+        descrambled_seq = process(Descrambler(8), scrambled_seq[20:])
+        self.assertEqual(seq[100:], descrambled_seq[80:])
+
+
+class Loopback(Module):
+    def __init__(self, nwords):
+        ks = [Signal() for k in range(nwords)]
+        ds = [Signal(8) for d in range(nwords)]
+        encoder = SimpleNamespace(k=ks, d=ds)
+        decoders = [SimpleNamespace(k=k, d=d) for k, d in zip(ks, ds)]
+        self.submodules.tx = LinkLayerTX(encoder)
+        self.submodules.rx = LinkLayerRX(decoders)
+
+
+class TestLinkLayer(unittest.TestCase):
+    def test_packets(self):
+        dut = Loopback(4)
+
+        rt_packets = [
+            [0x12459970, 0x9938cdef, 0x12340000],
+            [0xabcdef00, 0x12345678],
+            [0xeeeeeeee, 0xffffffff, 0x01020304, 0x11223344]
+        ]
+        def transmit_rt_packets():
+            for packet in rt_packets:
+                yield dut.tx.rt_frame.eq(1)
+                for data in packet:
+                    yield dut.tx.rt_data.eq(data)
+                    yield
+                yield dut.tx.rt_frame.eq(0)
+                yield
+            # flush
+            for i in range(20):
+                yield
+
+        rx_rt_packets = []
+        @passive
+        def receive_rt_packets():
+            while True:
+                packet = []
+                rx_rt_packets.append(packet)
+                while not (yield dut.rx.rt_frame):
+                    yield
+                while (yield dut.rx.rt_frame):
+                    packet.append((yield dut.rx.rt_data))
+                    yield
+        run_simulation(dut, [transmit_rt_packets(), receive_rt_packets()])
+
+        for packet in rx_rt_packets:
+            print(" ".join("{:08x}".format(x) for x in packet))
diff --git a/artiq/test/gateware/drtio/test_scrambler.py b/artiq/test/gateware/drtio/test_scrambler.py
deleted file mode 100644
index 5c57a4454..000000000
--- a/artiq/test/gateware/drtio/test_scrambler.py
+++ /dev/null
@@ -1,35 +0,0 @@
-import unittest
-
-from migen import *
-
-from artiq.gateware.drtio.link_layer import Scrambler, Descrambler
-
-
-def process(dut, seq):
-    rseq = []
-    def pump():
-        yield dut.i.eq(seq[0])
-        yield
-        for w in seq[1:]:
-            yield dut.i.eq(w)
-            yield
-            rseq.append((yield dut.o))
-        yield
-        rseq.append((yield dut.o))
-    run_simulation(dut, pump())
-    return rseq
-
-
-class TestScrambler(unittest.TestCase):
-    def test_roundtrip(self):
-        seq = list(range(256))*3
-        scrambled_seq = process(Scrambler(8), seq)
-        descrambled_seq = process(Descrambler(8), scrambled_seq)
-        self.assertNotEqual(seq, scrambled_seq)
-        self.assertEqual(seq, descrambled_seq)
-
-    def test_resync(self):
-        seq = list(range(256))
-        scrambled_seq = process(Scrambler(8), seq)
-        descrambled_seq = process(Descrambler(8), scrambled_seq[20:])
-        self.assertEqual(seq[100:], descrambled_seq[80:])

From e59142e3449d6ed6e6c261cfe74373b7563800c8 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Tue, 27 Sep 2016 11:38:05 +0800
Subject: [PATCH 004/134] drtio: use additive scrambler reset by link init

---
 artiq/gateware/drtio/link_layer.py           | 36 ++++++--------------
 artiq/test/gateware/drtio/test_link_layer.py | 34 ++++++++++++------
 2 files changed, 35 insertions(+), 35 deletions(-)

diff --git a/artiq/gateware/drtio/link_layer.py b/artiq/gateware/drtio/link_layer.py
index 0b6eba04f..aac149b4f 100644
--- a/artiq/gateware/drtio/link_layer.py
+++ b/artiq/gateware/drtio/link_layer.py
@@ -5,24 +5,6 @@ from migen import *
 
 
 class Scrambler(Module):
-    def __init__(self, n_io, n_state=23, taps=[17, 22]):
-        self.i = Signal(n_io)
-        self.o = Signal(n_io)
-
-        # # #
-
-        state = Signal(n_state, reset=1)
-        curval = [state[i] for i in range(n_state)]
-        for i in reversed(range(n_io)):
-            out = self.i[i] ^ reduce(xor, [curval[tap] for tap in taps])
-            self.sync += self.o[i].eq(out)
-            curval.insert(0, out)
-            curval.pop()
-
-        self.sync += state.eq(Cat(*curval[:n_state]))
-
-
-class Descrambler(Module):
     def __init__(self, n_io, n_state=23, taps=[17, 22]):
         self.i = Signal(n_io)
         self.o = Signal(n_io)
@@ -33,8 +15,8 @@ class Descrambler(Module):
         curval = [state[i] for i in range(n_state)]
         for i in reversed(range(n_io)):
             flip = reduce(xor, [curval[tap] for tap in taps])
-            self.sync += self.o[i].eq(self.i[i] ^ flip)
-            curval.insert(0, self.i[i])
+            self.sync += self.o[i].eq(flip ^ self.i[i])
+            curval.insert(0, flip)
             curval.pop()
 
         self.sync += state.eq(Cat(*curval[:n_state]))
@@ -70,7 +52,7 @@ class LinkLayerTX(Module):
         # the following meanings:
         #   100 idle/auxiliary framing
         #   0AB 2 bits of auxiliary data
-        aux_scrambler = CEInserter()(Scrambler(3*nwords))
+        aux_scrambler = ResetInserter()(CEInserter()(Scrambler(3*nwords)))
         self.submodules += aux_scrambler
         aux_data_ctl = []
         for i in range(nwords):
@@ -82,6 +64,7 @@ class LinkLayerTX(Module):
             ).Else(
                 aux_scrambler.i.eq(Replicate(0b100, nwords))
             ),
+            aux_scrambler.reset.eq(self.link_init),
             aux_scrambler.ce.eq(~self.rt_frame),
             self.aux_ack.eq(~self.rt_frame)
         ]
@@ -98,10 +81,11 @@ class LinkLayerTX(Module):
 
         # Real-time traffic uses data characters and is framed by the special
         # characters of auxiliary traffic. RT traffic is also scrambled.
-        rt_scrambler = CEInserter()(Scrambler(8*nwords))
+        rt_scrambler = ResetInserter()(CEInserter()(Scrambler(8*nwords)))
         self.submodules += rt_scrambler
         self.comb += [
             rt_scrambler.i.eq(self.rt_data),
+            rt_scrambler.reset.eq(self.link_init),
             rt_scrambler.ce.eq(self.rt_frame)
         ]
         rt_frame_r = Signal()
@@ -153,8 +137,8 @@ class LinkLayerRX(Module):
 
         # # #
 
-        aux_descrambler = CEInserter()(Descrambler(2*nwords))
-        rt_descrambler = CEInserter()(Descrambler(8*nwords))
+        aux_descrambler = ResetInserter()(CEInserter()(Scrambler(2*nwords)))
+        rt_descrambler = ResetInserter()(CEInserter()(Scrambler(8*nwords)))
         self.submodules += aux_descrambler, rt_descrambler
         self.comb += [
             self.aux_frame.eq(~aux_descrambler.o[2]),
@@ -173,7 +157,9 @@ class LinkLayerRX(Module):
         self.comb += [
             If(decoders[0].k,
                 If((decoders[0].d == K(28, 7)) | (decoders[0].d == K(29, 7)),
-                    link_init_d.eq(1)
+                    link_init_d.eq(1),
+                    aux_descrambler.reset.eq(1),
+                    rt_descrambler.reset.eq(1)
                 ),
                 aux_descrambler.ce.eq(1)
             ).Else(
diff --git a/artiq/test/gateware/drtio/test_link_layer.py b/artiq/test/gateware/drtio/test_link_layer.py
index 4e1906f52..bb3b71ba5 100644
--- a/artiq/test/gateware/drtio/test_link_layer.py
+++ b/artiq/test/gateware/drtio/test_link_layer.py
@@ -6,7 +6,8 @@ from migen import *
 from artiq.gateware.drtio.link_layer import *
 
 
-def process(dut, seq):
+def process(seq):
+    dut = Scrambler(8)
     rseq = []
     def pump():
         yield dut.i.eq(seq[0])
@@ -24,17 +25,11 @@ def process(dut, seq):
 class TestScrambler(unittest.TestCase):
     def test_roundtrip(self):
         seq = list(range(256))*3
-        scrambled_seq = process(Scrambler(8), seq)
-        descrambled_seq = process(Descrambler(8), scrambled_seq)
+        scrambled_seq = process(seq)
+        descrambled_seq = process(scrambled_seq)
         self.assertNotEqual(seq, scrambled_seq)
         self.assertEqual(seq, descrambled_seq)
 
-    def test_resync(self):
-        seq = list(range(256))
-        scrambled_seq = process(Scrambler(8), seq)
-        descrambled_seq = process(Descrambler(8), scrambled_seq[20:])
-        self.assertEqual(seq[100:], descrambled_seq[80:])
-
 
 class Loopback(Module):
     def __init__(self, nwords):
@@ -50,12 +45,24 @@ class TestLinkLayer(unittest.TestCase):
     def test_packets(self):
         dut = Loopback(4)
 
+        def link_init():
+            yield dut.tx.link_init.eq(1)
+            yield
+            yield
+            yield dut.tx.link_init.eq(0)
+            yield
+
         rt_packets = [
             [0x12459970, 0x9938cdef, 0x12340000],
             [0xabcdef00, 0x12345678],
             [0xeeeeeeee, 0xffffffff, 0x01020304, 0x11223344]
         ]
         def transmit_rt_packets():
+            while not (yield dut.tx.link_init):
+                yield
+            while (yield dut.tx.link_init):
+                yield
+
             for packet in rt_packets:
                 yield dut.tx.rt_frame.eq(1)
                 for data in packet:
@@ -70,6 +77,11 @@ class TestLinkLayer(unittest.TestCase):
         rx_rt_packets = []
         @passive
         def receive_rt_packets():
+            while not (yield dut.tx.link_init):
+                yield
+            while (yield dut.tx.link_init):
+                yield
+
             while True:
                 packet = []
                 rx_rt_packets.append(packet)
@@ -78,7 +90,9 @@ class TestLinkLayer(unittest.TestCase):
                 while (yield dut.rx.rt_frame):
                     packet.append((yield dut.rx.rt_data))
                     yield
-        run_simulation(dut, [transmit_rt_packets(), receive_rt_packets()])
+
+        run_simulation(dut, [link_init(),
+                             transmit_rt_packets(), receive_rt_packets()])
 
         for packet in rx_rt_packets:
             print(" ".join("{:08x}".format(x) for x in packet))

From 95d7cba34a0a2ff536f6bf339c786fe5cb76fd97 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Tue, 27 Sep 2016 12:46:01 +0800
Subject: [PATCH 005/134] drtio: fixes, add aux packet test

---
 artiq/gateware/drtio/link_layer.py           | 11 ++-
 artiq/test/gateware/drtio/test_link_layer.py | 78 +++++++++++++++++---
 2 files changed, 75 insertions(+), 14 deletions(-)

diff --git a/artiq/gateware/drtio/link_layer.py b/artiq/gateware/drtio/link_layer.py
index aac149b4f..4f98a0a4a 100644
--- a/artiq/gateware/drtio/link_layer.py
+++ b/artiq/gateware/drtio/link_layer.py
@@ -56,7 +56,7 @@ class LinkLayerTX(Module):
         self.submodules += aux_scrambler
         aux_data_ctl = []
         for i in range(nwords):
-            aux_data_ctl.append(self.aux_data[i*2:i*2+1])
+            aux_data_ctl.append(self.aux_data[i*2:i*2+2])
             aux_data_ctl.append(0)
         self.comb += [
             If(self.aux_frame,
@@ -129,6 +129,7 @@ class LinkLayerRX(Module):
 
         self.link_init = Signal()
 
+        self.aux_stb = Signal()
         self.aux_frame = Signal()
         self.aux_data = Signal(2*nwords)
 
@@ -137,7 +138,7 @@ class LinkLayerRX(Module):
 
         # # #
 
-        aux_descrambler = ResetInserter()(CEInserter()(Scrambler(2*nwords)))
+        aux_descrambler = ResetInserter()(CEInserter()(Scrambler(3*nwords)))
         rt_descrambler = ResetInserter()(CEInserter()(Scrambler(8*nwords)))
         self.submodules += aux_descrambler, rt_descrambler
         self.comb += [
@@ -148,9 +149,11 @@ class LinkLayerRX(Module):
         ]
 
         link_init_d = Signal()
+        aux_stb_d = Signal()
         rt_frame_d = Signal()
         self.sync += [
             self.link_init.eq(link_init_d),
+            self.aux_stb.eq(aux_stb_d),
             self.rt_frame.eq(rt_frame_d)
         ]
 
@@ -160,12 +163,14 @@ class LinkLayerRX(Module):
                     link_init_d.eq(1),
                     aux_descrambler.reset.eq(1),
                     rt_descrambler.reset.eq(1)
+                ).Else(
+                    aux_stb_d.eq(1)
                 ),
                 aux_descrambler.ce.eq(1)
             ).Else(
                 rt_frame_d.eq(1),
                 rt_descrambler.ce.eq(1)
             ),
-            aux_descrambler.i.eq(Cat(*[d.d >> 5 for d in decoders])),
+            aux_descrambler.i.eq(Cat(*[d.d[5:] for d in decoders])),
             rt_descrambler.i.eq(Cat(*[d.d for d in decoders]))
         ]
diff --git a/artiq/test/gateware/drtio/test_link_layer.py b/artiq/test/gateware/drtio/test_link_layer.py
index bb3b71ba5..25a2cd2a0 100644
--- a/artiq/test/gateware/drtio/test_link_layer.py
+++ b/artiq/test/gateware/drtio/test_link_layer.py
@@ -55,7 +55,8 @@ class TestLinkLayer(unittest.TestCase):
         rt_packets = [
             [0x12459970, 0x9938cdef, 0x12340000],
             [0xabcdef00, 0x12345678],
-            [0xeeeeeeee, 0xffffffff, 0x01020304, 0x11223344]
+            [0xeeeeeeee, 0xffffffff, 0x01020304, 0x11223344],
+            [0x88277475, 0x19883332, 0x19837662, 0x81726668, 0x81876261]
         ]
         def transmit_rt_packets():
             while not (yield dut.tx.link_init):
@@ -77,22 +78,77 @@ class TestLinkLayer(unittest.TestCase):
         rx_rt_packets = []
         @passive
         def receive_rt_packets():
+            while not (yield dut.rx.link_init):
+                yield
+            while (yield dut.rx.link_init):
+                yield
+
+            previous_frame = 0
+            while True:
+                frame = yield dut.rx.rt_frame
+                if frame and not previous_frame:
+                    packet = []
+                    rx_rt_packets.append(packet)
+                previous_frame = frame
+                if frame:
+                    packet.append((yield dut.rx.rt_data))
+                yield
+
+        aux_packets = [
+            [0x12, 0x34],
+            [0x44, 0x11, 0x98, 0x78],
+            [0xbb, 0xaa, 0xdd, 0xcc, 0x00, 0xff, 0xee]
+        ]
+        def transmit_aux_packets():
             while not (yield dut.tx.link_init):
                 yield
             while (yield dut.tx.link_init):
                 yield
 
+            for packet in aux_packets:
+                yield dut.tx.aux_frame.eq(1)
+                for data in packet:
+                    yield dut.tx.aux_data.eq(data)
+                    yield
+                    while not (yield dut.tx.aux_ack):
+                        yield
+                yield dut.tx.aux_frame.eq(0)
+                yield
+                while not (yield dut.tx.aux_ack):
+                    yield
+            # flush
+            for i in range(20):
+                yield
+
+        rx_aux_packets = []
+        @passive
+        def receive_aux_packets():
+            while not (yield dut.rx.link_init):
+                yield
+            while (yield dut.rx.link_init):
+                yield
+
+            previous_frame = 0
             while True:
-                packet = []
-                rx_rt_packets.append(packet)
-                while not (yield dut.rx.rt_frame):
-                    yield
-                while (yield dut.rx.rt_frame):
-                    packet.append((yield dut.rx.rt_data))
-                    yield
+                if (yield dut.rx.aux_stb):
+                    frame = yield dut.rx.aux_frame
+                    if frame and not previous_frame:
+                        packet = []
+                        rx_aux_packets.append(packet)
+                    previous_frame = frame
+                    if frame:
+                        packet.append((yield dut.rx.aux_data))
+                yield
 
         run_simulation(dut, [link_init(),
-                             transmit_rt_packets(), receive_rt_packets()])
+                             transmit_rt_packets(), receive_rt_packets(),
+                             transmit_aux_packets(), receive_aux_packets()])
 
-        for packet in rx_rt_packets:
-            print(" ".join("{:08x}".format(x) for x in packet))
+        # print("RT:")
+        # for packet in rx_rt_packets:
+        #     print(" ".join("{:08x}".format(x) for x in packet))
+        # print("AUX:")
+        # for packet in rx_aux_packets:
+        #     print(" ".join("{:02x}".format(x) for x in packet))
+        self.assertEqual(rt_packets, rx_rt_packets)
+        self.assertEqual(aux_packets, rx_aux_packets)

From 08772f7a712f84cd1987b2499cf2b97fdc544019 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Tue, 27 Sep 2016 19:02:54 +0800
Subject: [PATCH 006/134] drtio: add RX ready signaling

---
 artiq/gateware/drtio/link_layer.py | 52 ++++++++++++++++++++++++------
 1 file changed, 42 insertions(+), 10 deletions(-)

diff --git a/artiq/gateware/drtio/link_layer.py b/artiq/gateware/drtio/link_layer.py
index 4f98a0a4a..79d4a629c 100644
--- a/artiq/gateware/drtio/link_layer.py
+++ b/artiq/gateware/drtio/link_layer.py
@@ -2,6 +2,7 @@ from functools import reduce
 from operator import xor, or_
 
 from migen import *
+from migen.genlib.fsm import *
 
 
 class Scrambler(Module):
@@ -33,6 +34,7 @@ class LinkLayerTX(Module):
         assert nwords & (nwords - 1) == 0
 
         self.link_init = Signal()
+        self.signal_rx_ready = Signal()
 
         self.aux_frame = Signal()
         self.aux_data = Signal(2*nwords)
@@ -43,8 +45,8 @@ class LinkLayerTX(Module):
 
         # # #
 
-        # Idle and auxiliary traffic use special characters excluding
-        # K.28.7 and K.29.7 in order to easily separate the link initialization
+        # Idle and auxiliary traffic use special characters excluding K.28.7,
+        # K.29.7 and K.30.7 in order to easily separate the link initialization
         # phase (K.28.7 is additionally excluded as we cannot guarantee its
         # non-repetition here).
         # A set of 8 special characters is chosen using a 3-bit control word.
@@ -97,11 +99,14 @@ class LinkLayerTX(Module):
             )
         ]
 
-        # During link init, send a series of 1*K.28.7 (comma) + 31*K.29.7
+        # During link init, send a series of 1*K.28.7 (comma) + 31*K.29.7/K.30.7
         # The receiving end configures its transceiver to also place the comma
         # on its LSB, achieving fixed (or known) latency and alignment of
         # packet starts.
-        # K.29.7 is chosen to avoid comma alignment issues arising from K.28.7.
+        # K.29.7 and K.30.7 are chosen to avoid comma alignment issues arising
+        # from K.28.7.
+        # K.30.7 is sent instead of K.29.7 to signal the alignment of the local
+        # receiver, thus the remote can end its link initialization pattern.
         link_init_r = Signal()
         link_init_counter = Signal(max=32//nwords)
         self.sync += [
@@ -109,11 +114,19 @@ class LinkLayerTX(Module):
             If(link_init_r,
                 link_init_counter.eq(link_init_counter + 1),
                 [k.eq(1) for k in encoder.k],
-                [d.eq(K(29, 7)) for d in encoder.d[1:]],
+                If(self.signal_rx_ready,
+                    [d.eq(K(30, 7)) for d in encoder.d[1:]]
+                ).Else(
+                    [d.eq(K(29, 7)) for d in encoder.d[1:]]
+                ),
                 If(link_init_counter == 0,
                     encoder.d[0].eq(K(28, 7)),
                 ).Else(
-                    encoder.d[0].eq(K(29, 7)),
+                    If(self.signal_rx_ready,
+                        encoder.d[0].eq(K(30, 7))
+                    ).Else(
+                        encoder.d[0].eq(K(29, 7))
+                    )
                 )
             ).Else(
                 link_init_counter.eq(0)
@@ -128,6 +141,7 @@ class LinkLayerRX(Module):
         assert nwords & (nwords - 1) == 0
 
         self.link_init = Signal()
+        self.remote_rx_ready = Signal()
 
         self.aux_stb = Signal()
         self.aux_frame = Signal()
@@ -148,19 +162,21 @@ class LinkLayerRX(Module):
             self.rt_data.eq(rt_descrambler.o),
         ]
 
-        link_init_d = Signal()
         aux_stb_d = Signal()
         rt_frame_d = Signal()
         self.sync += [
-            self.link_init.eq(link_init_d),
             self.aux_stb.eq(aux_stb_d),
             self.rt_frame.eq(rt_frame_d)
         ]
 
+        link_init_char = Signal()
         self.comb += [
+            link_init_char.eq(
+                (decoders[0].d == K(28, 7)) |
+                (decoders[0].d == K(29, 7)) |
+                (decoders[0].d == K(30, 7))),
             If(decoders[0].k,
-                If((decoders[0].d == K(28, 7)) | (decoders[0].d == K(29, 7)),
-                    link_init_d.eq(1),
+                If(link_init_char,
                     aux_descrambler.reset.eq(1),
                     rt_descrambler.reset.eq(1)
                 ).Else(
@@ -174,3 +190,19 @@ class LinkLayerRX(Module):
             aux_descrambler.i.eq(Cat(*[d.d[5:] for d in decoders])),
             rt_descrambler.i.eq(Cat(*[d.d for d in decoders]))
         ]
+        self.sync += [
+            self.link_init.eq(0),
+            If(decoders[0].k,
+                If(link_init_char, self.link_init.eq(1)),
+                If(decoders[0].d == K(30, 7),
+                    self.remote_rx_ready.eq(1)
+                ).Elif(decoders[0].d != K(28, 7),
+                    self.remote_rx_ready.eq(0)
+                ),
+                If(decoders[0].d == K(30, 7),
+                    self.remote_rx_ready.eq(1)
+                ) if len(decoders) > 1 else None
+            ).Else(
+                self.remote_rx_ready.eq(0)
+            )
+        ]

From cefb9e140502ea437310b49a80ed4a597c8dcea9 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Tue, 27 Sep 2016 21:41:57 +0800
Subject: [PATCH 007/134] drtio: add full link layer

---
 artiq/gateware/drtio/link_layer.py | 56 ++++++++++++++++++++++++++++++
 1 file changed, 56 insertions(+)

diff --git a/artiq/gateware/drtio/link_layer.py b/artiq/gateware/drtio/link_layer.py
index 79d4a629c..65cab5e9c 100644
--- a/artiq/gateware/drtio/link_layer.py
+++ b/artiq/gateware/drtio/link_layer.py
@@ -206,3 +206,59 @@ class LinkLayerRX(Module):
                 self.remote_rx_ready.eq(0)
             )
         ]
+
+
+class LinkLayer(Module):
+    def __init__(self, encoder, decoders):
+        self.reset = Signal()
+        self.ready = Signal()
+
+        # pulsed to reset receiver, rx_ready must immediately go low
+        self.rx_reset = Signal()
+        # receiver locked including comma alignment
+        self.rx_ready = Signal()
+
+        tx = LinkLayerTX(encoder)
+        rx = LinkLayerRX(decoders)
+        self.submodules += tx, rx
+
+        self.tx_aux_frame = tx.aux_frame
+        self.tx_aux_data = tx.aux_data
+        self.tx_aux_ack = tx.aux_ack
+        self.tx_rt_frame = tx.rt_frame
+        self.tx_rt_data = tx.rt_data
+
+        self.rx_aux_stb = rx.aux_stb
+        self.rx_aux_frame = rx.aux_frame
+        self.rx_aux_data = rx.aux_data
+        self.rx_rt_frame = rx.rt_frame
+        self.rx_rt_data = rx.rt_data
+
+        # # #
+
+        fsm = ResetInserter()(FSM(reset_state="RESET_RX"))
+        self.submodules += fsm
+
+        self.comb += fsm.reset.eq(self.reset)
+
+        fsm.act("RESET_RX",
+            tx.link_init.eq(1),
+            self.rx_reset.eq(1),
+            NextState("WAIT_LOCAL_RX_READY")
+        )
+        fsm.act("WAIT_LOCAL_RX_READY",
+            tx.link_init.eq(1),
+            If(self.rx_ready,
+                NextState("WAIT_REMOTE_RX_READY")
+            )
+        )
+        fsm.act("WAIT_REMOTE_RX_READY",
+            tx.link_init.eq(1),
+            tx.signal_rx_ready.eq(1),
+            If(rx.remote_rx_ready,
+                NextState("READY")
+            )
+        )
+        fsm.act("READY",
+            self.ready.eq(1)
+        )

From 1e0c6d6d5dfffdf67ff9c5152eda956324a9739a Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Fri, 30 Sep 2016 11:25:06 +0800
Subject: [PATCH 008/134] drtio: monitor received link_init

---
 artiq/gateware/drtio/link_layer.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/artiq/gateware/drtio/link_layer.py b/artiq/gateware/drtio/link_layer.py
index 65cab5e9c..47bcf9336 100644
--- a/artiq/gateware/drtio/link_layer.py
+++ b/artiq/gateware/drtio/link_layer.py
@@ -256,9 +256,13 @@ class LinkLayer(Module):
             tx.link_init.eq(1),
             tx.signal_rx_ready.eq(1),
             If(rx.remote_rx_ready,
-                NextState("READY")
+                NextState("WAIT_REMOTE_LINK_UP")
             )
         )
+        fsm.act("WAIT_REMOTE_LINK_UP",
+            If(~rx.link_init, NextState("READY"))
+        )
         fsm.act("READY",
+            If(rx.link_init, NextState("RESET_RX")),
             self.ready.eq(1)
         )

From 76bac21d14e522f054cbf9f59c3a4e6f721505b1 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Thu, 6 Oct 2016 18:51:20 +0800
Subject: [PATCH 009/134] drtio: RT RX datapath, untested

---
 artiq/gateware/drtio/rt_packets.py | 146 +++++++++++++++++++++++++++++
 1 file changed, 146 insertions(+)
 create mode 100644 artiq/gateware/drtio/rt_packets.py

diff --git a/artiq/gateware/drtio/rt_packets.py b/artiq/gateware/drtio/rt_packets.py
new file mode 100644
index 000000000..dff21b6f5
--- /dev/null
+++ b/artiq/gateware/drtio/rt_packets.py
@@ -0,0 +1,146 @@
+from migen import *
+from migen.genlib.fsm import *
+from migen.genlib.record import *
+
+class PacketLayoutManager:
+    def __init__(self, alignment):
+        self.alignment = alignment
+        self.layouts = dict()
+        self.types = dict()
+    
+    def add_type(self, name, *fields, pad=True):
+        self.types[name] = len(self.types)
+        layout = [("ty", 8)] + list(fields)
+        misalignment = layout_len(layout) % self.alignment
+        if misalignment:
+            layout.append(("packet_pad", self.alignment - misalignment))
+        self.layouts[name] = layout
+
+
+def get_m2s_layouts(alignment):
+    plm = PacketLayoutManager(alignment)
+    plm.add_type("echo_request")
+    plm.add_type("set_time", ("timestamp", 64))
+    plm.add_type("write", ("timestamp", 64),
+                          ("channel", 16),
+                          ("address", 16),
+                          ("data_len", 8),
+                          ("short_data", 8))
+    plm.add_type("fifo_level_request", ("channel", 16))
+    return plm
+
+
+def get_s2m_layouts(alignment):
+    plm = PacketLayoutManager(alignment)
+    plm.add_type("echo_reply")
+    plm.add_type("fifo_level_reply", ("level", 24))
+    return plm
+
+
+class ReceiveDatapath(Module):
+    def __init__(self, ws, plm):
+        # inputs
+        self.frame = Signal()
+        self.data = Signal(ws)
+
+        # control
+        self.packet_buffer_load = Signal()
+
+        # outputs
+        self.frame_r = Signal()
+        self.data_r = Signal()
+        self.packet_type = Signal(8)
+        self.packet_last = Signal()
+        self.packet_as = dict()
+
+        # # #
+
+        # input pipeline stage - determine packet length based on type
+        lastword_per_type = [layout_len(l)//ws - 1
+                             for l in plm.layouts.values()]
+        packet_last_n = Signal(max=max(lastword_per_type)+1)
+        self.sync += [
+            self.frame_r.eq(self.frame),
+            self.data_r.eq(self.data),
+            If(self.frame & ~self.frame_r,
+                self.packet_type.eq(self.data[:8]),
+                packet_last_n.eq(Array(lastword_per_type)[self.data[:8]])
+            )
+        ]
+
+        # bufferize packet
+        packet_buffer = Signal(max(layout_len(l)
+                                   for l in plm.layouts.values()))
+        w_in_packet = len(packet_buffer)//ws
+        packet_buffer_count = Signal(max=w_in_packet+1)
+        self.sync += \
+            If(self.packet_buffer_load,
+                Case(packet_buffer_count, 
+                     {i: packet_buffer[i*ws:(i+1)*ws].eq(self.data_r)
+                      for i in range(w_in_packet)}),
+                packet_buffer_count.eq(packet_buffer_count + 1)
+            ).Else(
+                packet_buffer_count.eq(0)
+            )
+        self.comb += self.packet_last.eq(packet_buffer_count == packet_last_n)
+
+        # cast packet
+        for name, layout in plm.layouts.items():
+            self.packet_as[name] = Record(layout)
+            self.comb += self.packet_as[name].raw_bits().eq(packet_buffer)
+
+
+class RTPacketSatellite(Module):
+    def __init__(self, nwords):
+        ws = 8*nwords
+        self.rx_rt_frame = Signal()
+        self.rx_rt_data = Signal(ws)
+
+        self.tx_rt_frame = Signal()
+        self.tx_rt_data = Signal(ws)
+
+        # # #
+
+        rx_plm = get_m2s_layouts(ws)
+        rx_dp = ReceiveDatapath(ws, rx_plm)
+        self.submodules += rx_dp
+        self.comb += [
+            rx_dp.frame.eq(self.rx_rt_frame),
+            rx_dp.data.eq(self.rx_rt_data)
+        ]
+
+        fsm = FSM(reset_state="WAIT_FRAME")
+        self.submodules += fsm
+
+        continuation = Signal()
+        continuation_r = Signal()
+        frame_r_r = Signal()
+        self.sync += [
+            continuation_r.eq(continuation),
+            frame_r_r.eq(rx_dp.frame_r)
+        ]
+        fsm.act("WAIT_INPUT",
+            If(rx_dp.frame_r,
+                If(~frame_r_r | continuation_r,
+                    continuation.eq(1),
+                    packet_buffer_load.eq(1),
+                    If(rx_dp.packet_last,
+                        Case(rx_dp.packet_type, {
+                            rx_plm.types["echo_request"]: NextState("ECHO"),
+                            "default": NextState("ERROR_UNKNOWN_TYPE")
+                        })
+                    )
+                ).Else(
+                    NextState("ERROR_FRAME_MISSED")
+                )
+            )
+        )
+        fsm.state("ECHO",
+            NextState("WAIT_INPUT")
+        )
+        fsm.state("ERROR_FRAME_MISSED",
+            NextState("WAIT_INPUT")
+        )
+        fsm.state("ERROR_UNKNOWN_TYPE",
+            NextState("WAIT_INPUT")
+        )

From cb0d1549c6116415ff998492814b29cb7ef40556 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Fri, 7 Oct 2016 15:35:29 +0800
Subject: [PATCH 010/134] drtio: add rt_packets TX datapath, fixes

---
 artiq/gateware/drtio/rt_packets.py | 123 +++++++++++++++++++++++++----
 1 file changed, 108 insertions(+), 15 deletions(-)

diff --git a/artiq/gateware/drtio/rt_packets.py b/artiq/gateware/drtio/rt_packets.py
index dff21b6f5..199cc2071 100644
--- a/artiq/gateware/drtio/rt_packets.py
+++ b/artiq/gateware/drtio/rt_packets.py
@@ -1,15 +1,24 @@
+from types import SimpleNamespace
+
 from migen import *
 from migen.genlib.fsm import *
-from migen.genlib.record import *
+
+
+def layout_len(l):
+    return sum(e[1] for e in l)
+
 
 class PacketLayoutManager:
     def __init__(self, alignment):
         self.alignment = alignment
         self.layouts = dict()
         self.types = dict()
+        self.type_names = dict()
     
     def add_type(self, name, *fields, pad=True):
-        self.types[name] = len(self.types)
+        type_n = len(self.types)
+        self.types[name] = type_n
+        self.type_names[type_n] = name
         layout = [("ty", 8)] + list(fields)
         misalignment = layout_len(layout) % self.alignment
         if misalignment:
@@ -32,11 +41,18 @@ def get_m2s_layouts(alignment):
 
 def get_s2m_layouts(alignment):
     plm = PacketLayoutManager(alignment)
+    plm.add_type("error", ("code", 8))
     plm.add_type("echo_reply")
     plm.add_type("fifo_level_reply", ("level", 24))
     return plm
 
 
+error_codes = {
+    "frame_missed": 0,
+    "unknown_type": 1
+}
+
+
 class ReceiveDatapath(Module):
     def __init__(self, ws, plm):
         # inputs
@@ -56,8 +72,8 @@ class ReceiveDatapath(Module):
         # # #
 
         # input pipeline stage - determine packet length based on type
-        lastword_per_type = [layout_len(l)//ws - 1
-                             for l in plm.layouts.values()]
+        lastword_per_type = [layout_len(plm.layouts[plm.type_names[i]])//ws - 1
+                             for i in range(len(plm.layouts))]
         packet_last_n = Signal(max=max(lastword_per_type)+1)
         self.sync += [
             self.frame_r.eq(self.frame),
@@ -84,10 +100,73 @@ class ReceiveDatapath(Module):
             )
         self.comb += self.packet_last.eq(packet_buffer_count == packet_last_n)
 
-        # cast packet
+        # dissect packet
         for name, layout in plm.layouts.items():
-            self.packet_as[name] = Record(layout)
-            self.comb += self.packet_as[name].raw_bits().eq(packet_buffer)
+            fields = SimpleNamespace()
+            idx = 0
+            for field_name, field_size in layout:
+                setattr(fields, field_name, packet_buffer[idx:idx+field_size])
+                idx += field_size
+            self.packet_as[name] = fields
+
+
+class TransmitDatapath(Module):
+    def __init__(self, ws, plm):
+        self.ws = ws
+        self.plm = plm
+
+        # inputs
+        self.packet_buffer = Signal(max(layout_len(l)
+                                        for l in plm.layouts.values()))
+        w_in_packet = len(self.packet_buffer)//ws
+        self.packet_len = Signal(max=w_in_packet+1)
+
+        # control
+        self.stb = Signal()
+        self.done = Signal()
+
+        # outputs
+        self.frame = Signal()
+        self.data = Signal(ws)
+
+        # # #
+
+        packet_buffer_count = Signal(max=w_in_packet+1)
+        self.sync += [
+            self.done.eq(0),
+            self.frame.eq(0),
+            packet_buffer_count.eq(0),
+
+            If(self.stb & ~self.done,
+                If(packet_buffer_count == self.packet_len,
+                    self.done.eq(1)
+                ).Else(
+                    self.frame.eq(1),
+                    Case(packet_buffer_count, 
+                         {i: self.data.eq(self.packet_buffer[i*ws:(i+1)*ws])
+                          for i in range(w_in_packet)}),
+                    packet_buffer_count.eq(packet_buffer_count + 1)
+                )
+            )
+        ]
+
+    def send(self, ty, **kwargs):
+        idx = 8
+        value = self.plm.types[ty]
+        for field_name, field_size in self.plm.layouts[ty][1:]:
+            try:
+                fvalue = kwargs[field_name]
+                del kwargs[field_name]
+            except KeyError:
+                fvalue = 0
+            value = value | (fvalue << idx)
+            idx += field_size
+        if kwargs:
+            raise ValueError
+        return [
+            self.packet_buffer.eq(value),
+            self.packet_len.eq(idx//self.ws)
+        ]
 
 
 class RTPacketSatellite(Module):
@@ -109,7 +188,15 @@ class RTPacketSatellite(Module):
             rx_dp.data.eq(self.rx_rt_data)
         ]
 
-        fsm = FSM(reset_state="WAIT_FRAME")
+        tx_plm = get_s2m_layouts(ws)
+        tx_dp = TransmitDatapath(ws, tx_plm)
+        self.submodules += tx_dp
+        self.comb += [
+            self.tx_rt_frame.eq(tx_dp.frame),
+            self.tx_rt_data.eq(tx_dp.data)
+        ]
+
+        fsm = FSM(reset_state="WAIT_INPUT")
         self.submodules += fsm
 
         continuation = Signal()
@@ -123,7 +210,7 @@ class RTPacketSatellite(Module):
             If(rx_dp.frame_r,
                 If(~frame_r_r | continuation_r,
                     continuation.eq(1),
-                    packet_buffer_load.eq(1),
+                    rx_dp.packet_buffer_load.eq(1),
                     If(rx_dp.packet_last,
                         Case(rx_dp.packet_type, {
                             rx_plm.types["echo_request"]: NextState("ECHO"),
@@ -135,12 +222,18 @@ class RTPacketSatellite(Module):
                 )
             )
         )
-        fsm.state("ECHO",
-            NextState("WAIT_INPUT")
+        fsm.act("ECHO",
+            tx_dp.send("echo_reply"),
+            tx_dp.stb.eq(1),
+            If(tx_dp.done, NextState("WAIT_INPUT"))
         )
-        fsm.state("ERROR_FRAME_MISSED",
-            NextState("WAIT_INPUT")
+        fsm.act("ERROR_FRAME_MISSED",
+            tx_dp.send("error", code=error_codes["frame_missed"]),
+            tx_dp.stb.eq(1),
+            If(tx_dp.done, NextState("WAIT_INPUT"))
         )
-        fsm.state("ERROR_UNKNOWN_TYPE",
-            NextState("WAIT_INPUT")
+        fsm.act("ERROR_UNKNOWN_TYPE",
+            tx_dp.send("error", code=error_codes["unknown_type"]),
+            tx_dp.stb.eq(1),
+            If(tx_dp.done, NextState("WAIT_INPUT"))
         )

From 0574e882d2b1faf4c34c2be8801e2c05d7ac65ae Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Fri, 7 Oct 2016 15:36:32 +0800
Subject: [PATCH 011/134] drtio: basic RT packet echo test

---
 artiq/test/gateware/drtio/test_rt_packets.py | 67 ++++++++++++++++++++
 1 file changed, 67 insertions(+)
 create mode 100644 artiq/test/gateware/drtio/test_rt_packets.py

diff --git a/artiq/test/gateware/drtio/test_rt_packets.py b/artiq/test/gateware/drtio/test_rt_packets.py
new file mode 100644
index 000000000..f8bb1e907
--- /dev/null
+++ b/artiq/test/gateware/drtio/test_rt_packets.py
@@ -0,0 +1,67 @@
+import unittest
+
+from migen import *
+
+from artiq.gateware.drtio.rt_packets import *
+
+
+class PacketInterface:
+    def __init__(self, direction, frame, data):
+        if direction == "m2s":
+            self.plm = get_m2s_layouts(len(data))
+        elif direction == "s2m":
+            self.plm = get_s2m_layouts(len(data))
+        else:
+            raise ValueError
+        self.frame = frame
+        self.data = data
+
+    def send(self, ty, **kwargs):
+        idx = 8
+        value = self.plm.types[ty]
+        for field_name, field_size in self.plm.layouts[ty][1:]:
+            try:
+                fvalue = kwargs[field_name]
+                del kwargs[field_name]
+            except KeyError:
+                fvalue = 0
+            value = value | (fvalue << idx)
+            idx += field_size
+        if kwargs:
+            raise ValueError
+
+        ws = len(self.data)
+        yield self.frame.eq(1)
+        for i in range(idx//ws):
+            yield self.data.eq(value)
+            value >>= ws
+            yield
+        yield self.frame.eq(0)
+        yield
+
+    @passive
+    def receive(self, callback):
+        previous_frame = 0
+        frame_words = []
+        while True:
+            frame = yield self.frame
+            if frame:
+                frame_words.append((yield self.data))
+            if previous_frame and not frame:
+                callback(frame_words)
+                frame_words = []
+            previous_frame = frame
+            yield
+
+
+class TestSatellite(unittest.TestCase):
+    def test_echo(self):
+        nwords = 4
+        dut = RTPacketSatellite(nwords)
+        pt = PacketInterface("m2s", dut.rx_rt_frame, dut.rx_rt_data)
+        pr = PacketInterface("s2m", dut.tx_rt_frame, dut.tx_rt_data)
+        def send():
+            yield from pt.send("echo_request")
+            for i in range(40):
+                yield
+        run_simulation(dut, [send(), pr.receive(print)])

From 43caffc1684524e2e398845de4f95425948816ee Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Fri, 7 Oct 2016 17:31:51 +0800
Subject: [PATCH 012/134] drtio: self-checking echo test

---
 artiq/test/gateware/drtio/test_rt_packets.py | 46 +++++++++++++++-----
 1 file changed, 36 insertions(+), 10 deletions(-)

diff --git a/artiq/test/gateware/drtio/test_rt_packets.py b/artiq/test/gateware/drtio/test_rt_packets.py
index f8bb1e907..7a427950b 100644
--- a/artiq/test/gateware/drtio/test_rt_packets.py
+++ b/artiq/test/gateware/drtio/test_rt_packets.py
@@ -48,7 +48,27 @@ class PacketInterface:
             if frame:
                 frame_words.append((yield self.data))
             if previous_frame and not frame:
-                callback(frame_words)
+                packet_type = self.plm.type_names[frame_words[0] & 0xff]
+                packet_nwords = layout_len(self.plm.layouts[packet_type]) \
+                                //len(self.data)
+                packet, trailer = frame_words[:packet_nwords], \
+                                  frame_words[packet_nwords:]
+
+                n = 0
+                packet_int = 0
+                for w in packet:
+                    packet_int |= (w << n)
+                    n += len(self.data)
+
+                field_dict = dict()
+                idx = 0
+                for field_name, field_size in self.plm.layouts[packet_type]:
+                    v = (packet_int >> idx) & (2**field_size - 1)
+                    field_dict[field_name] = v
+                    idx += field_size
+
+                callback(packet_type, field_dict, trailer)
+
                 frame_words = []
             previous_frame = frame
             yield
@@ -56,12 +76,18 @@ class PacketInterface:
 
 class TestSatellite(unittest.TestCase):
     def test_echo(self):
-        nwords = 4
-        dut = RTPacketSatellite(nwords)
-        pt = PacketInterface("m2s", dut.rx_rt_frame, dut.rx_rt_data)
-        pr = PacketInterface("s2m", dut.tx_rt_frame, dut.tx_rt_data)
-        def send():
-            yield from pt.send("echo_request")
-            for i in range(40):
-                yield
-        run_simulation(dut, [send(), pr.receive(print)])
+        for nwords in range(1, 8):
+            dut = RTPacketSatellite(nwords)
+            pt = PacketInterface("m2s", dut.rx_rt_frame, dut.rx_rt_data)
+            pr = PacketInterface("s2m", dut.tx_rt_frame, dut.tx_rt_data)
+            completed = False
+            def send():
+                yield from pt.send("echo_request")
+                while not completed:
+                    yield
+            def receive(packet_type, field_dict, trailer):
+                nonlocal completed
+                self.assertEqual(packet_type, "echo_reply")
+                self.assertEqual(trailer, [])
+                completed = True
+            run_simulation(dut, [send(), pr.receive(receive)])

From 23b33022004254c982c14e5729b5e994bc8a42b7 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Fri, 7 Oct 2016 19:30:53 +0800
Subject: [PATCH 013/134] drtio: implement TSC load in satellite

---
 artiq/gateware/drtio/rt_packets.py           | 21 ++++++++++++++++---
 artiq/test/gateware/drtio/test_rt_packets.py | 22 ++++++++++++++++++++
 2 files changed, 40 insertions(+), 3 deletions(-)

diff --git a/artiq/gateware/drtio/rt_packets.py b/artiq/gateware/drtio/rt_packets.py
index 199cc2071..be7971316 100644
--- a/artiq/gateware/drtio/rt_packets.py
+++ b/artiq/gateware/drtio/rt_packets.py
@@ -64,7 +64,7 @@ class ReceiveDatapath(Module):
 
         # outputs
         self.frame_r = Signal()
-        self.data_r = Signal()
+        self.data_r = Signal(ws)
         self.packet_type = Signal(8)
         self.packet_last = Signal()
         self.packet_as = dict()
@@ -171,15 +171,20 @@ class TransmitDatapath(Module):
 
 class RTPacketSatellite(Module):
     def __init__(self, nwords):
+        # link layer interface
         ws = 8*nwords
         self.rx_rt_frame = Signal()
         self.rx_rt_data = Signal(ws)
-
         self.tx_rt_frame = Signal()
         self.tx_rt_data = Signal(ws)
 
+        # I/O Timer interface
+        self.tsc_load = Signal()
+        self.tsc_value = Signal(64)
+
         # # #
 
+        # RX/TX datapath
         rx_plm = get_m2s_layouts(ws)
         rx_dp = ReceiveDatapath(ws, rx_plm)
         self.submodules += rx_dp
@@ -187,7 +192,6 @@ class RTPacketSatellite(Module):
             rx_dp.frame.eq(self.rx_rt_frame),
             rx_dp.data.eq(self.rx_rt_data)
         ]
-
         tx_plm = get_s2m_layouts(ws)
         tx_dp = TransmitDatapath(ws, tx_plm)
         self.submodules += tx_dp
@@ -196,6 +200,12 @@ class RTPacketSatellite(Module):
             self.tx_rt_data.eq(tx_dp.data)
         ]
 
+        # glue
+        self.comb += [
+            self.tsc_value.eq(rx_dp.packet_as["set_time"].timestamp)
+        ]
+
+        # main control FSM
         fsm = FSM(reset_state="WAIT_INPUT")
         self.submodules += fsm
 
@@ -214,6 +224,7 @@ class RTPacketSatellite(Module):
                     If(rx_dp.packet_last,
                         Case(rx_dp.packet_type, {
                             rx_plm.types["echo_request"]: NextState("ECHO"),
+                            rx_plm.types["set_time"]: NextState("SET_TIME"),
                             "default": NextState("ERROR_UNKNOWN_TYPE")
                         })
                     )
@@ -227,6 +238,10 @@ class RTPacketSatellite(Module):
             tx_dp.stb.eq(1),
             If(tx_dp.done, NextState("WAIT_INPUT"))
         )
+        fsm.act("SET_TIME",
+            self.tsc_load.eq(1),
+            NextState("WAIT_INPUT")
+        )
         fsm.act("ERROR_FRAME_MISSED",
             tx_dp.send("error", code=error_codes["frame_missed"]),
             tx_dp.stb.eq(1),
diff --git a/artiq/test/gateware/drtio/test_rt_packets.py b/artiq/test/gateware/drtio/test_rt_packets.py
index 7a427950b..594c37f15 100644
--- a/artiq/test/gateware/drtio/test_rt_packets.py
+++ b/artiq/test/gateware/drtio/test_rt_packets.py
@@ -91,3 +91,25 @@ class TestSatellite(unittest.TestCase):
                 self.assertEqual(trailer, [])
                 completed = True
             run_simulation(dut, [send(), pr.receive(receive)])
+
+    def test_set_time(self):
+        for nwords in range(1, 8):
+            dut = RTPacketSatellite(nwords)
+            pt = PacketInterface("m2s", dut.rx_rt_frame, dut.rx_rt_data)
+            tx_times = [0x12345678aabbccdd, 0x0102030405060708,
+                        0xaabbccddeeff1122]
+            def send():
+                for t in tx_times:
+                    yield from pt.send("set_time", timestamp=t)
+                # flush
+                for i in range(10):
+                    yield
+            rx_times = []
+            @passive
+            def receive():
+                while True:
+                    if (yield dut.tsc_load):
+                        rx_times.append((yield dut.tsc_value))
+                    yield
+            run_simulation(dut, [send(), receive()], vcd_name="foo.vcd")
+            self.assertEqual(tx_times, rx_times)

From 87ec333f5556a20ce533e3913e9ed90210b3f20a Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Mon, 10 Oct 2016 00:13:41 +0800
Subject: [PATCH 014/134] drtio: implement basic writes, errors, fifo levels on
 satellite

---
 artiq/gateware/drtio/rt_packets.py | 154 +++++++++++++++++++++--------
 1 file changed, 113 insertions(+), 41 deletions(-)

diff --git a/artiq/gateware/drtio/rt_packets.py b/artiq/gateware/drtio/rt_packets.py
index be7971316..9dbc825fb 100644
--- a/artiq/gateware/drtio/rt_packets.py
+++ b/artiq/gateware/drtio/rt_packets.py
@@ -48,8 +48,12 @@ def get_s2m_layouts(alignment):
 
 
 error_codes = {
-    "frame_missed": 0,
-    "unknown_type": 1
+    "unknown_type": 0,
+    # The transmitter is normally responsible for avoiding
+    # overflows and underflows. Those error reports are only
+    # for diagnosing internal ARTIQ bugs.
+    "write_overflow": 1,
+    "write_underflow": 2
 }
 
 
@@ -181,6 +185,21 @@ class RTPacketSatellite(Module):
         # I/O Timer interface
         self.tsc_load = Signal()
         self.tsc_value = Signal(64)
+        
+        self.fifo_level_channel = Signal(16)
+        self.fifo_level_update = Signal()
+        self.fifo_level = Signal(24)
+        
+        self.write_stb = Signal()
+        self.write_timestamp = Signal(64)
+        self.write_channel = Signal(16)
+        self.write_address = Signal(16)
+        self.write_data = Signal(256)
+        self.write_overflow = Signal()
+        self.write_overflow_ack = Signal()
+        self.write_underflow = Signal()
+        self.write_underflow_ack = Signal()
+
 
         # # #
 
@@ -200,55 +219,108 @@ class RTPacketSatellite(Module):
             self.tx_rt_data.eq(tx_dp.data)
         ]
 
-        # glue
-        self.comb += [
-            self.tsc_value.eq(rx_dp.packet_as["set_time"].timestamp)
-        ]
-
-        # main control FSM
-        fsm = FSM(reset_state="WAIT_INPUT")
-        self.submodules += fsm
-
-        continuation = Signal()
-        continuation_r = Signal()
-        frame_r_r = Signal()
+        # RX->TX
+        echo_req = Signal()
+        err_set = Signal()
+        err_req = Signal()
+        err_ack = Signal()
+        fifo_level_set = Signal()
+        fifo_level_req = Signal()
+        fifo_level_ack = Signal()
         self.sync += [
-            continuation_r.eq(continuation),
-            frame_r_r.eq(rx_dp.frame_r)
+            If(err_ack, err_req.eq(0)),
+            If(err_set, err_req.eq(1)),
+            If(fifo_level_ack, fifo_level_req.eq(0)),
+            If(fifo_level_set, fifo_level_req.eq(1)),
         ]
-        fsm.act("WAIT_INPUT",
+        err_code = Signal(max=len(error_codes)+1)
+
+        # RX FSM
+        self.comb += [
+            self.tsc_value.eq(
+                rx_dp.packet_as["set_time"].timestamp),
+            self.fifo_level_channel.eq(
+                rx_dp.packet_as["fifo_level_request"].channel),
+            self.write_timestamp.eq(
+                rx_dp.packet_as["write"].timestamp),
+            self.write_channel.eq(
+                rx_dp.packet_as["write"].channel),
+            self.write_address.eq(
+                rx_dp.packet_as["write"].address),
+            self.write_data.eq(
+                rx_dp.packet_as["write"].short_data)
+        ]
+
+        rx_fsm = FSM(reset_state="INPUT")
+        self.submodules += rx_fsm
+
+        rx_fsm.act("INPUT",
             If(rx_dp.frame_r,
-                If(~frame_r_r | continuation_r,
-                    continuation.eq(1),
-                    rx_dp.packet_buffer_load.eq(1),
-                    If(rx_dp.packet_last,
-                        Case(rx_dp.packet_type, {
-                            rx_plm.types["echo_request"]: NextState("ECHO"),
-                            rx_plm.types["set_time"]: NextState("SET_TIME"),
-                            "default": NextState("ERROR_UNKNOWN_TYPE")
-                        })
-                    )
-                ).Else(
-                    NextState("ERROR_FRAME_MISSED")
+                rx_dp.packet_buffer_load.eq(1),
+                If(rx_dp.packet_last,
+                    Case(rx_dp.packet_type, {
+                        rx_plm.types["echo_request"]: echo_req.eq(1),
+                        rx_plm.types["set_time"]: NextState("SET_TIME"),
+                        rx_plm.types["write"]: NextState("WRITE"),
+                        rx_plm.types["fifo_level_request"]: NextState("FIFO_LEVEL"),
+                        "default": [
+                            err_set.eq(1),
+                            NextValue(err_code, error_codes["unknown_type"])]
+                    })
                 )
             )
         )
-        fsm.act("ECHO",
+        rx_fsm.act("SET_TIME",
+            self.tsc_load.eq(1),
+            NextState("INPUT")
+        )
+        rx_fsm.act("WRITE",
+            self.write_stb.eq(1),
+            NextState("INPUT")
+        )
+        rx_fsm.act("FIFO_LEVEL",
+            fifo_level_set.eq(1),
+            self.fifo_level_update.eq(1),
+            NextState("INPUT")
+        )
+
+        # TX FSM
+        tx_fsm = FSM(reset_state="IDLE")
+        self.submodules += tx_fsm
+
+        tx_fsm.act("IDLE",
+            If(echo_req, NextState("ECHO")),
+            If(fifo_level_req, NextState("FIFO_LEVEL")),
+            If(self.write_overflow, NextState("ERROR_WRITE_OVERFLOW")),
+            If(self.write_underflow, NextState("ERROR_WRITE_UNDERFLOW")),
+            If(err_req, NextState("ERROR"))
+        )
+        tx_fsm.act("ECHO",
             tx_dp.send("echo_reply"),
             tx_dp.stb.eq(1),
-            If(tx_dp.done, NextState("WAIT_INPUT"))
+            If(tx_dp.done, NextState("IDLE"))
         )
-        fsm.act("SET_TIME",
-            self.tsc_load.eq(1),
-            NextState("WAIT_INPUT")
-        )
-        fsm.act("ERROR_FRAME_MISSED",
-            tx_dp.send("error", code=error_codes["frame_missed"]),
+        tx_fsm.act("FIFO_LEVEL",
+            fifo_level_ack.eq(1),
+            tx_dp.send("fifo_level_reply", level=self.fifo_level),
             tx_dp.stb.eq(1),
-            If(tx_dp.done, NextState("WAIT_INPUT"))
+            If(tx_dp.done, NextState("IDLE"))
         )
-        fsm.act("ERROR_UNKNOWN_TYPE",
-            tx_dp.send("error", code=error_codes["unknown_type"]),
+        tx_fsm.act("ERROR_WRITE_OVERFLOW",
+            self.write_overflow_ack.eq(1),
+            tx_dp.send("error", code=error_codes["write_overflow"]),
             tx_dp.stb.eq(1),
-            If(tx_dp.done, NextState("WAIT_INPUT"))
+            If(tx_dp.done, NextState("IDLE"))
+        )
+        tx_fsm.act("ERROR_WRITE_UNDERFLOW",
+            self.write_underflow_ack.eq(1),
+            tx_dp.send("error", code=error_codes["write_underflow"]),
+            tx_dp.stb.eq(1),
+            If(tx_dp.done, NextState("IDLE"))
+        )
+        tx_fsm.act("ERROR",
+            err_ack.eq(1),
+            tx_dp.send("error", code=err_code),
+            tx_dp.stb.eq(1),
+            If(tx_dp.done, NextState("IDLE"))
         )

From a40b39e9a2797c07a0764555820d51e7d60fa59b Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Mon, 10 Oct 2016 23:12:12 +0800
Subject: [PATCH 015/134] drtio: structure

---
 artiq/gateware/drtio/__init__.py             |  2 +
 artiq/gateware/drtio/core.py                 | 18 ++++++
 artiq/gateware/drtio/iot.py                  |  9 +++
 artiq/gateware/drtio/rt_packets.py           | 63 ++++++++------------
 artiq/test/gateware/drtio/test_rt_packets.py | 28 +++++----
 5 files changed, 71 insertions(+), 49 deletions(-)
 create mode 100644 artiq/gateware/drtio/core.py
 create mode 100644 artiq/gateware/drtio/iot.py

diff --git a/artiq/gateware/drtio/__init__.py b/artiq/gateware/drtio/__init__.py
index e69de29bb..7e3143e30 100644
--- a/artiq/gateware/drtio/__init__.py
+++ b/artiq/gateware/drtio/__init__.py
@@ -0,0 +1,2 @@
+from artiq.gateware.drtio.core import DRTIOSatellite, DRTIOMaster
+
diff --git a/artiq/gateware/drtio/core.py b/artiq/gateware/drtio/core.py
new file mode 100644
index 000000000..d8bd1d165
--- /dev/null
+++ b/artiq/gateware/drtio/core.py
@@ -0,0 +1,18 @@
+from migen import *
+
+from artiq.gateware.drtio import link_layer, rt_packets, iot
+
+
+class DRTIOSatellite(Module):
+    def __init__(self, transceiver, channels, full_ts_width=63, fine_ts_width=3):
+        self.submodules.link_layer = link_layer.LinkLayer(
+            transceiver.encoder, transceiver.decoders)
+        self.submodules.rt_packets = rt_packets.RTPacketSatellite(
+            self.link_layer)
+        self.submodules.iot = iot.IOT(
+            self.rt_packets, channels, full_ts_width, fine_ts_width)
+
+
+class DRTIOMaster(Module):
+    def __init__(self):
+        pass
diff --git a/artiq/gateware/drtio/iot.py b/artiq/gateware/drtio/iot.py
new file mode 100644
index 000000000..93e008796
--- /dev/null
+++ b/artiq/gateware/drtio/iot.py
@@ -0,0 +1,9 @@
+from migen import *
+from migen.genlib.fifo import SyncFIFOBuffered
+
+from artiq.gateware.rtio import rtlink
+
+
+class IOT(Module):
+    def __init__(self, rt_packets, channels, full_ts_width, fine_ts_width):
+        pass
diff --git a/artiq/gateware/drtio/rt_packets.py b/artiq/gateware/drtio/rt_packets.py
index 9dbc825fb..93e7781eb 100644
--- a/artiq/gateware/drtio/rt_packets.py
+++ b/artiq/gateware/drtio/rt_packets.py
@@ -58,10 +58,8 @@ error_codes = {
 
 
 class ReceiveDatapath(Module):
-    def __init__(self, ws, plm):
-        # inputs
-        self.frame = Signal()
-        self.data = Signal(ws)
+    def __init__(self, frame, data, plm):
+        ws = len(data)
 
         # control
         self.packet_buffer_load = Signal()
@@ -80,11 +78,11 @@ class ReceiveDatapath(Module):
                              for i in range(len(plm.layouts))]
         packet_last_n = Signal(max=max(lastword_per_type)+1)
         self.sync += [
-            self.frame_r.eq(self.frame),
-            self.data_r.eq(self.data),
-            If(self.frame & ~self.frame_r,
-                self.packet_type.eq(self.data[:8]),
-                packet_last_n.eq(Array(lastword_per_type)[self.data[:8]])
+            self.frame_r.eq(frame),
+            self.data_r.eq(data),
+            If(frame & ~self.frame_r,
+                self.packet_type.eq(data[:8]),
+                packet_last_n.eq(Array(lastword_per_type)[data[:8]])
             )
         ]
 
@@ -115,7 +113,9 @@ class ReceiveDatapath(Module):
 
 
 class TransmitDatapath(Module):
-    def __init__(self, ws, plm):
+    def __init__(self, frame, data, plm):
+        ws = len(data)
+        assert ws % 8 == 0
         self.ws = ws
         self.plm = plm
 
@@ -129,25 +129,21 @@ class TransmitDatapath(Module):
         self.stb = Signal()
         self.done = Signal()
 
-        # outputs
-        self.frame = Signal()
-        self.data = Signal(ws)
-
         # # #
 
         packet_buffer_count = Signal(max=w_in_packet+1)
         self.sync += [
             self.done.eq(0),
-            self.frame.eq(0),
+            frame.eq(0),
             packet_buffer_count.eq(0),
 
             If(self.stb & ~self.done,
                 If(packet_buffer_count == self.packet_len,
                     self.done.eq(1)
                 ).Else(
-                    self.frame.eq(1),
+                    frame.eq(1),
                     Case(packet_buffer_count, 
-                         {i: self.data.eq(self.packet_buffer[i*ws:(i+1)*ws])
+                         {i: data.eq(self.packet_buffer[i*ws:(i+1)*ws])
                           for i in range(w_in_packet)}),
                     packet_buffer_count.eq(packet_buffer_count + 1)
                 )
@@ -174,15 +170,7 @@ class TransmitDatapath(Module):
 
 
 class RTPacketSatellite(Module):
-    def __init__(self, nwords):
-        # link layer interface
-        ws = 8*nwords
-        self.rx_rt_frame = Signal()
-        self.rx_rt_data = Signal(ws)
-        self.tx_rt_frame = Signal()
-        self.tx_rt_data = Signal(ws)
-
-        # I/O Timer interface
+    def __init__(self, link_layer):
         self.tsc_load = Signal()
         self.tsc_value = Signal(64)
         
@@ -200,24 +188,20 @@ class RTPacketSatellite(Module):
         self.write_underflow = Signal()
         self.write_underflow_ack = Signal()
 
-
         # # #
 
         # RX/TX datapath
+        assert len(link_layer.tx_rt_data) == len(link_layer.rx_rt_data)
+        assert len(link_layer.tx_rt_data) % 8 == 0
+        ws = len(link_layer.tx_rt_data)
         rx_plm = get_m2s_layouts(ws)
-        rx_dp = ReceiveDatapath(ws, rx_plm)
+        rx_dp = ReceiveDatapath(
+            link_layer.rx_rt_frame, link_layer.rx_rt_data, rx_plm)
         self.submodules += rx_dp
-        self.comb += [
-            rx_dp.frame.eq(self.rx_rt_frame),
-            rx_dp.data.eq(self.rx_rt_data)
-        ]
         tx_plm = get_s2m_layouts(ws)
-        tx_dp = TransmitDatapath(ws, tx_plm)
+        tx_dp = TransmitDatapath(
+            link_layer.tx_rt_frame, link_layer.tx_rt_data, tx_plm)
         self.submodules += tx_dp
-        self.comb += [
-            self.tx_rt_frame.eq(tx_dp.frame),
-            self.tx_rt_data.eq(tx_dp.data)
-        ]
 
         # RX->TX
         echo_req = Signal()
@@ -259,10 +243,13 @@ class RTPacketSatellite(Module):
                 rx_dp.packet_buffer_load.eq(1),
                 If(rx_dp.packet_last,
                     Case(rx_dp.packet_type, {
+                        # echo must have fixed latency, so there is no memory
+                        # mechanism
                         rx_plm.types["echo_request"]: echo_req.eq(1),
                         rx_plm.types["set_time"]: NextState("SET_TIME"),
                         rx_plm.types["write"]: NextState("WRITE"),
-                        rx_plm.types["fifo_level_request"]: NextState("FIFO_LEVEL"),
+                        rx_plm.types["fifo_level_request"]:
+                            NextState("FIFO_LEVEL"),
                         "default": [
                             err_set.eq(1),
                             NextValue(err_code, error_codes["unknown_type"])]
diff --git a/artiq/test/gateware/drtio/test_rt_packets.py b/artiq/test/gateware/drtio/test_rt_packets.py
index 594c37f15..8709c596b 100644
--- a/artiq/test/gateware/drtio/test_rt_packets.py
+++ b/artiq/test/gateware/drtio/test_rt_packets.py
@@ -1,4 +1,5 @@
 import unittest
+from types import SimpleNamespace
 
 from migen import *
 
@@ -6,15 +7,15 @@ from artiq.gateware.drtio.rt_packets import *
 
 
 class PacketInterface:
-    def __init__(self, direction, frame, data):
+    def __init__(self, direction, ws):
         if direction == "m2s":
-            self.plm = get_m2s_layouts(len(data))
+            self.plm = get_m2s_layouts(ws)
         elif direction == "s2m":
-            self.plm = get_s2m_layouts(len(data))
+            self.plm = get_s2m_layouts(ws)
         else:
             raise ValueError
-        self.frame = frame
-        self.data = data
+        self.frame = Signal()
+        self.data = Signal(ws)
 
     def send(self, ty, **kwargs):
         idx = 8
@@ -75,11 +76,17 @@ class PacketInterface:
 
 
 class TestSatellite(unittest.TestCase):
+    def create_dut(self, nwords):
+        pt = PacketInterface("m2s", nwords*8)
+        pr = PacketInterface("s2m", nwords*8)
+        dut = RTPacketSatellite(SimpleNamespace(
+            rx_rt_frame=pt.frame, rx_rt_data=pt.data,
+            tx_rt_frame=pr.frame, tx_rt_data=pr.data))
+        return pt, pr, dut
+
     def test_echo(self):
         for nwords in range(1, 8):
-            dut = RTPacketSatellite(nwords)
-            pt = PacketInterface("m2s", dut.rx_rt_frame, dut.rx_rt_data)
-            pr = PacketInterface("s2m", dut.tx_rt_frame, dut.tx_rt_data)
+            pt, pr, dut = self.create_dut(nwords)
             completed = False
             def send():
                 yield from pt.send("echo_request")
@@ -94,8 +101,7 @@ class TestSatellite(unittest.TestCase):
 
     def test_set_time(self):
         for nwords in range(1, 8):
-            dut = RTPacketSatellite(nwords)
-            pt = PacketInterface("m2s", dut.rx_rt_frame, dut.rx_rt_data)
+            pt, _, dut = self.create_dut(nwords)
             tx_times = [0x12345678aabbccdd, 0x0102030405060708,
                         0xaabbccddeeff1122]
             def send():
@@ -111,5 +117,5 @@ class TestSatellite(unittest.TestCase):
                     if (yield dut.tsc_load):
                         rx_times.append((yield dut.tsc_value))
                     yield
-            run_simulation(dut, [send(), receive()], vcd_name="foo.vcd")
+            run_simulation(dut, [send(), receive()])
             self.assertEqual(tx_times, rx_times)

From 018f6d1b5227114e906bb0bb403bf9363c496bde Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Tue, 11 Oct 2016 17:59:22 +0800
Subject: [PATCH 016/134] drtio: implement basic IOT

---
 artiq/gateware/drtio/core.py |  4 +-
 artiq/gateware/drtio/iot.py  | 71 +++++++++++++++++++++++++++++++++++-
 2 files changed, 71 insertions(+), 4 deletions(-)

diff --git a/artiq/gateware/drtio/core.py b/artiq/gateware/drtio/core.py
index d8bd1d165..1063822d4 100644
--- a/artiq/gateware/drtio/core.py
+++ b/artiq/gateware/drtio/core.py
@@ -4,13 +4,13 @@ from artiq.gateware.drtio import link_layer, rt_packets, iot
 
 
 class DRTIOSatellite(Module):
-    def __init__(self, transceiver, channels, full_ts_width=63, fine_ts_width=3):
+    def __init__(self, transceiver, channels, fine_ts_width=3, full_ts_width=63):
         self.submodules.link_layer = link_layer.LinkLayer(
             transceiver.encoder, transceiver.decoders)
         self.submodules.rt_packets = rt_packets.RTPacketSatellite(
             self.link_layer)
         self.submodules.iot = iot.IOT(
-            self.rt_packets, channels, full_ts_width, fine_ts_width)
+            self.rt_packets, channels, fine_ts_width, full_ts_width)
 
 
 class DRTIOMaster(Module):
diff --git a/artiq/gateware/drtio/iot.py b/artiq/gateware/drtio/iot.py
index 93e008796..5416893fa 100644
--- a/artiq/gateware/drtio/iot.py
+++ b/artiq/gateware/drtio/iot.py
@@ -1,9 +1,76 @@
 from migen import *
 from migen.genlib.fifo import SyncFIFOBuffered
+from migen.genlib.record import *
 
 from artiq.gateware.rtio import rtlink
 
 
 class IOT(Module):
-    def __init__(self, rt_packets, channels, full_ts_width, fine_ts_width):
-        pass
+    def __init__(self, rt_packets, channels, max_fine_ts_width, full_ts_width):
+        tsc = Signal(full_ts_width - max_fine_ts_width)
+        self.sync += \
+            If(rt_packets.tsc_load,
+                tsc.eq(rt_packets.tsc_value)
+            ).Else(
+                tsc.eq(tsc + 1)
+            )
+
+        for n, channel in enumerate(channels):
+            data_width = rtlink.get_data_width(channel.interface)
+            address_width = rtlink.get_address_width(channel.interface)
+            fine_ts_width = rtlink.get_fine_ts_width(channel.interface)
+            assert fine_ts_width <= max_fine_ts_width
+
+            # FIFO
+            ev_layout = []
+            if data_width:
+                ev_layout.append(("data", data_width))
+            if address_width:
+                ev_layout.append(("address", address_width))
+            ev_layout.append(("timestamp", len(tsc) + fine_ts_width))
+
+            fifo = SyncFIFOBuffered(layout_len(ev_layout), channel.ofifo_depth)
+            self.submodules += fifo
+            fifo_in = Record(ev_layout)
+            fifo_out = Record(ev_layout)
+            self.comb += [
+                fifo.din.eq(fifo_in.raw_bits()),
+                fifo_out.raw_bits().eq(fifo.dout)
+            ]
+
+            # FIFO write
+            self.comb += fifo.we.eq(rt_packets.write_stb)
+            self.sync += \
+                If(rt_packets.write_stb,
+                    If(rt_packets.write_overflow_ack,
+                        rt_packets.write_overflow.eq(0)),
+                    If(~fifo.writable, rt_packets.write_overflow.eq(1)),
+                    If(rt_packets.write_underflow_ack,
+                        rt_packets.write_underflow.eq(0)),
+                    If(rt_packets.timestamp[max_fine_ts_width:] < (tsc + 4),
+                        rt_packets.write_underflow.eq(1)
+                    )
+                )
+            if data_width:
+                self.comb += fifo_in.data.eq(rt_packets.write_data)
+            if address_width:
+                self.comb += fifo_in.address.eq(rt_packets.write_address)
+            self.comb += fifo_in.timestamp.eq(
+                rt_packets.timestamp[max_fine_ts_width-fine_ts_width:])
+
+            # FIFO read
+            self.sync += [
+                fifo.re.eq(0),
+                interface.stb.eq(0),
+                If(fifo.readable &
+                   (fifo_out.timestamp[fine_ts_width:] == tsc),
+                    fifo.re.eq(1),
+                    interface.stb.eq(1)
+                )
+            ]
+            if data_width:
+                self.sync += interface.data.eq(fifo_out.data)
+            if address_width:
+                self.sync += interface.address.eq(fifo_out.address)
+            if fine_ts_width:
+                self.sync += interface.fine_ts.eq(fifo_out.timestamp[:fine_ts_width])

From c548a65ec38e6b1ea8334dfbf0edfccc146523f2 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Fri, 14 Oct 2016 00:34:59 +0800
Subject: [PATCH 017/134] drtio: clock domains

---
 artiq/gateware/drtio/core.py       | 25 ++++++++++++++++++++-----
 artiq/gateware/drtio/link_layer.py | 25 ++++++++++++++++++-------
 2 files changed, 38 insertions(+), 12 deletions(-)

diff --git a/artiq/gateware/drtio/core.py b/artiq/gateware/drtio/core.py
index 1063822d4..2c6c08fbe 100644
--- a/artiq/gateware/drtio/core.py
+++ b/artiq/gateware/drtio/core.py
@@ -1,16 +1,31 @@
+from types import SimpleNamespace
+
 from migen import *
 
 from artiq.gateware.drtio import link_layer, rt_packets, iot
 
 
 class DRTIOSatellite(Module):
-    def __init__(self, transceiver, channels, fine_ts_width=3, full_ts_width=63):
+    def __init__(self, transceiver, rx_synchronizer, channels, fine_ts_width=3, full_ts_width=63):
         self.submodules.link_layer = link_layer.LinkLayer(
             transceiver.encoder, transceiver.decoders)
-        self.submodules.rt_packets = rt_packets.RTPacketSatellite(
-            self.link_layer)
-        self.submodules.iot = iot.IOT(
-            self.rt_packets, channels, fine_ts_width, full_ts_width)
+        link_layer_sync = SimpleNamespace(
+            tx_aux_frame=self.link_layer.tx.aux_frame,
+            tx_aux_data=self.link_layer.tx_aux_data,
+            tx_aux_ack=self.link_layer.tx_aux_ack,
+            tx_rt_frame=self.link_layer.tx_rt_frame,
+            tx_rt_data=self.link_layer.tx_rt_data,
+
+            rx_aux_stb=rx_synchronizer.sync(self.link_layer.rx_aux_stb),
+            rx_aux_frame=rx_synchronizer.sync(self.link_layer.rx_aux_frame),
+            rx_aux_data=rx_synchronizer.sync(self.link_layer.rx_aux_data),
+            rx_rt_frame=rx_synchronizer.sync(self.link_layer.rx_rt_frame),
+            rx_rt_data=rx_synchronizer.sync(self.link_layer.rx_rt_data)
+        )
+        self.submodules.rt_packets = ClockDomainsRenamer("rtio")(
+            rt_packets.RTPacketSatellite(link_layer_sync))
+        self.submodules.iot = ClockDomainsRenamer("rtio")(
+            iot.IOT(self.rt_packets, channels, fine_ts_width, full_ts_width))
 
 
 class DRTIOMaster(Module):
diff --git a/artiq/gateware/drtio/link_layer.py b/artiq/gateware/drtio/link_layer.py
index 47bcf9336..46626ab77 100644
--- a/artiq/gateware/drtio/link_layer.py
+++ b/artiq/gateware/drtio/link_layer.py
@@ -3,6 +3,7 @@ from operator import xor, or_
 
 from migen import *
 from migen.genlib.fsm import *
+from migen.genlib.cdc import MultiReg
 
 
 class Scrambler(Module):
@@ -210,24 +211,26 @@ class LinkLayerRX(Module):
 
 class LinkLayer(Module):
     def __init__(self, encoder, decoders):
+        # control signals, in rtio clock domain
         self.reset = Signal()
         self.ready = Signal()
-
         # pulsed to reset receiver, rx_ready must immediately go low
         self.rx_reset = Signal()
         # receiver locked including comma alignment
         self.rx_ready = Signal()
 
-        tx = LinkLayerTX(encoder)
-        rx = LinkLayerRX(decoders)
+        tx = ClockDomainsRenamer("rtio")(LinkLayerTX(encoder))
+        rx = ClockDomainsRenamer("rtio_rx")(LinkLayerRX(decoders))
         self.submodules += tx, rx
 
+        # in rtio clock domain
         self.tx_aux_frame = tx.aux_frame
         self.tx_aux_data = tx.aux_data
         self.tx_aux_ack = tx.aux_ack
         self.tx_rt_frame = tx.rt_frame
         self.tx_rt_data = tx.rt_data
 
+        # in rtio_rx clock domain
         self.rx_aux_stb = rx.aux_stb
         self.rx_aux_frame = rx.aux_frame
         self.rx_aux_data = rx.aux_data
@@ -236,11 +239,19 @@ class LinkLayer(Module):
 
         # # #
 
-        fsm = ResetInserter()(FSM(reset_state="RESET_RX"))
+        fsm = ClockDomainsRenamer("rtio")(
+            ResetInserter()(FSM(reset_state="RESET_RX")))
         self.submodules += fsm
 
         self.comb += fsm.reset.eq(self.reset)
 
+        rx_remote_rx_ready = Signal()
+        rx_link_init = Signal()
+        self.specials += [
+            MultiReg(rx.remote_rx_ready, rx_remote_rx_ready, "rtio"),
+            MultiReg(rx.link_init, rx_link_init, "rtio")
+        ]
+
         fsm.act("RESET_RX",
             tx.link_init.eq(1),
             self.rx_reset.eq(1),
@@ -255,14 +266,14 @@ class LinkLayer(Module):
         fsm.act("WAIT_REMOTE_RX_READY",
             tx.link_init.eq(1),
             tx.signal_rx_ready.eq(1),
-            If(rx.remote_rx_ready,
+            If(rx_remote_rx_ready,
                 NextState("WAIT_REMOTE_LINK_UP")
             )
         )
         fsm.act("WAIT_REMOTE_LINK_UP",
-            If(~rx.link_init, NextState("READY"))
+            If(~rx_link_init, NextState("READY"))
         )
         fsm.act("READY",
-            If(rx.link_init, NextState("RESET_RX")),
+            If(rx_link_init, NextState("RESET_RX")),
             self.ready.eq(1)
         )

From 08e4aa3e3f04f674c3acfe74bff1c5d0c9f32395 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Fri, 14 Oct 2016 00:36:13 +0800
Subject: [PATCH 018/134] drtio: GTX WIP

---
 artiq/gateware/drtio/transceiver/__init__.py  |   0
 .../gateware/drtio/transceiver/gtx_7series.py | 192 ++++++++++++++++++
 .../drtio/transceiver/gtx_7series_init.py     | 168 +++++++++++++++
 3 files changed, 360 insertions(+)
 create mode 100644 artiq/gateware/drtio/transceiver/__init__.py
 create mode 100644 artiq/gateware/drtio/transceiver/gtx_7series.py
 create mode 100644 artiq/gateware/drtio/transceiver/gtx_7series_init.py

diff --git a/artiq/gateware/drtio/transceiver/__init__.py b/artiq/gateware/drtio/transceiver/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/artiq/gateware/drtio/transceiver/gtx_7series.py b/artiq/gateware/drtio/transceiver/gtx_7series.py
new file mode 100644
index 000000000..d90aa0bcc
--- /dev/null
+++ b/artiq/gateware/drtio/transceiver/gtx_7series.py
@@ -0,0 +1,192 @@
+from migen import *
+from migen.genlib.resetsync import AsyncResetSynchronizer
+
+from misoc.cores.code_8b10b import Encoder, Decoder
+
+from artiq.gateware.drtio.transceiver.gtx_7series_init import *
+
+
+class GTX_1000BASE_BX10(Module):
+    def __init__(self, clock_pads, tx_pads, rx_pads, sys_clk_freq):
+        self.submodules.encoder = ClockDomainsRenamer("rtio")(
+            Encoder(2, True))
+        self.decoders = [ClockDomainsRenamer("rtio_rx")(
+            Decoder(True)) for _ in range(2)]
+        self.submodules += self.decoders
+
+        self.rx_reset = Signal()
+        self.rx_ready = Signal()
+
+        # # #
+
+        refclk = Signal()
+        self.specials += Instance("IBUFDS_GTE2",
+            i_CEB=0,
+            i_I=clock_pads.p,
+            i_IB=clock_pads.n,
+            o_O=refclk
+        )
+
+        cplllock = Signal()
+        # TX generates RTIO clock, init must be in system domain
+        tx_init = GTXInit(sys_clk_freq, False)
+        # RX receives restart commands from RTIO domain
+        rx_init = ClockDomainsRenamer("rtio")(
+            GTXInit(62.5e6, True))
+        self.submodules += tx_init, rx_init
+        self.comb += tx_init.cplllock.eq(cplllock), \
+                     rx_init.cplllock.eq(cplllock), \
+                     rx_init.restart.eq(self.rx_reset)
+
+        txoutclk = Signal()
+        txdata = Signal(20)
+        rxoutclk = Signal()
+        rxdata = Signal(20)
+        self.specials += \
+            Instance("GTXE2_CHANNEL",
+                # PMA Attributes
+                p_PMA_RSV=0x00018480,
+                p_PMA_RSV2=0x2050,
+                p_PMA_RSV3=0,
+                p_PMA_RSV4=0,
+                p_RX_BIAS_CFG=0b100,
+                p_RX_CM_TRIM=0b010,
+                p_RX_OS_CFG=0b10000000,
+                p_RX_CLK25_DIV=5,
+                p_TX_CLK25_DIV=5,
+
+                # Power-Down Attributes
+                p_PD_TRANS_TIME_FROM_P2=0x3c,
+                p_PD_TRANS_TIME_NONE_P2=0x3c,
+                p_PD_TRANS_TIME_TO_P2=0x64,
+
+                # CPLL
+                p_CPLL_CFG=0xBC07DC,
+                p_CPLL_FBDIV=4,
+                p_CPLL_FBDIV_45=5,
+                p_CPLL_REFCLK_DIV=1,
+                p_RXOUT_DIV=2,
+                p_TXOUT_DIV=2,
+                o_CPLLLOCK=cplllock,
+                i_CPLLLOCKEN=1,
+                i_CPLLREFCLKSEL=0b001,
+                i_TSTIN=2**20-1,
+                i_GTREFCLK0=refclk,
+
+                # TX clock
+                p_TXBUF_EN="FALSE",
+                p_TX_XCLK_SEL="TXUSR",
+                o_TXOUTCLK=txoutclk,
+                i_TXSYSCLKSEL=0b00,
+                i_TXOUTCLKSEL=0b11,
+
+                # TX Startup/Reset
+                i_GTTXRESET=tx_init.gtXxreset,
+                o_TXRESETDONE=tx_init.Xxresetdone,
+                i_TXDLYSRESET=tx_init.Xxdlysreset,
+                o_TXDLYSRESETDONE=tx_init.Xxdlysresetdone,
+                o_TXPHALIGNDONE=tx_init.Xxphaligndone,
+                i_TXUSERRDY=tx_init.Xxuserrdy,
+
+                # TX data
+                p_TX_DATA_WIDTH=20,
+                p_TX_INT_DATAWIDTH=0,
+                i_TXCHARDISPMODE=Cat(txdata[9], txdata[19]),
+                i_TXCHARDISPVAL=Cat(txdata[8], txdata[18]),
+                i_TXDATA=Cat(txdata[:8], txdata[10:18]),
+                i_TXUSRCLK=ClockSignal("rtio"),
+                i_TXUSRCLK2=ClockSignal("rtio"),
+
+                # TX electrical
+                i_TXBUFDIFFCTRL=0b100,
+                i_TXDIFFCTRL=0b1000,
+
+                # RX Startup/Reset
+                i_GTRXRESET=rx_init.gtXxreset,
+                o_RXRESETDONE=rx_init.Xxresetdone,
+                i_RXDLYSRESET=rx_init.Xxdlysreset,
+                o_RXDLYSRESETDONE=rx_init.Xxdlysresetdone,
+                o_RXPHALIGNDONE=rx_init.Xxphaligndone,
+                i_RXUSERRDY=rx_init.Xxuserrdy,
+
+                # RX AFE
+                p_RX_DFE_XYD_CFG=0,
+                i_RXDFEXYDEN=1,
+                i_RXDFEXYDHOLD=0,
+                i_RXDFEXYDOVRDEN=0,
+                i_RXLPMEN=0,
+
+                # RX clock
+                p_RXBUF_EN="FALSE",
+                p_RX_XCLK_SEL="RXUSR",
+                i_RXDDIEN=1,
+                i_RXSYSCLKSEL=0b00,
+                i_RXOUTCLKSEL=0b010,
+                o_RXOUTCLK=rxoutclk,
+                i_RXUSRCLK=ClockSignal("rtio_rx"),
+                i_RXUSRCLK2=ClockSignal("rtio_rx"),
+                p_RXCDR_CFG=0x03000023FF10100020,
+
+                # RX Clock Correction Attributes
+                p_CLK_CORRECT_USE="FALSE",
+                p_CLK_COR_SEQ_1_1=0b0100000000,
+                p_CLK_COR_SEQ_2_1=0b0100000000,
+                p_CLK_COR_SEQ_1_ENABLE=0b1111,
+                p_CLK_COR_SEQ_2_ENABLE=0b1111,
+
+                # RX data
+                p_RX_DATA_WIDTH=20,
+                p_RX_INT_DATAWIDTH=0,
+                o_RXDISPERR=Cat(rxdata[9], rxdata[19]),
+                o_RXCHARISK=Cat(rxdata[8], rxdata[18]),
+                o_RXDATA=Cat(rxdata[:8], rxdata[10:18]),
+
+                # Pads
+                i_GTXRXP=rx_pads.p,
+                i_GTXRXN=rx_pads.n,
+                o_GTXTXP=tx_pads.p,
+                o_GTXTXN=tx_pads.n,
+            )
+
+        self.clock_domains.cd_rtio = ClockDomain()
+        self.specials += [
+            Instance("BUFG", i_I=txoutclk, o_O=self.cd_rtio.clk),
+            AsyncResetSynchronizer(self.cd_rtio, ~tx_init.done)
+        ]
+        self.clock_domains.cd_rtio_rx = ClockDomain()
+        self.specials += [
+            Instance("BUFG", i_I=rxoutclk, o_O=self.cd_rtio_rx.clk),
+            AsyncResetSynchronizer(self.cd_rtio_rx, ~rx_init.done)
+        ]
+
+        self.comb += [
+            txdata.eq(Cat(self.encoder.output[0], self.encoder.output[1])),
+            self.decoders[0].input.eq(rxdata[:10]),
+            self.decoders[1].input.eq(rxdata[10:])
+        ]
+        # TODO: clock aligner, reset/ready
+
+
+class RXSynchronizer(Module):
+    """Delays the data received in the rtio_rx by a configurable amount
+    so that it meets s/h in the rtio domain, and recapture it in the rtio
+    domain. This has fixed latency.
+
+    Since Xilinx doesn't provide decent on-chip delay lines, we implement the
+    delay with MMCM that provides a clock and a finely configurable phase, used
+    to resample the data.
+
+    The phase has to be determined either empirically or by making sense of the
+    Xilinx scriptures (when existent) and should be constant for a given design
+    placement.
+    """
+    def __init__(self):
+        self.cd_rtio_delayed = ClockDomain()
+        # TODO
+
+    def sync(self, signal):
+        delayed = Signal.like(signal, related=signal)
+        synchronized = Signal.like(signal, related=signal)
+        self.sync.rtio_delayed += delayed.eq(signal)
+        self.sync.rtio += synchronized.eq(delayed)
+        return synchronized
diff --git a/artiq/gateware/drtio/transceiver/gtx_7series_init.py b/artiq/gateware/drtio/transceiver/gtx_7series_init.py
new file mode 100644
index 000000000..fa01c88f3
--- /dev/null
+++ b/artiq/gateware/drtio/transceiver/gtx_7series_init.py
@@ -0,0 +1,168 @@
+from math import ceil
+
+from migen import *
+from migen.genlib.cdc import MultiReg, PulseSynchronizer
+from migen.genlib.misc import WaitTimer
+from migen.genlib.fsm import FSM
+
+
+class GTXInit(Module):
+    # Based on LiteSATA by Enjoy-Digital
+    def __init__(self, sys_clk_freq, rx):
+        self.done = Signal()
+        self.restart = Signal()
+
+        # GTX signals
+        self.cplllock = Signal()
+        self.gtXxreset = Signal()
+        self.Xxresetdone = Signal()
+        self.Xxdlysreset = Signal()
+        self.Xxdlysresetdone = Signal()
+        self.Xxphaligndone = Signal()
+        self.Xxuserrdy = Signal()
+
+        # # #
+
+        # Double-latch transceiver asynch outputs
+        cplllock = Signal()
+        Xxresetdone = Signal()
+        Xxdlysresetdone = Signal()
+        Xxphaligndone = Signal()
+        self.specials += [
+            MultiReg(self.cplllock, cplllock),
+            MultiReg(self.Xxresetdone, Xxresetdone),
+            MultiReg(self.Xxdlysresetdone, Xxdlysresetdone),
+            MultiReg(self.Xxphaligndone, Xxphaligndone),
+        ]
+
+        # Deglitch FSM outputs driving transceiver asynch inputs
+        gtXxreset = Signal()
+        Xxdlysreset = Signal()
+        Xxuserrdy = Signal()
+        self.sync += [
+            self.gtXxreset.eq(gtXxreset),
+            self.Xxdlysreset.eq(Xxdlysreset),
+            self.Xxuserrdy.eq(Xxuserrdy)
+        ]
+
+        # After configuration, transceiver resets have to stay low for
+        # at least 500ns (see AR43482)
+        startup_cycles = ceil(500*sys_clk_freq/1000000000)
+        startup_timer = WaitTimer(startup_cycles)
+        self.submodules += startup_timer
+
+        startup_fsm = FSM(reset_state="INITIAL")
+        self.submodules += startup_fsm
+
+        if rx:
+            cdr_stable_timer = WaitTimer(1024)
+            self.submodules += cdr_stable_timer
+
+        Xxphaligndone_r = Signal(reset=1)
+        Xxphaligndone_rising = Signal()
+        self.sync += Xxphaligndone_r.eq(Xxphaligndone)
+        self.comb += Xxphaligndone_rising.eq(Xxphaligndone & ~Xxphaligndone_r)
+
+        startup_fsm.act("INITIAL",
+            startup_timer.wait.eq(1),
+            If(startup_timer.done, NextState("RESET_GTX"))
+        )
+        startup_fsm.act("RESET_GTX",
+            gtXxreset.eq(1),
+            NextState("WAIT_CPLL")
+        )
+        startup_fsm.act("WAIT_CPLL",
+            gtXxreset.eq(1),
+            If(cplllock, NextState("RELEASE_RESET"))
+        )
+        # Release GTX reset and wait for GTX resetdone
+        # (from UG476, GTX is reset on falling edge
+        # of gttxreset)
+        if rx:
+            startup_fsm.act("RELEASE_RESET",
+                Xxuserrdy.eq(1),
+                cdr_stable_timer.wait.eq(1),
+                If(Xxresetdone & cdr_stable_timer.done, NextState("ALIGN"))
+            )
+        else:
+            startup_fsm.act("RELEASE_RESET",
+                Xxuserrdy.eq(1),
+                If(Xxresetdone, NextState("ALIGN"))
+            )
+        # Start delay alignment (pulse)
+        startup_fsm.act("ALIGN",
+            Xxuserrdy.eq(1),
+            Xxdlysreset.eq(1),
+            NextState("WAIT_ALIGN")
+        )
+        # Wait for delay alignment
+        startup_fsm.act("WAIT_ALIGN",
+            Xxuserrdy.eq(1),
+            If(Xxdlysresetdone, NextState("WAIT_FIRST_ALIGN_DONE"))
+        )
+        # Wait 2 rising edges of rxphaligndone
+        # (from UG476 in buffer bypass config)
+        startup_fsm.act("WAIT_FIRST_ALIGN_DONE",
+            Xxuserrdy.eq(1),
+            If(Xxphaligndone_rising, NextState("WAIT_SECOND_ALIGN_DONE"))
+        )
+        startup_fsm.act("WAIT_SECOND_ALIGN_DONE",
+            Xxuserrdy.eq(1),
+            If(Xxphaligndone_rising, NextState("READY"))
+        )
+        startup_fsm.act("READY",
+            Xxuserrdy.eq(1),
+            self.done.eq(1),
+            If(self.restart, NextState("RESET_GTX"))
+        )
+
+
+# Changes the phase of the transceiver RX clock to align the comma to
+# the MSBs of RXDATA, fixing the latency.
+#
+# This is implemented by repeatedly resetting the transceiver until it
+# gives out the correct phase. Each reset gives a random phase.
+#
+# If Xilinx had designed the GTX transceiver correctly, RXSLIDE_MODE=PMA
+# would achieve this faster and in a cleaner way. But:
+#  * the phase jumps are of 2 UI at every second RXSLIDE pulse, instead
+#    of 1 UI at every pulse. It is unclear what the latency becomes.
+#  * RXSLIDE_MODE=PMA cannot be used with the RX buffer bypassed.
+# Those design flaws make RXSLIDE_MODE=PMA yet another broken and useless
+# transceiver "feature".
+class BruteforceClockAligner(Module):
+    def __init__(self, comma, sys_clk_freq, check_period=6e-3):
+        self.rxdata = Signal(20)
+        self.restart = Signal()
+
+        check_max_val = ceil(check_period*sys_clk_freq)
+        check_counter = Signal(max=check_max_val+1)
+        check = Signal()
+        self.sync += [
+            check.eq(0),
+            If(check_counter == 0,
+                check.eq(1),
+                check_counter.eq(check_max_val)
+            ).Else(
+                check_counter.eq(check_counter-1)
+            )
+        ]
+
+        comma_n = ~comma & 0b1111111111
+        comma_seen_rxclk = Signal()
+        comma_seen = Signal()
+        self.specials += MultiReg(comma_seen_rxclk, comma_seen)
+        comma_seen_reset = PulseSynchronizer("sys", "rx")
+        self.submodules += comma_seen_reset
+        self.sync.rx += \
+            If(comma_seen_reset.o,
+                comma_seen_rxclk.eq(0)
+            ).Elif((self.rxdata[:10] == comma) | (self.rxdata[:10] == comma_n),
+                comma_seen_rxclk.eq(1)
+            )
+
+        self.comb += \
+            If(check,
+                If(~comma_seen, self.restart.eq(1)),
+                comma_seen_reset.i.eq(1)
+            )

From 03d3a85e75de14f982b0fc49037d51b78d3c3186 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Sat, 15 Oct 2016 18:36:27 +0800
Subject: [PATCH 019/134] drtio: RX clock alignment and ready

---
 artiq/gateware/drtio/core.py                  |  6 +++
 .../gateware/drtio/transceiver/gtx_7series.py | 10 ++++-
 .../drtio/transceiver/gtx_7series_init.py     | 38 ++++++++++++++-----
 3 files changed, 44 insertions(+), 10 deletions(-)

diff --git a/artiq/gateware/drtio/core.py b/artiq/gateware/drtio/core.py
index 2c6c08fbe..98bb47960 100644
--- a/artiq/gateware/drtio/core.py
+++ b/artiq/gateware/drtio/core.py
@@ -9,6 +9,11 @@ class DRTIOSatellite(Module):
     def __init__(self, transceiver, rx_synchronizer, channels, fine_ts_width=3, full_ts_width=63):
         self.submodules.link_layer = link_layer.LinkLayer(
             transceiver.encoder, transceiver.decoders)
+        self.comb += [
+            transceiver.rx_reset.eq(self.link_layer.rx_reset),
+            self.link_layer.rx_ready.eq(transceiver.rx_ready)
+        ]
+
         link_layer_sync = SimpleNamespace(
             tx_aux_frame=self.link_layer.tx.aux_frame,
             tx_aux_data=self.link_layer.tx_aux_data,
@@ -24,6 +29,7 @@ class DRTIOSatellite(Module):
         )
         self.submodules.rt_packets = ClockDomainsRenamer("rtio")(
             rt_packets.RTPacketSatellite(link_layer_sync))
+
         self.submodules.iot = ClockDomainsRenamer("rtio")(
             iot.IOT(self.rt_packets, channels, fine_ts_width, full_ts_width))
 
diff --git a/artiq/gateware/drtio/transceiver/gtx_7series.py b/artiq/gateware/drtio/transceiver/gtx_7series.py
index d90aa0bcc..1c34359e3 100644
--- a/artiq/gateware/drtio/transceiver/gtx_7series.py
+++ b/artiq/gateware/drtio/transceiver/gtx_7series.py
@@ -164,7 +164,15 @@ class GTX_1000BASE_BX10(Module):
             self.decoders[0].input.eq(rxdata[:10]),
             self.decoders[1].input.eq(rxdata[10:])
         ]
-        # TODO: clock aligner, reset/ready
+
+        clock_aligner = BruteforceClockAligner(0b0011111000, 62.5e6)
+        self.submodules += clock_aligner
+        self.comb += [
+            clock_aligner.rxdata.eq(rxdata),
+            rx_init.restart.eq(clock_aligner.restart),
+            clock_aligner.reset.eq(self.rx_reset),
+            self.rx_ready.eq(clock_aligner.ready)
+        ]
 
 
 class RXSynchronizer(Module):
diff --git a/artiq/gateware/drtio/transceiver/gtx_7series_init.py b/artiq/gateware/drtio/transceiver/gtx_7series_init.py
index fa01c88f3..1c6fc5a9f 100644
--- a/artiq/gateware/drtio/transceiver/gtx_7series_init.py
+++ b/artiq/gateware/drtio/transceiver/gtx_7series_init.py
@@ -131,20 +131,25 @@ class GTXInit(Module):
 # Those design flaws make RXSLIDE_MODE=PMA yet another broken and useless
 # transceiver "feature".
 class BruteforceClockAligner(Module):
-    def __init__(self, comma, sys_clk_freq, check_period=6e-3):
+    def __init__(self, comma, rtio_clk_freq, check_period=6e-3, ready_time=50e-3):
         self.rxdata = Signal(20)
         self.restart = Signal()
 
-        check_max_val = ceil(check_period*sys_clk_freq)
+        self.reset = Signal()
+        self.ready = Signal()
+
+        check_max_val = ceil(check_period*rtio_clk_freq)
         check_counter = Signal(max=check_max_val+1)
         check = Signal()
-        self.sync += [
+        self.sync.rtio += [
             check.eq(0),
-            If(check_counter == 0,
-                check.eq(1),
-                check_counter.eq(check_max_val)
-            ).Else(
-                check_counter.eq(check_counter-1)
+            If(~self.ready,
+                If(check_counter == 0,
+                    check.eq(1),
+                    check_counter.eq(check_max_val)
+                ).Else(
+                    check_counter.eq(check_counter-1)
+                )
             )
         ]
 
@@ -154,7 +159,7 @@ class BruteforceClockAligner(Module):
         self.specials += MultiReg(comma_seen_rxclk, comma_seen)
         comma_seen_reset = PulseSynchronizer("sys", "rx")
         self.submodules += comma_seen_reset
-        self.sync.rx += \
+        self.sync.rtio_rx += \
             If(comma_seen_reset.o,
                 comma_seen_rxclk.eq(0)
             ).Elif((self.rxdata[:10] == comma) | (self.rxdata[:10] == comma_n),
@@ -166,3 +171,18 @@ class BruteforceClockAligner(Module):
                 If(~comma_seen, self.restart.eq(1)),
                 comma_seen_reset.i.eq(1)
             )
+
+        ready_counts = ceil(ready_time/check_period)
+        assert ready_counts > 1
+        ready_counter = Signal(max=ready_counts+1, reset=ready_counts)
+        self.sync.rtio += [
+            If(check,
+                If(comma_seen,
+                    If(ready_counter != 0, ready_counter.eq(ready_counter-1))
+                ).Else(
+                    ready_counter.eq(ready_counts)
+                )
+            ),
+            If(self.reset, ready_counter.eq(ready_counts))
+        ]
+        self.comb += self.ready.eq(ready_counter == 0)

From d3b274fc4d03b181957b3e50cdcff1d44155a36c Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Sun, 16 Oct 2016 17:40:58 +0800
Subject: [PATCH 020/134] drtio: synchronizer MMCM

---
 .../gateware/drtio/transceiver/gtx_7series.py | 42 +++++++++++++++++--
 1 file changed, 38 insertions(+), 4 deletions(-)

diff --git a/artiq/gateware/drtio/transceiver/gtx_7series.py b/artiq/gateware/drtio/transceiver/gtx_7series.py
index 1c34359e3..063a86a4e 100644
--- a/artiq/gateware/drtio/transceiver/gtx_7series.py
+++ b/artiq/gateware/drtio/transceiver/gtx_7series.py
@@ -2,6 +2,7 @@ from migen import *
 from migen.genlib.resetsync import AsyncResetSynchronizer
 
 from misoc.cores.code_8b10b import Encoder, Decoder
+from misoc.interconnect.csr import *
 
 from artiq.gateware.drtio.transceiver.gtx_7series_init import *
 
@@ -175,7 +176,7 @@ class GTX_1000BASE_BX10(Module):
         ]
 
 
-class RXSynchronizer(Module):
+class RXSynchronizer(Module, AutoCSR):
     """Delays the data received in the rtio_rx by a configurable amount
     so that it meets s/h in the rtio domain, and recapture it in the rtio
     domain. This has fixed latency.
@@ -188,9 +189,42 @@ class RXSynchronizer(Module):
     Xilinx scriptures (when existent) and should be constant for a given design
     placement.
     """
-    def __init__(self):
-        self.cd_rtio_delayed = ClockDomain()
-        # TODO
+    def __init__(self, rtio_clk_freq):
+        self.phase_shift = CSR()
+        self.phase_shift_done = CSRStatus()
+
+        self.cd_rtio_delayed = ClockDomain(reset_less=True)
+
+        mmcm_output = Signal()
+        mmcm_fb = Signal()
+        # maximize VCO frequency to maximize phase shift resolution
+        mmcm_mult = 1200e9//rtio_clk_freq
+        self.specials += [
+            Instance("MMCME2_ADV",
+                p_CLKIN1_PERIOD=1e9/rtio_clk_freq,
+                i_CLKIN1=ClockSignal("rtio_rx"),
+                i_CLKINSEL=1,  # yes, 1=CLKIN1 0=CLKIN2
+
+                p_CLKFBOUT_MULT_F=mmcm_mult,
+                p_CLKOUT0_DIVIDE_F=mmcm_mult,
+                p_DIVCLK_DIVIDE=1,
+
+                # According to Xilinx, there is no guarantee of input/output
+                # phase relationship when using internal feedback. We assume
+                # here that the input/ouput skew is constant to save BUFGs.
+                o_CLKFBOUT=mmcm_fb,
+                i_CLKFBIN=mmcm_fb,
+
+                p_CLKOUT0_USE_FINE_PS="TRUE",
+                o_CLKOUT0=mmcm_output,
+
+                i_PSCLK=ClockSignal(),
+                i_PSEN=self.phase_shift.re,
+                i_PSINCDEC=self.phase_shift.r,
+                o_PSDONE=self.phase_shift_done.status,
+            ),
+            Instance("BUFR", i_I=mmcm_output, o_O=self.cd_rtio_delayed.clk)
+        ]
 
     def sync(self, signal):
         delayed = Signal.like(signal, related=signal)

From cce29e8b831046a9497962c60fa82e4a29d45425 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Mon, 17 Oct 2016 14:06:35 +0800
Subject: [PATCH 021/134] gateware/spi: fix import

---
 artiq/gateware/spi.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/artiq/gateware/spi.py b/artiq/gateware/spi.py
index 17ab523ad..23e52915f 100644
--- a/artiq/gateware/spi.py
+++ b/artiq/gateware/spi.py
@@ -2,7 +2,7 @@ from itertools import product
 
 from migen import *
 from misoc.interconnect import wishbone
-from misoc.cores.spi.core import SPIMachine
+from misoc.cores.spi import SPIMachine
 
 
 class SPIMaster(Module):

From 9752ffe3d156b9e9a81c7f86358ee2dd6b47472e Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Mon, 17 Oct 2016 19:23:08 +0800
Subject: [PATCH 022/134] drtio: various fixes

---
 artiq/gateware/drtio/core.py                  | 22 ++++++++++++++-----
 artiq/gateware/drtio/iot.py                   | 11 +++++-----
 .../gateware/drtio/transceiver/gtx_7series.py | 10 +++++----
 .../drtio/transceiver/gtx_7series_init.py     |  2 +-
 4 files changed, 29 insertions(+), 16 deletions(-)

diff --git a/artiq/gateware/drtio/core.py b/artiq/gateware/drtio/core.py
index 98bb47960..1388e187e 100644
--- a/artiq/gateware/drtio/core.py
+++ b/artiq/gateware/drtio/core.py
@@ -15,17 +15,17 @@ class DRTIOSatellite(Module):
         ]
 
         link_layer_sync = SimpleNamespace(
-            tx_aux_frame=self.link_layer.tx.aux_frame,
+            tx_aux_frame=self.link_layer.tx_aux_frame,
             tx_aux_data=self.link_layer.tx_aux_data,
             tx_aux_ack=self.link_layer.tx_aux_ack,
             tx_rt_frame=self.link_layer.tx_rt_frame,
             tx_rt_data=self.link_layer.tx_rt_data,
 
-            rx_aux_stb=rx_synchronizer.sync(self.link_layer.rx_aux_stb),
-            rx_aux_frame=rx_synchronizer.sync(self.link_layer.rx_aux_frame),
-            rx_aux_data=rx_synchronizer.sync(self.link_layer.rx_aux_data),
-            rx_rt_frame=rx_synchronizer.sync(self.link_layer.rx_rt_frame),
-            rx_rt_data=rx_synchronizer.sync(self.link_layer.rx_rt_data)
+            rx_aux_stb=rx_synchronizer.resync(self.link_layer.rx_aux_stb),
+            rx_aux_frame=rx_synchronizer.resync(self.link_layer.rx_aux_frame),
+            rx_aux_data=rx_synchronizer.resync(self.link_layer.rx_aux_data),
+            rx_rt_frame=rx_synchronizer.resync(self.link_layer.rx_rt_frame),
+            rx_rt_data=rx_synchronizer.resync(self.link_layer.rx_rt_data)
         )
         self.submodules.rt_packets = ClockDomainsRenamer("rtio")(
             rt_packets.RTPacketSatellite(link_layer_sync))
@@ -33,6 +33,16 @@ class DRTIOSatellite(Module):
         self.submodules.iot = ClockDomainsRenamer("rtio")(
             iot.IOT(self.rt_packets, channels, fine_ts_width, full_ts_width))
 
+        # TODO: remote resets
+        self.clock_domains.cd_rio = ClockDomain()
+        self.clock_domains.cd_rio_phy = ClockDomain()
+        self.comb += [
+            self.cd_rio.clk.eq(ClockSignal("rtio")),
+            self.cd_rio.rst.eq(ResetSignal("rtio")),
+            self.cd_rio_phy.clk.eq(ClockSignal("rtio")),
+            self.cd_rio_phy.rst.eq(ResetSignal("rtio")),
+        ]
+
 
 class DRTIOMaster(Module):
     def __init__(self):
diff --git a/artiq/gateware/drtio/iot.py b/artiq/gateware/drtio/iot.py
index 5416893fa..f8e07c299 100644
--- a/artiq/gateware/drtio/iot.py
+++ b/artiq/gateware/drtio/iot.py
@@ -16,9 +16,10 @@ class IOT(Module):
             )
 
         for n, channel in enumerate(channels):
-            data_width = rtlink.get_data_width(channel.interface)
-            address_width = rtlink.get_address_width(channel.interface)
-            fine_ts_width = rtlink.get_fine_ts_width(channel.interface)
+            interface = channel.interface.o
+            data_width = rtlink.get_data_width(interface)
+            address_width = rtlink.get_address_width(interface)
+            fine_ts_width = rtlink.get_fine_ts_width(interface)
             assert fine_ts_width <= max_fine_ts_width
 
             # FIFO
@@ -47,7 +48,7 @@ class IOT(Module):
                     If(~fifo.writable, rt_packets.write_overflow.eq(1)),
                     If(rt_packets.write_underflow_ack,
                         rt_packets.write_underflow.eq(0)),
-                    If(rt_packets.timestamp[max_fine_ts_width:] < (tsc + 4),
+                    If(rt_packets.write_timestamp[max_fine_ts_width:] < (tsc + 4),
                         rt_packets.write_underflow.eq(1)
                     )
                 )
@@ -56,7 +57,7 @@ class IOT(Module):
             if address_width:
                 self.comb += fifo_in.address.eq(rt_packets.write_address)
             self.comb += fifo_in.timestamp.eq(
-                rt_packets.timestamp[max_fine_ts_width-fine_ts_width:])
+                rt_packets.write_timestamp[max_fine_ts_width-fine_ts_width:])
 
             # FIFO read
             self.sync += [
diff --git a/artiq/gateware/drtio/transceiver/gtx_7series.py b/artiq/gateware/drtio/transceiver/gtx_7series.py
index 063a86a4e..b7d720b4b 100644
--- a/artiq/gateware/drtio/transceiver/gtx_7series.py
+++ b/artiq/gateware/drtio/transceiver/gtx_7series.py
@@ -8,6 +8,8 @@ from artiq.gateware.drtio.transceiver.gtx_7series_init import *
 
 
 class GTX_1000BASE_BX10(Module):
+    rtio_clk_freq = 62.5e6
+
     def __init__(self, clock_pads, tx_pads, rx_pads, sys_clk_freq):
         self.submodules.encoder = ClockDomainsRenamer("rtio")(
             Encoder(2, True))
@@ -33,7 +35,7 @@ class GTX_1000BASE_BX10(Module):
         tx_init = GTXInit(sys_clk_freq, False)
         # RX receives restart commands from RTIO domain
         rx_init = ClockDomainsRenamer("rtio")(
-            GTXInit(62.5e6, True))
+            GTXInit(self.rtio_clk_freq, True))
         self.submodules += tx_init, rx_init
         self.comb += tx_init.cplllock.eq(cplllock), \
                      rx_init.cplllock.eq(cplllock), \
@@ -166,7 +168,7 @@ class GTX_1000BASE_BX10(Module):
             self.decoders[1].input.eq(rxdata[10:])
         ]
 
-        clock_aligner = BruteforceClockAligner(0b0011111000, 62.5e6)
+        clock_aligner = BruteforceClockAligner(0b0011111000, self.rtio_clk_freq)
         self.submodules += clock_aligner
         self.comb += [
             clock_aligner.rxdata.eq(rxdata),
@@ -193,7 +195,7 @@ class RXSynchronizer(Module, AutoCSR):
         self.phase_shift = CSR()
         self.phase_shift_done = CSRStatus()
 
-        self.cd_rtio_delayed = ClockDomain(reset_less=True)
+        self.clock_domains.cd_rtio_delayed = ClockDomain(reset_less=True)
 
         mmcm_output = Signal()
         mmcm_fb = Signal()
@@ -226,7 +228,7 @@ class RXSynchronizer(Module, AutoCSR):
             Instance("BUFR", i_I=mmcm_output, o_O=self.cd_rtio_delayed.clk)
         ]
 
-    def sync(self, signal):
+    def resync(self, signal):
         delayed = Signal.like(signal, related=signal)
         synchronized = Signal.like(signal, related=signal)
         self.sync.rtio_delayed += delayed.eq(signal)
diff --git a/artiq/gateware/drtio/transceiver/gtx_7series_init.py b/artiq/gateware/drtio/transceiver/gtx_7series_init.py
index 1c6fc5a9f..c46e3eef0 100644
--- a/artiq/gateware/drtio/transceiver/gtx_7series_init.py
+++ b/artiq/gateware/drtio/transceiver/gtx_7series_init.py
@@ -157,7 +157,7 @@ class BruteforceClockAligner(Module):
         comma_seen_rxclk = Signal()
         comma_seen = Signal()
         self.specials += MultiReg(comma_seen_rxclk, comma_seen)
-        comma_seen_reset = PulseSynchronizer("sys", "rx")
+        comma_seen_reset = PulseSynchronizer("rtio", "rtio_rx")
         self.submodules += comma_seen_reset
         self.sync.rtio_rx += \
             If(comma_seen_reset.o,

From e7dbed3b02920ea4f17ba7a768e0864b5e7a0d23 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Mon, 17 Oct 2016 19:23:45 +0800
Subject: [PATCH 023/134] gateware: KC705 satellite target

---
 .../gateware/targets/kc705_drtio_satellite.py | 163 ++++++++++++++++++
 1 file changed, 163 insertions(+)
 create mode 100644 artiq/gateware/targets/kc705_drtio_satellite.py

diff --git a/artiq/gateware/targets/kc705_drtio_satellite.py b/artiq/gateware/targets/kc705_drtio_satellite.py
new file mode 100644
index 000000000..dbb4b8958
--- /dev/null
+++ b/artiq/gateware/targets/kc705_drtio_satellite.py
@@ -0,0 +1,163 @@
+import argparse
+
+from migen import *
+from migen.build.platforms import kc705
+
+from misoc.cores.i2c import *
+from misoc.cores.sequencer import *
+
+from artiq.gateware import rtio
+from artiq.gateware.rtio.phy import ttl_simple
+from artiq.gateware.drtio.transceiver import gtx_7series
+from artiq.gateware.drtio import DRTIOSatellite
+
+
+# TODO: program Si5324 free-run mode, with automatic switching
+def get_i2c_program(sys_clk_freq):
+    # NOTE: the logical parameters DO NOT MAP to physical values written
+    # into registers. They have to be mapped; see the datasheet.
+    # DSPLLsim reports the logical parameters in the design summary, not
+    # the physical register values (but those are present separately).
+    N1_HS  = 0   # 4
+    NC1_LS = 19  # 20
+    N2_HS  = 1   # 5
+    N2_LS  = 511 # 512
+    N31    = 31  # 32
+
+    i2c_sequence = [
+        # PCA9548: select channel 7
+        [(0x74 << 1), 1 << 7],
+        # Si5324: configure
+        [(0x68 << 1), 2,   0b0010 | (4 << 4)], # BWSEL=4
+        [(0x68 << 1), 3,   0b0101 | 0x10],     # SQ_ICAL=1
+        [(0x68 << 1), 6,            0x07],     # SFOUT1_REG=b111
+        [(0x68 << 1), 25,  (N1_HS  << 5 ) & 0xff],
+        [(0x68 << 1), 31,  (NC1_LS >> 16) & 0xff],
+        [(0x68 << 1), 32,  (NC1_LS >> 8 ) & 0xff],
+        [(0x68 << 1), 33,  (NC1_LS)       & 0xff],
+        [(0x68 << 1), 40,  (N2_HS  << 5 ) & 0xff |
+                           (N2_LS  >> 16) & 0xff],
+        [(0x68 << 1), 41,  (N2_LS  >> 8 ) & 0xff],
+        [(0x68 << 1), 42,  (N2_LS)        & 0xff],
+        [(0x68 << 1), 43,  (N31    >> 16) & 0xff],
+        [(0x68 << 1), 44,  (N31    >> 8)  & 0xff],
+        [(0x68 << 1), 45,  (N31)          & 0xff],
+        [(0x68 << 1), 137,          0x01],     # FASTLOCK=1
+        [(0x68 << 1), 136,          0x40],     # ICAL=1
+    ]
+
+    program = [
+        InstWrite(I2C_CONFIG_ADDR, int(sys_clk_freq/1e3)),
+    ]
+    for subseq in i2c_sequence:
+        program += [
+            InstWrite(I2C_XFER_ADDR, I2C_START),
+            InstWait(I2C_XFER_ADDR, I2C_IDLE),
+        ]
+        for octet in subseq:
+            program += [
+                InstWrite(I2C_XFER_ADDR, I2C_WRITE | octet),
+                InstWait(I2C_XFER_ADDR, I2C_IDLE),
+            ]
+        program += [
+            InstWrite(I2C_XFER_ADDR, I2C_STOP),
+            InstWait(I2C_XFER_ADDR, I2C_IDLE),
+        ]
+    program += [
+        InstEnd(),
+    ]
+    return program
+
+
+class Si5324ResetClock(Module):
+    def __init__(self, platform, sys_clk_freq):
+        self.si5324_not_ready = Signal(reset=1)
+
+        # minimum reset pulse 1us
+        reset_done = Signal()
+        si5324_rst_n = platform.request("si5324").rst_n
+        reset_val = int(sys_clk_freq*1.1e-6)
+        reset_ctr = Signal(max=reset_val+1, reset=reset_val)
+        self.sync += \
+            If(reset_ctr != 0,
+                reset_ctr.eq(reset_ctr - 1)
+            ).Else(
+                si5324_rst_n.eq(1),
+                reset_done.eq(1)
+            )
+        # 10ms after reset to microprocessor access ready
+        ready_val = int(sys_clk_freq*11e-3)
+        ready_ctr = Signal(max=ready_val+1, reset=ready_val)
+        self.sync += \
+            If(reset_done,
+                If(ready_ctr != 0,
+                    ready_ctr.eq(ready_ctr - 1)
+                ).Else(
+                    self.si5324_not_ready.eq(0)
+                )
+            )
+
+        si5324_clkin = platform.request("si5324_clkin")
+        self.specials += \
+            Instance("OBUFDS",
+                i_I=ClockSignal("rtio_rx"),
+                o_O=si5324_clkin.p, o_OB=si5324_clkin.n
+            )
+
+
+class Satellite(Module):
+    def __init__(self, toolchain="vivado"):
+        self.platform = platform = kc705.Platform(toolchain=toolchain)
+
+        rtio_channels = []
+        for i in range(8):
+            phy = ttl_simple.Output(platform.request("user_led", i))
+            self.submodules += phy
+            rtio_channels.append(rtio.Channel.from_phy(phy))
+        for sma in "user_sma_gpio_p", "user_sma_gpio_n":
+            phy = ttl_simple.Inout(platform.request(sma))
+            self.submodules += phy
+            rtio_channels.append(rtio.Channel.from_phy(phy))
+
+        sys_clock_pads = platform.request("clk156")
+        self.clock_domains.cd_sys = ClockDomain(reset_less=True)
+        self.specials += Instance("IBUFGDS",
+            i_I=sys_clock_pads.p, i_IB=sys_clock_pads.n,
+            o_O=self.cd_sys.clk)
+        sys_clk_freq = 156000000
+
+        i2c_master = I2CMaster(platform.request("i2c"))
+        sequencer = ResetInserter()(Sequencer(get_i2c_program(sys_clk_freq)))
+        si5324_reset_clock = Si5324ResetClock(platform, sys_clk_freq)
+        self.submodules += i2c_master, sequencer, si5324_reset_clock
+        self.comb += [
+            sequencer.bus.connect(i2c_master.bus),
+            sequencer.reset.eq(si5324_reset_clock.si5324_not_ready)
+        ]
+
+        self.comb += platform.request("sfp_tx_disable_n").eq(1)
+        self.submodules.transceiver = gtx_7series.GTX_1000BASE_BX10(
+            clock_pads=platform.request("si5324_clkout"),
+            tx_pads=platform.request("sfp_tx"),
+            rx_pads=platform.request("sfp_rx"),
+            sys_clk_freq=sys_clk_freq)
+        self.submodules.rx_synchronizer = gtx_7series.RXSynchronizer(
+            self.transceiver.rtio_clk_freq)
+        self.submodules.drtio = DRTIOSatellite(
+            self.transceiver, self.rx_synchronizer, rtio_channels)
+
+    def build(self, *args, **kwargs):
+        self.platform.build(self, *args, **kwargs)
+
+
+def main():
+    parser = argparse.ArgumentParser(description="KC705 DRTIO satellite")
+    parser.add_argument("--toolchain", default="vivado",
+                        help="FPGA toolchain to use: ise, vivado")
+    args = parser.parse_args()
+
+    top = Satellite(args.toolchain)
+    top.build()
+
+if __name__ == "__main__":
+    main()

From 71480c4d15c03c6f748f5bc4e1a1adbd9feb6311 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Tue, 18 Oct 2016 17:28:03 +0800
Subject: [PATCH 024/134] drtio: fix mmcm_mult

---
 artiq/gateware/drtio/transceiver/gtx_7series.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/artiq/gateware/drtio/transceiver/gtx_7series.py b/artiq/gateware/drtio/transceiver/gtx_7series.py
index b7d720b4b..70e54b8b0 100644
--- a/artiq/gateware/drtio/transceiver/gtx_7series.py
+++ b/artiq/gateware/drtio/transceiver/gtx_7series.py
@@ -200,7 +200,7 @@ class RXSynchronizer(Module, AutoCSR):
         mmcm_output = Signal()
         mmcm_fb = Signal()
         # maximize VCO frequency to maximize phase shift resolution
-        mmcm_mult = 1200e9//rtio_clk_freq
+        mmcm_mult = 1200e6//rtio_clk_freq
         self.specials += [
             Instance("MMCME2_ADV",
                 p_CLKIN1_PERIOD=1e9/rtio_clk_freq,

From 9790c5d9edd84ca1a767ebe20f521ba9ee7a7522 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Wed, 19 Oct 2016 18:04:03 +0800
Subject: [PATCH 025/134] drtio/iot: FIFO level

---
 artiq/gateware/drtio/iot.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/artiq/gateware/drtio/iot.py b/artiq/gateware/drtio/iot.py
index f8e07c299..c05e6b1a7 100644
--- a/artiq/gateware/drtio/iot.py
+++ b/artiq/gateware/drtio/iot.py
@@ -39,6 +39,12 @@ class IOT(Module):
                 fifo_out.raw_bits().eq(fifo.dout)
             ]
 
+            # FIFO level
+            self.sync += \
+                If(rt_packets.fifo_level_update &
+                   (rt_packets.fifo_level_channel == n),
+                    rt_packets.fifo_level.eq(fifo.level))
+
             # FIFO write
             self.comb += fifo.we.eq(rt_packets.write_stb)
             self.sync += \

From 6a88229e6ac5fc36ab73b11312604ec8e711bcdd Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Thu, 20 Oct 2016 23:37:59 +0800
Subject: [PATCH 026/134] drtio: CrossDomainRequest

---
 artiq/gateware/drtio/rt_packets.py           | 33 +++++++++++++
 artiq/test/gateware/drtio/test_rt_packets.py | 49 ++++++++++++++++++++
 2 files changed, 82 insertions(+)

diff --git a/artiq/gateware/drtio/rt_packets.py b/artiq/gateware/drtio/rt_packets.py
index 93e7781eb..b243f4065 100644
--- a/artiq/gateware/drtio/rt_packets.py
+++ b/artiq/gateware/drtio/rt_packets.py
@@ -2,6 +2,8 @@ from types import SimpleNamespace
 
 from migen import *
 from migen.genlib.fsm import *
+from migen.genlib.fifo import AsyncFIFO
+from migen.genlib.cdc import PulseSynchronizer, NoRetiming
 
 
 def layout_len(l):
@@ -311,3 +313,34 @@ class RTPacketSatellite(Module):
             tx_dp.stb.eq(1),
             If(tx_dp.done, NextState("IDLE"))
         )
+
+
+class _CrossDomainRequest(Module):
+    def __init__(self, domain,
+                 req_stb, req_ack, req_data,
+                 srv_stb, srv_ack, srv_data):
+        dsync = getattr(self.sync, domain)
+
+        request = PulseSynchronizer("sys", domain)
+        reply = PulseSynchronizer(domain, "sys")
+        self.submodules += request, reply
+
+        ongoing = Signal()
+        self.comb += request.i.eq(~ongoing & req_stb)
+        self.sync += [
+            req_ack.eq(reply.o),
+            If(req_stb, ongoing.eq(1)),
+            If(req_ack, ongoing.eq(0))
+        ]
+        if req_data is not None:
+            srv_data_r = Signal.like(srv_data)
+            dsync += If(srv_stb & srv_ack, srv_data_r.eq(srv_data))
+            self.specials += NoRetiming(srv_data_r)
+            self.sync += If(reply.o, req_data.eq(srv_data_r))
+        dsync += [
+            If(request.o, srv_stb.eq(1)),
+            If(srv_ack, srv_stb.eq(0))
+        ]
+        self.comb += reply.i.eq(srv_stb & srv_ack)
+
+
diff --git a/artiq/test/gateware/drtio/test_rt_packets.py b/artiq/test/gateware/drtio/test_rt_packets.py
index 8709c596b..b1168d424 100644
--- a/artiq/test/gateware/drtio/test_rt_packets.py
+++ b/artiq/test/gateware/drtio/test_rt_packets.py
@@ -1,9 +1,11 @@
 import unittest
 from types import SimpleNamespace
+import random
 
 from migen import *
 
 from artiq.gateware.drtio.rt_packets import *
+from artiq.gateware.drtio.rt_packets import _CrossDomainRequest
 
 
 class PacketInterface:
@@ -119,3 +121,50 @@ class TestSatellite(unittest.TestCase):
                     yield
             run_simulation(dut, [send(), receive()])
             self.assertEqual(tx_times, rx_times)
+
+
+class TestCrossDomainRequest(unittest.TestCase):
+    def test_cross_domain_request(self):
+        for sys_freq in 3, 6, 11:
+            for srv_freq in 3, 6, 11:
+                prng = random.Random(1)
+
+                req_stb = Signal()
+                req_ack = Signal()
+                req_data = Signal(8)
+                srv_stb = Signal()
+                srv_ack = Signal()
+                srv_data = Signal(8)
+                test_seq = [93, 92, 19, 39, 91, 30, 12, 91, 38, 42]
+                received_seq = []
+
+                def requester():
+                    for i in range(len(test_seq)):
+                        yield req_stb.eq(1)
+                        yield
+                        while not (yield req_ack):
+                            yield
+                        received_seq.append((yield req_data))
+                        yield req_stb.eq(0)
+                        for j in range(prng.randrange(0, 10)):
+                            yield
+
+                def server():
+                    for data in test_seq:
+                        while not (yield srv_stb):
+                            yield
+                        for j in range(prng.randrange(0, 10)):
+                            yield
+                        yield srv_data.eq(data)
+                        yield srv_ack.eq(1)
+                        yield
+                        yield srv_ack.eq(0)
+                        yield
+
+                dut = _CrossDomainRequest("srv",
+                         req_stb, req_ack, req_data,
+                         srv_stb, srv_ack, srv_data)
+                run_simulation(dut,
+                    {"sys": requester(), "srv": server()},
+                    {"sys": sys_freq, "srv": srv_freq})
+                self.assertEqual(test_seq, received_seq)

From c71c4c89e0230900c0dd4d8e64cce80c76b95144 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Fri, 21 Oct 2016 22:44:47 +0800
Subject: [PATCH 027/134] drtio: change data direction in _CrossDomainRequest

---
 artiq/gateware/drtio/rt_packets.py           |  9 +++++----
 artiq/test/gateware/drtio/test_rt_packets.py | 11 +++++------
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/artiq/gateware/drtio/rt_packets.py b/artiq/gateware/drtio/rt_packets.py
index b243f4065..05a8ae1cf 100644
--- a/artiq/gateware/drtio/rt_packets.py
+++ b/artiq/gateware/drtio/rt_packets.py
@@ -333,14 +333,15 @@ class _CrossDomainRequest(Module):
             If(req_ack, ongoing.eq(0))
         ]
         if req_data is not None:
-            srv_data_r = Signal.like(srv_data)
-            dsync += If(srv_stb & srv_ack, srv_data_r.eq(srv_data))
-            self.specials += NoRetiming(srv_data_r)
-            self.sync += If(reply.o, req_data.eq(srv_data_r))
+            req_data_r = Signal.like(req_data)
+            self.specials += NoRetiming(req_data_r)
+            self.sync += If(req_stb, req_data_r.eq(req_data))
         dsync += [
             If(request.o, srv_stb.eq(1)),
             If(srv_ack, srv_stb.eq(0))
         ]
+        if req_data is not None:
+            dsync += If(request.o, srv_data.eq(req_data_r))
         self.comb += reply.i.eq(srv_stb & srv_ack)
 
 
diff --git a/artiq/test/gateware/drtio/test_rt_packets.py b/artiq/test/gateware/drtio/test_rt_packets.py
index b1168d424..693176392 100644
--- a/artiq/test/gateware/drtio/test_rt_packets.py
+++ b/artiq/test/gateware/drtio/test_rt_packets.py
@@ -125,10 +125,9 @@ class TestSatellite(unittest.TestCase):
 
 class TestCrossDomainRequest(unittest.TestCase):
     def test_cross_domain_request(self):
+        prng = random.Random(1)
         for sys_freq in 3, 6, 11:
             for srv_freq in 3, 6, 11:
-                prng = random.Random(1)
-
                 req_stb = Signal()
                 req_ack = Signal()
                 req_data = Signal(8)
@@ -139,23 +138,23 @@ class TestCrossDomainRequest(unittest.TestCase):
                 received_seq = []
 
                 def requester():
-                    for i in range(len(test_seq)):
+                    for data in test_seq:
+                        yield req_data.eq(data)
                         yield req_stb.eq(1)
                         yield
                         while not (yield req_ack):
                             yield
-                        received_seq.append((yield req_data))
                         yield req_stb.eq(0)
                         for j in range(prng.randrange(0, 10)):
                             yield
 
                 def server():
-                    for data in test_seq:
+                    for i in range(len(test_seq)):
                         while not (yield srv_stb):
                             yield
+                        received_seq.append((yield srv_data))
                         for j in range(prng.randrange(0, 10)):
                             yield
-                        yield srv_data.eq(data)
                         yield srv_ack.eq(1)
                         yield
                         yield srv_ack.eq(0)

From 1e313afe642c6628924d60aad22fb28c73c336f9 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Fri, 21 Oct 2016 22:45:45 +0800
Subject: [PATCH 028/134] drtio: CrossDomainNotification

---
 artiq/gateware/drtio/rt_packets.py           | 20 +++++++++
 artiq/test/gateware/drtio/test_rt_packets.py | 47 +++++++++++++++++++-
 2 files changed, 65 insertions(+), 2 deletions(-)

diff --git a/artiq/gateware/drtio/rt_packets.py b/artiq/gateware/drtio/rt_packets.py
index 05a8ae1cf..51123654f 100644
--- a/artiq/gateware/drtio/rt_packets.py
+++ b/artiq/gateware/drtio/rt_packets.py
@@ -345,3 +345,23 @@ class _CrossDomainRequest(Module):
         self.comb += reply.i.eq(srv_stb & srv_ack)
 
 
+class _CrossDomainNotification(Module):
+    def __init__(self, domain,
+                 emi_stb, emi_data,
+                 rec_stb, rec_ack, rec_data):
+        emi_data_r = Signal.like(emi_data)
+        self.specials += NoRetiming(emi_data_r)
+        dsync = getattr(self.sync, domain)
+        dsync += If(emi_stb, emi_data_r.eq(emi_data))
+
+        ps = PulseSynchronizer(domain, "sys")
+        self.submodules += ps
+        self.comb += ps.i.eq(emi_stb)
+        self.sync += [
+            If(rec_ack, rec_stb.eq(0)),
+            If(ps.o,
+                rec_data.eq(emi_data_r),
+                rec_stb.eq(1)
+            )
+        ]
+
diff --git a/artiq/test/gateware/drtio/test_rt_packets.py b/artiq/test/gateware/drtio/test_rt_packets.py
index 693176392..72f7531e0 100644
--- a/artiq/test/gateware/drtio/test_rt_packets.py
+++ b/artiq/test/gateware/drtio/test_rt_packets.py
@@ -5,7 +5,8 @@ import random
 from migen import *
 
 from artiq.gateware.drtio.rt_packets import *
-from artiq.gateware.drtio.rt_packets import _CrossDomainRequest
+from artiq.gateware.drtio.rt_packets import (_CrossDomainRequest,
+                                             _CrossDomainNotification)
 
 
 class PacketInterface:
@@ -123,7 +124,7 @@ class TestSatellite(unittest.TestCase):
             self.assertEqual(tx_times, rx_times)
 
 
-class TestCrossDomainRequest(unittest.TestCase):
+class TestCDC(unittest.TestCase):
     def test_cross_domain_request(self):
         prng = random.Random(1)
         for sys_freq in 3, 6, 11:
@@ -167,3 +168,45 @@ class TestCrossDomainRequest(unittest.TestCase):
                     {"sys": requester(), "srv": server()},
                     {"sys": sys_freq, "srv": srv_freq})
                 self.assertEqual(test_seq, received_seq)
+
+    def test_cross_domain_notification(self):
+        prng = random.Random(1)
+
+        emi_stb = Signal()
+        emi_data = Signal(8)
+        rec_stb = Signal()
+        rec_ack = Signal()
+        rec_data = Signal(8)
+
+        test_seq = [23, 12, 8, 3, 28]
+        received_seq = []
+
+        def emitter():
+            for data in test_seq:
+                yield emi_stb.eq(1)
+                yield emi_data.eq(data)
+                yield
+                yield emi_stb.eq(0)
+                yield
+                for j in range(prng.randrange(0, 3)):
+                    yield
+
+        def receiver():
+            for i in range(len(test_seq)):
+                while not (yield rec_stb):
+                    yield
+                received_seq.append((yield rec_data))
+                yield rec_ack.eq(1)
+                yield
+                yield rec_ack.eq(0)
+                yield
+                for j in range(prng.randrange(0, 3)):
+                    yield
+
+        dut = _CrossDomainNotification("emi",
+            emi_stb, emi_data,
+            rec_stb, rec_ack, rec_data)
+        run_simulation(dut,
+            {"emi": emitter(), "sys": receiver()},
+            {"emi": 13, "sys": 3})
+        self.assertEqual(test_seq, received_seq)

From 3b4a40401ad7dbbe294e0fae677c76c3efde50ad Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Fri, 21 Oct 2016 22:46:14 +0800
Subject: [PATCH 029/134] drtio: RTPacketMaster TX (WIP)

---
 artiq/gateware/drtio/core.py       |  10 +-
 artiq/gateware/drtio/rt_packets.py | 147 ++++++++++++++++++++++++++++-
 2 files changed, 153 insertions(+), 4 deletions(-)

diff --git a/artiq/gateware/drtio/core.py b/artiq/gateware/drtio/core.py
index 1388e187e..6c805c97c 100644
--- a/artiq/gateware/drtio/core.py
+++ b/artiq/gateware/drtio/core.py
@@ -45,5 +45,11 @@ class DRTIOSatellite(Module):
 
 
 class DRTIOMaster(Module):
-    def __init__(self):
-        pass
+    def __init__(self, transceiver):
+        self.submodules.link_layer = link_layer.LinkLayer(
+            transceiver.encoder, transceiver.decoders)
+        self.comb += [
+            transceiver.rx_reset.eq(self.link_layer.rx_reset),
+            self.link_layer.rx_ready.eq(transceiver.rx_ready)
+        ]
+        self.submodules.rt_packets = rt_packets.RTPacketMaster(self.link_layer)
diff --git a/artiq/gateware/drtio/rt_packets.py b/artiq/gateware/drtio/rt_packets.py
index 51123654f..4d7037aa7 100644
--- a/artiq/gateware/drtio/rt_packets.py
+++ b/artiq/gateware/drtio/rt_packets.py
@@ -45,7 +45,7 @@ def get_s2m_layouts(alignment):
     plm = PacketLayoutManager(alignment)
     plm.add_type("error", ("code", 8))
     plm.add_type("echo_reply")
-    plm.add_type("fifo_level_reply", ("level", 24))
+    plm.add_type("fifo_level_reply", ("level", 16))
     return plm
 
 
@@ -178,7 +178,7 @@ class RTPacketSatellite(Module):
         
         self.fifo_level_channel = Signal(16)
         self.fifo_level_update = Signal()
-        self.fifo_level = Signal(24)
+        self.fifo_level = Signal(16)
         
         self.write_stb = Signal()
         self.write_timestamp = Signal(64)
@@ -365,3 +365,146 @@ class _CrossDomainNotification(Module):
             )
         ]
 
+
+class RTPacketMaster(Module):
+    def __init__(self, link_layer, write_fifo_depth=4):
+        # all interface signals in sys domain unless otherwise specified
+
+        # write interface, optimized for throughput
+        self.write_stb = Signal()
+        self.write_ack = Signal()
+        self.write_timestamp = Signal(64)
+        self.write_channel = Signal(16)
+        self.write_address = Signal(16)
+        self.write_data = Signal(256)
+
+        # fifo level interface
+        # write with timestamp[48:] == 0xffff to make a fifo level request
+        # (level requests have to be ordered wrt writes)
+        self.fifo_level_not = Signal()
+        self.fifo_level_not_ack = Signal()
+        self.fifo_level = Signal(16)
+
+        # echo interface
+        self.echo_stb = Signal()
+        self.echo_ack = Signal()
+        self.echo_sent_now = Signal()  # in rtio domain
+        self.echo_received_now = Signal()  # in rtio_rx domain
+
+        # set_time interface
+        self.set_time_stb = Signal()
+        self.set_time_ack = Signal()
+        self.tsc_value = Signal(64)  # in rtio domain, must be valid all time
+
+        # errors
+        self.error_not = Signal()
+        self.error_not_ack = Signal()
+        self.error_code = Signal(8)
+
+        # # #
+
+        # CDC
+        wfifo = AsyncFIFO(64+16+16+256, write_fifo_depth)
+        self.submodules += wfifo
+        write_timestamp = Signal(64)
+        write_channel = Signal(16)
+        write_address = Signal(16)
+        write_data = Signal(256)
+        self.comb += [
+            wfifo.we.eq(self.write_stb),
+            self.write_ack.eq(wfifo.writable),
+            wfifo.din.eq(Cat(self.write_timestamp, self.write_channel,
+                             self.write_address, self.write_data)),
+            Cat(write_timestamp, write_channel,
+                write_address, write_data).eq(fifo.dout)
+        ]
+
+        fifo_level_not = Signal()
+        fifo_level = Signal(16)
+        self.submodules += _CrossDomainNotification("rtio_rx",
+            fifo_level_not, fifo_level,
+            self.fifo_level_not, self.fifo_level_not_ack, self.fifo_level)
+
+        set_time_stb = Signal()
+        set_time_ack = Signal()
+        self.submodules += _CrossDomainRequest("rtio",
+            self.set_time_stb, self.set_time_ack, None,
+            set_time_stb, set_time_ack, None)
+
+        echo_stb = Signal()
+        echo_ack = Signal()
+        self.submodules += _CrossDomainRequest("rtio",
+            self.echo_stb, self.echo_ack, None,
+            echo_stb, echo_ack, None)
+
+        error_not = Signal()
+        error_code = Signal(8)
+        self.submodules += _CrossDomainNotification("rtio_rx",
+            error_not, error_code,
+            self.error_not, self.error_not_ack, self.error_code)
+
+        # RX/TX datapath
+        assert len(link_layer.tx_rt_data) == len(link_layer.rx_rt_data)
+        assert len(link_layer.tx_rt_data) % 8 == 0
+        ws = len(link_layer.tx_rt_data)
+        tx_plm = get_m2s_layouts(ws)
+        tx_dp = ClockDomainsRenamer("rtio")(TransmitDatapath(
+            link_layer.tx_rt_frame, link_layer.tx_rt_data, tx_plm))
+        self.submodules += tx_dp
+        rx_plm = get_s2m_layouts(ws)
+        rx_dp = ClockDomainsRenamer("rtio_rx")(ReceiveDatapath(
+            link_layer.rx_rt_frame, link_layer.rx_rt_data, rx_plm))
+        self.submodules += rx_dp
+
+        # TX FSM
+        tx_fsm = ClockDomainsRenamer("rtio")(FSM(reset_state="IDLE_WRITE"))
+        self.submodules += tx_fsm
+
+        echo_sent_now = Signal()
+        self.sync.rtio += self.echo_sent_now.eq(echo_sent_now)
+        tsc_value = Signal(64)
+        tsc_value_load = Signal()
+        self.sync.rtio += If(tsc_value_load, tsc_value.eq(self.tsc_value))
+
+        tx_fsm.act("IDLE_WRITE",
+            tx_dp.send("write",
+                timestamp=write_timestamp,
+                channel=write_channel,
+                address=write_address,
+                short_data=write_data),
+            If(wfifo.readable,
+                If(write_timestamp[48:] == 0xffff,
+                    NextState("FIFO_LEVEL")
+                ).Else(
+                    tx_dp.stb.eq(1),
+                    wfifo.re.eq(tx_dp.done)
+                )
+            ).Else(
+                If(echo_stb,
+                    echo_sent_now.eq(1),
+                    NextState("ECHO")
+                ).Elif(set_time_stb,
+                    tsc_value_load.eq(1),
+                    NextState("SET_TIME")
+                )
+            )
+        )
+        tx_fsm.act("FIFO_LEVEL",
+            tx_dp.send("fifo_level_request", channel=write_channel),
+            tx_dp.stb.eq(1),
+            If(tx_dp.done, NextState("IDLE_WRITE"))
+        )
+        tx_fsm.act("ECHO",
+            tx_dp.send("echo_request"),
+            tx_dp.stb.eq(1),
+            If(tx_dp.done, NextState("IDLE_WRITE"))
+        )
+        tx_fsm.act("SET_TIME",
+            tx_dp.send("set_time", timestamp=tsc_value),
+            tx_dp.stb.eq(1),
+            If(tx_dp.done, NextState("IDLE_WRITE"))
+        )
+
+        # RX FSM
+        rx_fsm = ClockDomainsRenamer("rtio_rx")(FSM())
+        self.submodules += rx_fsm

From 67c19ab178ddc663e95da436106edb3557e8a408 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Sat, 22 Oct 2016 01:04:14 +0800
Subject: [PATCH 030/134] drtio: RTPacketMaster RX, untested

---
 artiq/gateware/drtio/rt_packets.py | 32 +++++++++++++++++++++++++++++-
 1 file changed, 31 insertions(+), 1 deletion(-)

diff --git a/artiq/gateware/drtio/rt_packets.py b/artiq/gateware/drtio/rt_packets.py
index 4d7037aa7..37570ce3b 100644
--- a/artiq/gateware/drtio/rt_packets.py
+++ b/artiq/gateware/drtio/rt_packets.py
@@ -506,5 +506,35 @@ class RTPacketMaster(Module):
         )
 
         # RX FSM
-        rx_fsm = ClockDomainsRenamer("rtio_rx")(FSM())
+        rx_fsm = ClockDomainsRenamer("rtio_rx")(FSM(reset_state="INPUT"))
         self.submodules += rx_fsm
+
+        echo_reply_now = Signal()
+        self.sync.rtio_rx += self.echo_reply_now.eq(echo_reply_now)
+
+        rx_fsm.act("INPUT",
+            If(rx_dp.frame_r,
+                rx_dp.packet_buffer_load.eq(1),
+                If(rx_dp.packet_last,
+                    Case(rx_dp.packet_type, {
+                        rx_plm.types["error"]: NextState("ERROR"),
+                        rx_plm.types["echo_reply"]: echo_reply_now.eq(1),
+                        rx_plm.types["fifo_level_reply"]: NextState("FIFO_LEVEL"),
+                        "default": [
+                            error_not.eq(1),
+                            error_code.eq(error_codes["unknown_type"])
+                        ]
+                    })
+                )
+            )
+        )
+        rx_fsm.act("ERROR",
+            error_not.eq(1),
+            error_code.eq(rx_dp.packet_as["error"].code),
+            NextState("INPUT")
+        )
+        rx_fsm.act("FIFO_LEVEL",
+            fifo_level_not.eq(1),
+            fifo_level.eq(rx_dp.packet_as["fifo_level_reply"].level),
+            NextState("INPUT")
+        )

From 449d1c4dc6a66a934ea765017d553dff02748bdc Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Sat, 22 Oct 2016 13:03:10 +0800
Subject: [PATCH 031/134] rtio: export CDC modules

---
 artiq/gateware/rtio/cdc.py  | 68 +++++++++++++++++++++++++++++++++++
 artiq/gateware/rtio/core.py | 71 +++----------------------------------
 2 files changed, 72 insertions(+), 67 deletions(-)
 create mode 100644 artiq/gateware/rtio/cdc.py

diff --git a/artiq/gateware/rtio/cdc.py b/artiq/gateware/rtio/cdc.py
new file mode 100644
index 000000000..7b31eedcd
--- /dev/null
+++ b/artiq/gateware/rtio/cdc.py
@@ -0,0 +1,68 @@
+from migen import *
+from migen.genlib.cdc import *
+
+
+__all__ = ["GrayCodeTransfer", "RTIOCounter", "BlindTransfer"]
+
+
+# note: transfer is in rtio/sys domains and not affected by the reset CSRs
+class GrayCodeTransfer(Module):
+    def __init__(self, width):
+        self.i = Signal(width)  # in rtio domain
+        self.o = Signal(width)  # in sys domain
+
+        # # #
+
+        # convert to Gray code
+        value_gray_rtio = Signal(width)
+        self.sync.rtio += value_gray_rtio.eq(self.i ^ self.i[1:])
+        # transfer to system clock domain
+        value_gray_sys = Signal(width)
+        self.specials += [
+            NoRetiming(value_gray_rtio),
+            MultiReg(value_gray_rtio, value_gray_sys)
+        ]
+        # convert back to binary
+        value_sys = Signal(width)
+        self.comb += value_sys[-1].eq(value_gray_sys[-1])
+        for i in reversed(range(width-1)):
+            self.comb += value_sys[i].eq(value_sys[i+1] ^ value_gray_sys[i])
+        self.sync += self.o.eq(value_sys)
+
+
+class RTIOCounter(Module):
+    def __init__(self, width):
+        self.width = width
+        # Timestamp counter in RTIO domain
+        self.value_rtio = Signal(width)
+        # Timestamp counter resynchronized to sys domain
+        # Lags behind value_rtio, monotonic and glitch-free
+        self.value_sys = Signal(width)
+
+        # # #
+
+        # note: counter is in rtio domain and never affected by the reset CSRs
+        self.sync.rtio += self.value_rtio.eq(self.value_rtio + 1)
+        gt = GrayCodeTransfer(width)
+        self.submodules += gt
+        self.comb += gt.i.eq(self.value_rtio), self.value_sys.eq(gt.o)
+
+
+class BlindTransfer(Module):
+    def __init__(self):
+        self.i = Signal()
+        self.o = Signal()
+
+        ps = PulseSynchronizer("rio", "rsys")
+        ps_ack = PulseSynchronizer("rsys", "rio")
+        self.submodules += ps, ps_ack
+        blind = Signal()
+        self.sync.rio += [
+            If(self.i, blind.eq(1)),
+            If(ps_ack.o, blind.eq(0))
+        ]
+        self.comb += [
+            ps.i.eq(self.i & ~blind),
+            ps_ack.i.eq(ps.o),
+            self.o.eq(ps.o)
+        ]
diff --git a/artiq/gateware/rtio/core.py b/artiq/gateware/rtio/core.py
index 69babebac..a64bace11 100644
--- a/artiq/gateware/rtio/core.py
+++ b/artiq/gateware/rtio/core.py
@@ -3,75 +3,12 @@ from operator import and_
 
 from migen import *
 from migen.genlib.record import Record
-from migen.genlib.cdc import *
 from migen.genlib.fifo import AsyncFIFO
 from migen.genlib.resetsync import AsyncResetSynchronizer
 from misoc.interconnect.csr import *
 
 from artiq.gateware.rtio import rtlink
-
-
-# note: transfer is in rtio/sys domains and not affected by the reset CSRs
-class _GrayCodeTransfer(Module):
-    def __init__(self, width):
-        self.i = Signal(width)  # in rtio domain
-        self.o = Signal(width)  # in sys domain
-
-        # # #
-
-        # convert to Gray code
-        value_gray_rtio = Signal(width)
-        self.sync.rtio += value_gray_rtio.eq(self.i ^ self.i[1:])
-        # transfer to system clock domain
-        value_gray_sys = Signal(width)
-        self.specials += [
-            NoRetiming(value_gray_rtio),
-            MultiReg(value_gray_rtio, value_gray_sys)
-        ]
-        # convert back to binary
-        value_sys = Signal(width)
-        self.comb += value_sys[-1].eq(value_gray_sys[-1])
-        for i in reversed(range(width-1)):
-            self.comb += value_sys[i].eq(value_sys[i+1] ^ value_gray_sys[i])
-        self.sync += self.o.eq(value_sys)
-
-
-class _RTIOCounter(Module):
-    def __init__(self, width):
-        self.width = width
-        # Timestamp counter in RTIO domain
-        self.value_rtio = Signal(width)
-        # Timestamp counter resynchronized to sys domain
-        # Lags behind value_rtio, monotonic and glitch-free
-        self.value_sys = Signal(width)
-
-        # # #
-
-        # note: counter is in rtio domain and never affected by the reset CSRs
-        self.sync.rtio += self.value_rtio.eq(self.value_rtio + 1)
-        gt = _GrayCodeTransfer(width)
-        self.submodules += gt
-        self.comb += gt.i.eq(self.value_rtio), self.value_sys.eq(gt.o)
-
-
-class _BlindTransfer(Module):
-    def __init__(self):
-        self.i = Signal()
-        self.o = Signal()
-
-        ps = PulseSynchronizer("rio", "rsys")
-        ps_ack = PulseSynchronizer("rsys", "rio")
-        self.submodules += ps, ps_ack
-        blind = Signal()
-        self.sync.rio += [
-            If(self.i, blind.eq(1)),
-            If(ps_ack.o, blind.eq(0))
-        ]
-        self.comb += [
-            ps.i.eq(self.i & ~blind),
-            ps_ack.i.eq(ps.o),
-            self.o.eq(ps.o)
-        ]
+from artiq.gateware.rtio.cdc import *
 
 
 # CHOOSING A GUARD TIME
@@ -229,7 +166,7 @@ class _OutputManager(Module):
             interface.stb.eq(dout_stb & dout_ack)
         ]
 
-        busy_transfer = _BlindTransfer()
+        busy_transfer = BlindTransfer()
         self.submodules += busy_transfer
         self.comb += [
             busy_transfer.i.eq(interface.stb & interface.busy),
@@ -291,7 +228,7 @@ class _InputManager(Module):
             fifo.re.eq(self.re)
         ]
 
-        overflow_transfer = _BlindTransfer()
+        overflow_transfer = BlindTransfer()
         self.submodules += overflow_transfer
         self.comb += [
             overflow_transfer.i.eq(fifo.we & ~fifo.writable),
@@ -398,7 +335,7 @@ class RTIO(Module):
                                                        allow_reset_less=True))
 
         # Managers
-        self.submodules.counter = _RTIOCounter(full_ts_width - fine_ts_width)
+        self.submodules.counter = RTIOCounter(full_ts_width - fine_ts_width)
 
         i_datas, i_timestamps = [], []
         o_statuses, i_statuses = [], []

From aa8e211735594ea087aef4531fbac9c9d5f2d6eb Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Sat, 22 Oct 2016 13:03:35 +0800
Subject: [PATCH 032/134] drtio/rt_packets: fix

---
 artiq/gateware/drtio/rt_packets.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/artiq/gateware/drtio/rt_packets.py b/artiq/gateware/drtio/rt_packets.py
index 37570ce3b..a8602d0eb 100644
--- a/artiq/gateware/drtio/rt_packets.py
+++ b/artiq/gateware/drtio/rt_packets.py
@@ -509,8 +509,8 @@ class RTPacketMaster(Module):
         rx_fsm = ClockDomainsRenamer("rtio_rx")(FSM(reset_state="INPUT"))
         self.submodules += rx_fsm
 
-        echo_reply_now = Signal()
-        self.sync.rtio_rx += self.echo_reply_now.eq(echo_reply_now)
+        echo_received_now = Signal()
+        self.sync.rtio_rx += self.echo_received_now.eq(echo_received_now)
 
         rx_fsm.act("INPUT",
             If(rx_dp.frame_r,
@@ -518,7 +518,7 @@ class RTPacketMaster(Module):
                 If(rx_dp.packet_last,
                     Case(rx_dp.packet_type, {
                         rx_plm.types["error"]: NextState("ERROR"),
-                        rx_plm.types["echo_reply"]: echo_reply_now.eq(1),
+                        rx_plm.types["echo_reply"]: echo_received_now.eq(1),
                         rx_plm.types["fifo_level_reply"]: NextState("FIFO_LEVEL"),
                         "default": [
                             error_not.eq(1),

From 83bec06226a0089db45c72a374dcd4a8d1b5cae9 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Mon, 24 Oct 2016 15:59:12 +0800
Subject: [PATCH 033/134] drtio: fifo level -> fifo space

---
 artiq/gateware/drtio/iot.py        |  6 +--
 artiq/gateware/drtio/rt_packets.py | 80 +++++++++++++++---------------
 2 files changed, 44 insertions(+), 42 deletions(-)

diff --git a/artiq/gateware/drtio/iot.py b/artiq/gateware/drtio/iot.py
index c05e6b1a7..85367adc3 100644
--- a/artiq/gateware/drtio/iot.py
+++ b/artiq/gateware/drtio/iot.py
@@ -41,9 +41,9 @@ class IOT(Module):
 
             # FIFO level
             self.sync += \
-                If(rt_packets.fifo_level_update &
-                   (rt_packets.fifo_level_channel == n),
-                    rt_packets.fifo_level.eq(fifo.level))
+                If(rt_packets.fifo_space_update &
+                   (rt_packets.fifo_space_channel == n),
+                    rt_packets.fifo_space.eq(channel.ofifo_depth - fifo.level))
 
             # FIFO write
             self.comb += fifo.we.eq(rt_packets.write_stb)
diff --git a/artiq/gateware/drtio/rt_packets.py b/artiq/gateware/drtio/rt_packets.py
index a8602d0eb..7b71201b2 100644
--- a/artiq/gateware/drtio/rt_packets.py
+++ b/artiq/gateware/drtio/rt_packets.py
@@ -37,7 +37,7 @@ def get_m2s_layouts(alignment):
                           ("address", 16),
                           ("data_len", 8),
                           ("short_data", 8))
-    plm.add_type("fifo_level_request", ("channel", 16))
+    plm.add_type("fifo_space_request", ("channel", 16))
     return plm
 
 
@@ -45,7 +45,7 @@ def get_s2m_layouts(alignment):
     plm = PacketLayoutManager(alignment)
     plm.add_type("error", ("code", 8))
     plm.add_type("echo_reply")
-    plm.add_type("fifo_level_reply", ("level", 16))
+    plm.add_type("fifo_space_reply", ("space", 16))
     return plm
 
 
@@ -176,9 +176,9 @@ class RTPacketSatellite(Module):
         self.tsc_load = Signal()
         self.tsc_value = Signal(64)
         
-        self.fifo_level_channel = Signal(16)
-        self.fifo_level_update = Signal()
-        self.fifo_level = Signal(16)
+        self.fifo_space_channel = Signal(16)
+        self.fifo_space_update = Signal()
+        self.fifo_space = Signal(16)
         
         self.write_stb = Signal()
         self.write_timestamp = Signal(64)
@@ -210,14 +210,14 @@ class RTPacketSatellite(Module):
         err_set = Signal()
         err_req = Signal()
         err_ack = Signal()
-        fifo_level_set = Signal()
-        fifo_level_req = Signal()
-        fifo_level_ack = Signal()
+        fifo_space_set = Signal()
+        fifo_space_req = Signal()
+        fifo_space_ack = Signal()
         self.sync += [
             If(err_ack, err_req.eq(0)),
             If(err_set, err_req.eq(1)),
-            If(fifo_level_ack, fifo_level_req.eq(0)),
-            If(fifo_level_set, fifo_level_req.eq(1)),
+            If(fifo_space_ack, fifo_space_req.eq(0)),
+            If(fifo_space_set, fifo_space_req.eq(1)),
         ]
         err_code = Signal(max=len(error_codes)+1)
 
@@ -225,8 +225,8 @@ class RTPacketSatellite(Module):
         self.comb += [
             self.tsc_value.eq(
                 rx_dp.packet_as["set_time"].timestamp),
-            self.fifo_level_channel.eq(
-                rx_dp.packet_as["fifo_level_request"].channel),
+            self.fifo_space_channel.eq(
+                rx_dp.packet_as["fifo_space_request"].channel),
             self.write_timestamp.eq(
                 rx_dp.packet_as["write"].timestamp),
             self.write_channel.eq(
@@ -250,8 +250,8 @@ class RTPacketSatellite(Module):
                         rx_plm.types["echo_request"]: echo_req.eq(1),
                         rx_plm.types["set_time"]: NextState("SET_TIME"),
                         rx_plm.types["write"]: NextState("WRITE"),
-                        rx_plm.types["fifo_level_request"]:
-                            NextState("FIFO_LEVEL"),
+                        rx_plm.types["fifo_space_request"]:
+                            NextState("FIFO_SPACE"),
                         "default": [
                             err_set.eq(1),
                             NextValue(err_code, error_codes["unknown_type"])]
@@ -267,9 +267,9 @@ class RTPacketSatellite(Module):
             self.write_stb.eq(1),
             NextState("INPUT")
         )
-        rx_fsm.act("FIFO_LEVEL",
-            fifo_level_set.eq(1),
-            self.fifo_level_update.eq(1),
+        rx_fsm.act("FIFO_SPACE",
+            fifo_space_set.eq(1),
+            self.fifo_space_update.eq(1),
             NextState("INPUT")
         )
 
@@ -279,7 +279,7 @@ class RTPacketSatellite(Module):
 
         tx_fsm.act("IDLE",
             If(echo_req, NextState("ECHO")),
-            If(fifo_level_req, NextState("FIFO_LEVEL")),
+            If(fifo_space_req, NextState("FIFO_SPACE")),
             If(self.write_overflow, NextState("ERROR_WRITE_OVERFLOW")),
             If(self.write_underflow, NextState("ERROR_WRITE_UNDERFLOW")),
             If(err_req, NextState("ERROR"))
@@ -289,9 +289,9 @@ class RTPacketSatellite(Module):
             tx_dp.stb.eq(1),
             If(tx_dp.done, NextState("IDLE"))
         )
-        tx_fsm.act("FIFO_LEVEL",
-            fifo_level_ack.eq(1),
-            tx_dp.send("fifo_level_reply", level=self.fifo_level),
+        tx_fsm.act("FIFO_SPACE",
+            fifo_space_ack.eq(1),
+            tx_dp.send("fifo_space_reply", space=self.fifo_space),
             tx_dp.stb.eq(1),
             If(tx_dp.done, NextState("IDLE"))
         )
@@ -378,12 +378,12 @@ class RTPacketMaster(Module):
         self.write_address = Signal(16)
         self.write_data = Signal(256)
 
-        # fifo level interface
-        # write with timestamp[48:] == 0xffff to make a fifo level request
-        # (level requests have to be ordered wrt writes)
-        self.fifo_level_not = Signal()
-        self.fifo_level_not_ack = Signal()
-        self.fifo_level = Signal(16)
+        # fifo space interface
+        # write with timestamp[48:] == 0xffff to make a fifo space request
+        # (space requests have to be ordered wrt writes)
+        self.fifo_space_not = Signal()
+        self.fifo_space_not_ack = Signal()
+        self.fifo_space = Signal(16)
 
         # echo interface
         self.echo_stb = Signal()
@@ -394,7 +394,9 @@ class RTPacketMaster(Module):
         # set_time interface
         self.set_time_stb = Signal()
         self.set_time_ack = Signal()
-        self.tsc_value = Signal(64)  # in rtio domain, must be valid all time
+        # in rtio domain, must be valid all time while there is
+        # a set_time request pending
+        self.tsc_value = Signal(64)
 
         # errors
         self.error_not = Signal()
@@ -419,11 +421,11 @@ class RTPacketMaster(Module):
                 write_address, write_data).eq(fifo.dout)
         ]
 
-        fifo_level_not = Signal()
-        fifo_level = Signal(16)
+        fifo_space_not = Signal()
+        fifo_space = Signal(16)
         self.submodules += _CrossDomainNotification("rtio_rx",
-            fifo_level_not, fifo_level,
-            self.fifo_level_not, self.fifo_level_not_ack, self.fifo_level)
+            fifo_space_not, fifo_space,
+            self.fifo_space_not, self.fifo_space_not_ack, self.fifo_space)
 
         set_time_stb = Signal()
         set_time_ack = Signal()
@@ -474,7 +476,7 @@ class RTPacketMaster(Module):
                 short_data=write_data),
             If(wfifo.readable,
                 If(write_timestamp[48:] == 0xffff,
-                    NextState("FIFO_LEVEL")
+                    NextState("FIFO_SPACE")
                 ).Else(
                     tx_dp.stb.eq(1),
                     wfifo.re.eq(tx_dp.done)
@@ -489,8 +491,8 @@ class RTPacketMaster(Module):
                 )
             )
         )
-        tx_fsm.act("FIFO_LEVEL",
-            tx_dp.send("fifo_level_request", channel=write_channel),
+        tx_fsm.act("FIFO_SPACE",
+            tx_dp.send("fifo_space_request", channel=write_channel),
             tx_dp.stb.eq(1),
             If(tx_dp.done, NextState("IDLE_WRITE"))
         )
@@ -519,7 +521,7 @@ class RTPacketMaster(Module):
                     Case(rx_dp.packet_type, {
                         rx_plm.types["error"]: NextState("ERROR"),
                         rx_plm.types["echo_reply"]: echo_received_now.eq(1),
-                        rx_plm.types["fifo_level_reply"]: NextState("FIFO_LEVEL"),
+                        rx_plm.types["fifo_space_reply"]: NextState("FIFO_SPACE"),
                         "default": [
                             error_not.eq(1),
                             error_code.eq(error_codes["unknown_type"])
@@ -533,8 +535,8 @@ class RTPacketMaster(Module):
             error_code.eq(rx_dp.packet_as["error"].code),
             NextState("INPUT")
         )
-        rx_fsm.act("FIFO_LEVEL",
-            fifo_level_not.eq(1),
-            fifo_level.eq(rx_dp.packet_as["fifo_level_reply"].level),
+        rx_fsm.act("FIFO_SPACE",
+            fifo_space_not.eq(1),
+            fifo_space.eq(rx_dp.packet_as["fifo_space_reply"].space),
             NextState("INPUT")
         )

From 7dd6eb2f5efcc85bc42b2ea78298d3cdc31e9d23 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Mon, 24 Oct 2016 19:50:13 +0800
Subject: [PATCH 034/134] drtio: add RT write controller

---
 artiq/gateware/drtio/core.py          |   6 +-
 artiq/gateware/drtio/rt_controller.py | 159 ++++++++++++++++++++++++++
 2 files changed, 164 insertions(+), 1 deletion(-)
 create mode 100644 artiq/gateware/drtio/rt_controller.py

diff --git a/artiq/gateware/drtio/core.py b/artiq/gateware/drtio/core.py
index 6c805c97c..5e92eb559 100644
--- a/artiq/gateware/drtio/core.py
+++ b/artiq/gateware/drtio/core.py
@@ -2,7 +2,7 @@ from types import SimpleNamespace
 
 from migen import *
 
-from artiq.gateware.drtio import link_layer, rt_packets, iot
+from artiq.gateware.drtio import link_layer, rt_packets, iot, bus_interface
 
 
 class DRTIOSatellite(Module):
@@ -53,3 +53,7 @@ class DRTIOMaster(Module):
             self.link_layer.rx_ready.eq(transceiver.rx_ready)
         ]
         self.submodules.rt_packets = rt_packets.RTPacketMaster(self.link_layer)
+        self.submodules.rt_controller = rt_controller.RTController(self.rt_packets)
+
+    def get_csrs(self):
+        return self.rt_controller.get_csrs()
diff --git a/artiq/gateware/drtio/rt_controller.py b/artiq/gateware/drtio/rt_controller.py
new file mode 100644
index 000000000..a8c313fc0
--- /dev/null
+++ b/artiq/gateware/drtio/rt_controller.py
@@ -0,0 +1,159 @@
+from migen import *
+
+from misoc.interconnect.csr import *
+
+from artiq.gateware.rtio.cdc import RTIOCounter
+
+
+class _KernelCSRs(AutoCSR):
+    def __init__(self):
+        # chan_sel must be written at least 1 cycle before we
+        # and held stable until the transaction is complete.
+        # timestamp must be written at least 1 cycle before we.
+        self.chan_sel = CSRStorage(16)
+        self.o_data = CSRStorage(64)
+        self.o_address = CSRStorage(16)
+        self.o_timestamp = CSRStorage(64)
+        self.o_we = CSR()
+        self.o_status = CSRStatus(3)
+        self.o_underflow_reset = CSR()
+        self.o_sequence_error_reset = CSR()
+
+        self.counter = CSRStatus(64)
+        self.counter_update = CSR()
+
+        self.tsc_correction = CSRStorage(64)
+        self.set_time = CSR()
+        self.underflow_margin = CSRStorage(16, reset=50)
+
+        self.get_fifo_space = CSR()
+        self.dbg_fifo_space = CSRStatus(16)
+        self.dbg_last_timestamp = CSRStatus(64)
+        self.reset_channel_status = CSR()
+
+
+class RTController(Module):
+    def __init__(self, rt_packets, channel_count=1024):
+        self.kcsrs = _KernelCSRs()
+
+        self.submodules.counter = RTIOCounter(64)
+        self.sync += If(self.kcsrs.counter_update.re, 
+                        self.kcsrs.counter.status.eq(self.counter.value_sys))
+        tsc_correction = Signal(64)
+        self.specials += MultiReg(self.tsc_correction.storage, tsc_correction)
+        self.comb += [ 
+            rt_packets.tsc_value.eq(
+                self.counter.value_rtio + tsc_correction),
+            self.set_time.value.eq(rt_packets.set_time_stb)
+        ]
+        self.sync += [
+            If(rt_packets.set_time_ack, rt_packets.set_time_stb.eq(0)),
+            If(self.set_time_stb.re, rt_packets.set_time_stb.eq(1))
+        ]
+
+        fifo_spaces_mem = Memory(16, channel_count)
+        fifo_spaces = fifo_spaces_mem.get_port(write_capable=True)
+        self.specials += fifo_spaces_mem, fifo_spaces
+        last_timestamps_mem = Memory(64, channel_count)
+        last_timestamps = last_timestamps_mem.get_port(write_capable=True)
+        self.specials += last_timestamps_mem, last_timestamps
+
+        rt_packets_fifo_request = Signal()
+        self.comb += [
+            fifo_spaces.adr.eq(self.kcsrs.chan_sel.storage),
+            last_timestamps.adr.eq(self.kcsrs.chan_sel.storage),
+            last_timestamps.dat_w.eq(self.kcsrs.timestamp.storage),
+            rt_packets.write_channel.eq(self.kcsrs.chan_sel.storage),
+            rt_packets.write_address.eq(self.kcsrs.o_address.storage),
+            rt_packets.write_data.eq(self.kcsrs.o_data.storage),
+            If(rt_packets_fifo_request,
+                rt_packets.write_timestamp.eq(0xffff000000000000)
+            ).Else(
+                rt_packets.write_timestamp.eq(self.o_timestamp.storage)
+            )
+        ]
+
+        fsm = FSM()
+        self.submodules += fsm
+
+        status_wait = Signal()
+        status_underflow = Signal()
+        status_sequence_error = Signal()
+        self.comb += self.kcsrs.o_status.status.eq(Cat(
+            status_wait, status_underflow, status_sequence_error))
+
+        # TODO: collision, replace, busy
+        cond_sequence_error = self.o_timestamp.storage < last_timestamps.dat_r
+        cond_underflow = (self.o_timestamp.storage - self.kcsrs.underflow_margin.storage
+                          < self.counter.value_sys)
+        cond_fifo_emptied = ((last_timestamps.dat_r
+                              < self.counter.value_sys - self.kcsrs.underflow_margin.storage)
+                             & (last_timestamps.dat_r != 0))
+
+        fsm.act("IDLE",
+            If(self.o_we.re,
+                If(cond_sequence_error,
+                    sequence_error_set.eq(1)
+                ).Elif(cond_underflow,
+                    underflow_set.eq(1)
+                ).Else(
+                    NextState("WRITE")
+                )
+            ),
+            If(self.get_fifo_space.re,
+                NextState("GET_FIFO_SPACE")
+            )
+        )
+        fsm.act("WRITE",
+            status_wait.eq(1),
+            rt_packets.write_stb.eq(1),
+            If(rt_packets.write_ack,
+                fifo_spaces.we.eq(1),
+                If(cond_fifo_emptied,
+                    fifo_spaces.dat_w.eq(1),
+                ).Else(
+                    fifo_spaces.dat_w.eq(fifo_spaces.dat_r - 1)
+                ),
+                last_timestamps.we.eq(1),
+                If(fifo_spaces.dat_r <= 1,
+                    NextState("GET_FIFO_SPACE")
+                ).Else(
+                    NextState("IDLE")
+                )
+            )
+        )
+        fsm.act("GET_FIFO_SPACE",
+            status_wait.eq(1),
+            rt_packets_fifo_request.eq(1),
+            rt_packets.write_stb.eq(1),
+            If(rt_packets.write_ack,
+                NextState("GET_FIFO_SPACE_REPLY")
+            )
+        )
+        fsm.act("GET_FIFO_SPACE_REPLY",
+            status_wait.eq(1),
+            fifo_spaces.dat_w.eq(rt_packets.fifo_space),
+            fifo_spaces.we.eq(1),
+            fifo_space_not_ack.eq(1),
+            If(rt_packets.fifo_space_not,
+                If(rt_packets.fifo_spaces > 0,
+                    NextState("IDLE")
+                ).Else(
+                    NextState("GET_FIFO_SPACE")
+                )
+            )
+        )
+
+        self.comb += [
+            self.kcsrs.dbg_fifo_space.status.eq(fifo_spaces.dat_r),
+            self.kcsrs.dbg_last_timestamp.status.eq(last_timestamps.dat_r),
+            If(self.kcsrs.reset_channel_status.re,
+                fifo_spaces.dat_w.eq(0),
+                fifo_spaces.we.eq(1),
+                last_timestamps.dat_w.eq(0),
+                last_timestamps.we.eq(1)
+            )
+        ]
+
+    def get_csrs(self):
+        return self.kcsrs.get_csrs()

From c39987b61792082c72e62d08d5e77322b4c9e03a Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Mon, 24 Oct 2016 20:46:55 +0800
Subject: [PATCH 035/134] drtio: handle underflow/sequence error CSRs

---
 artiq/gateware/drtio/rt_controller.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/artiq/gateware/drtio/rt_controller.py b/artiq/gateware/drtio/rt_controller.py
index a8c313fc0..69eb09b6f 100644
--- a/artiq/gateware/drtio/rt_controller.py
+++ b/artiq/gateware/drtio/rt_controller.py
@@ -81,6 +81,14 @@ class RTController(Module):
         status_sequence_error = Signal()
         self.comb += self.kcsrs.o_status.status.eq(Cat(
             status_wait, status_underflow, status_sequence_error))
+        sequence_error_set = Signal()
+        underflow_set = Signal()
+        self.sync += [
+            If(self.kcsrs.o_underflow_reset.re, status_underflow.eq(0)),
+            If(self.kcsrs.o_sequence_error_reset, status_sequence_error.eq(0)),
+            If(underflow_set, status_underflow.eq(1)),
+            If(sequence_error_set, status_sequence_error.eq(1)),
+        ]
 
         # TODO: collision, replace, busy
         cond_sequence_error = self.o_timestamp.storage < last_timestamps.dat_r

From 029e0d95b70334db36a713de92019b3ace0748db Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Mon, 24 Oct 2016 23:10:15 +0800
Subject: [PATCH 036/134] drtio: simple fixes

---
 artiq/gateware/drtio/core.py          |  2 +-
 artiq/gateware/drtio/rt_controller.py | 25 +++++++++++++------------
 artiq/gateware/drtio/rt_packets.py    |  2 +-
 3 files changed, 15 insertions(+), 14 deletions(-)

diff --git a/artiq/gateware/drtio/core.py b/artiq/gateware/drtio/core.py
index 5e92eb559..7fcfb7110 100644
--- a/artiq/gateware/drtio/core.py
+++ b/artiq/gateware/drtio/core.py
@@ -2,7 +2,7 @@ from types import SimpleNamespace
 
 from migen import *
 
-from artiq.gateware.drtio import link_layer, rt_packets, iot, bus_interface
+from artiq.gateware.drtio import link_layer, rt_packets, iot, rt_controller
 
 
 class DRTIOSatellite(Module):
diff --git a/artiq/gateware/drtio/rt_controller.py b/artiq/gateware/drtio/rt_controller.py
index 69eb09b6f..a5826767d 100644
--- a/artiq/gateware/drtio/rt_controller.py
+++ b/artiq/gateware/drtio/rt_controller.py
@@ -1,4 +1,5 @@
 from migen import *
+from migen.genlib.cdc import MultiReg
 
 from misoc.interconnect.csr import *
 
@@ -40,15 +41,15 @@ class RTController(Module):
         self.sync += If(self.kcsrs.counter_update.re, 
                         self.kcsrs.counter.status.eq(self.counter.value_sys))
         tsc_correction = Signal(64)
-        self.specials += MultiReg(self.tsc_correction.storage, tsc_correction)
+        self.specials += MultiReg(self.kcsrs.tsc_correction.storage, tsc_correction)
         self.comb += [ 
             rt_packets.tsc_value.eq(
                 self.counter.value_rtio + tsc_correction),
-            self.set_time.value.eq(rt_packets.set_time_stb)
+            self.kcsrs.set_time.r.eq(rt_packets.set_time_stb)
         ]
         self.sync += [
             If(rt_packets.set_time_ack, rt_packets.set_time_stb.eq(0)),
-            If(self.set_time_stb.re, rt_packets.set_time_stb.eq(1))
+            If(self.kcsrs.set_time.re, rt_packets.set_time_stb.eq(1))
         ]
 
         fifo_spaces_mem = Memory(16, channel_count)
@@ -62,14 +63,14 @@ class RTController(Module):
         self.comb += [
             fifo_spaces.adr.eq(self.kcsrs.chan_sel.storage),
             last_timestamps.adr.eq(self.kcsrs.chan_sel.storage),
-            last_timestamps.dat_w.eq(self.kcsrs.timestamp.storage),
+            last_timestamps.dat_w.eq(self.kcsrs.o_timestamp.storage),
             rt_packets.write_channel.eq(self.kcsrs.chan_sel.storage),
             rt_packets.write_address.eq(self.kcsrs.o_address.storage),
             rt_packets.write_data.eq(self.kcsrs.o_data.storage),
             If(rt_packets_fifo_request,
                 rt_packets.write_timestamp.eq(0xffff000000000000)
             ).Else(
-                rt_packets.write_timestamp.eq(self.o_timestamp.storage)
+                rt_packets.write_timestamp.eq(self.kcsrs.o_timestamp.storage)
             )
         ]
 
@@ -85,21 +86,21 @@ class RTController(Module):
         underflow_set = Signal()
         self.sync += [
             If(self.kcsrs.o_underflow_reset.re, status_underflow.eq(0)),
-            If(self.kcsrs.o_sequence_error_reset, status_sequence_error.eq(0)),
+            If(self.kcsrs.o_sequence_error_reset.re, status_sequence_error.eq(0)),
             If(underflow_set, status_underflow.eq(1)),
             If(sequence_error_set, status_sequence_error.eq(1)),
         ]
 
         # TODO: collision, replace, busy
-        cond_sequence_error = self.o_timestamp.storage < last_timestamps.dat_r
-        cond_underflow = (self.o_timestamp.storage - self.kcsrs.underflow_margin.storage
+        cond_sequence_error = self.kcsrs.o_timestamp.storage < last_timestamps.dat_r
+        cond_underflow = (self.kcsrs.o_timestamp.storage - self.kcsrs.underflow_margin.storage
                           < self.counter.value_sys)
         cond_fifo_emptied = ((last_timestamps.dat_r
                               < self.counter.value_sys - self.kcsrs.underflow_margin.storage)
                              & (last_timestamps.dat_r != 0))
 
         fsm.act("IDLE",
-            If(self.o_we.re,
+            If(self.kcsrs.o_we.re,
                 If(cond_sequence_error,
                     sequence_error_set.eq(1)
                 ).Elif(cond_underflow,
@@ -108,7 +109,7 @@ class RTController(Module):
                     NextState("WRITE")
                 )
             ),
-            If(self.get_fifo_space.re,
+            If(self.kcsrs.get_fifo_space.re,
                 NextState("GET_FIFO_SPACE")
             )
         )
@@ -142,9 +143,9 @@ class RTController(Module):
             status_wait.eq(1),
             fifo_spaces.dat_w.eq(rt_packets.fifo_space),
             fifo_spaces.we.eq(1),
-            fifo_space_not_ack.eq(1),
+            rt_packets.fifo_space_not_ack.eq(1),
             If(rt_packets.fifo_space_not,
-                If(rt_packets.fifo_spaces > 0,
+                If(rt_packets.fifo_space > 0,
                     NextState("IDLE")
                 ).Else(
                     NextState("GET_FIFO_SPACE")
diff --git a/artiq/gateware/drtio/rt_packets.py b/artiq/gateware/drtio/rt_packets.py
index 7b71201b2..1f0dc6039 100644
--- a/artiq/gateware/drtio/rt_packets.py
+++ b/artiq/gateware/drtio/rt_packets.py
@@ -418,7 +418,7 @@ class RTPacketMaster(Module):
             wfifo.din.eq(Cat(self.write_timestamp, self.write_channel,
                              self.write_address, self.write_data)),
             Cat(write_timestamp, write_channel,
-                write_address, write_data).eq(fifo.dout)
+                write_address, write_data).eq(wfifo.dout)
         ]
 
         fifo_space_not = Signal()

From a4e85081aaab3598a1645f72f2bad2e309cf7ce9 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Mon, 24 Oct 2016 23:32:49 +0800
Subject: [PATCH 037/134] drtio: more simple fixes

---
 artiq/gateware/drtio/core.py       | 4 ++--
 artiq/gateware/drtio/rt_packets.py | 3 ++-
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/artiq/gateware/drtio/core.py b/artiq/gateware/drtio/core.py
index 7fcfb7110..95260f773 100644
--- a/artiq/gateware/drtio/core.py
+++ b/artiq/gateware/drtio/core.py
@@ -38,9 +38,9 @@ class DRTIOSatellite(Module):
         self.clock_domains.cd_rio_phy = ClockDomain()
         self.comb += [
             self.cd_rio.clk.eq(ClockSignal("rtio")),
-            self.cd_rio.rst.eq(ResetSignal("rtio")),
+            self.cd_rio.rst.eq(ResetSignal("rtio", allow_reset_less=True)),
             self.cd_rio_phy.clk.eq(ClockSignal("rtio")),
-            self.cd_rio_phy.rst.eq(ResetSignal("rtio")),
+            self.cd_rio_phy.rst.eq(ResetSignal("rtio", allow_reset_less=True)),
         ]
 
 
diff --git a/artiq/gateware/drtio/rt_packets.py b/artiq/gateware/drtio/rt_packets.py
index 1f0dc6039..51ad16368 100644
--- a/artiq/gateware/drtio/rt_packets.py
+++ b/artiq/gateware/drtio/rt_packets.py
@@ -406,7 +406,8 @@ class RTPacketMaster(Module):
         # # #
 
         # CDC
-        wfifo = AsyncFIFO(64+16+16+256, write_fifo_depth)
+        wfifo = ClockDomainsRenamer({"write": "sys", "read": "rtio"})(
+            AsyncFIFO(64+16+16+256, write_fifo_depth))
         self.submodules += wfifo
         write_timestamp = Signal(64)
         write_channel = Signal(16)

From 94e68dbae44a70d34ed68171354acc9822e1ef90 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Mon, 24 Oct 2016 23:36:33 +0800
Subject: [PATCH 038/134] drtio: test_full_stack (WIP)

---
 artiq/test/gateware/drtio/test_full_stack.py | 61 ++++++++++++++++++++
 1 file changed, 61 insertions(+)
 create mode 100644 artiq/test/gateware/drtio/test_full_stack.py

diff --git a/artiq/test/gateware/drtio/test_full_stack.py b/artiq/test/gateware/drtio/test_full_stack.py
new file mode 100644
index 000000000..a86013b82
--- /dev/null
+++ b/artiq/test/gateware/drtio/test_full_stack.py
@@ -0,0 +1,61 @@
+import unittest
+from types import SimpleNamespace
+
+from migen import *
+
+from artiq.gateware.drtio import *
+from artiq.gateware import rtio
+from artiq.gateware.rtio.phy import ttl_simple
+
+
+class DummyTransceiverPair:
+    def __init__(self, nwords):
+        a2b_k = [Signal() for _ in range(nwords)]
+        a2b_d = [Signal(8) for _ in range(nwords)]
+        b2a_k = [Signal() for _ in range(nwords)]
+        b2a_d = [Signal(8) for _ in range(nwords)]
+
+        self.alice = SimpleNamespace(
+            encoder=SimpleNamespace(k=a2b_k, d=a2b_d),
+            decoders=[SimpleNamespace(k=k, d=d) for k, d in zip(b2a_k, b2a_d)],
+            rx_reset=Signal(),
+            rx_ready=1
+        )
+        self.bob = SimpleNamespace(
+            encoder=SimpleNamespace(k=b2a_k, d=b2a_d),
+            decoders=[SimpleNamespace(k=k, d=d) for k, d in zip(a2b_k, a2b_d)],
+            rx_reset=Signal(),
+            rx_ready=1
+        )
+
+
+class DummyRXSynchronizer:
+    def resync(self, signal):
+        return signal
+
+
+class DUT(Module):
+    def __init__(self, nwords):
+        self.ttl = Signal()
+        self.transceivers = DummyTransceiverPair(2)
+        
+        self.submodules.master = DRTIOMaster(self.transceivers.alice)
+
+        rx_synchronizer = DummyRXSynchronizer()
+        self.submodules.phy = ttl_simple.Output(self.ttl)
+        self.submodules.satellite = DRTIOSatellite(
+            self.transceivers.bob, rx_synchronizer, [rtio.Channel.from_phy(self.phy)])
+        
+
+class TestFullStack(unittest.TestCase):
+    def test_full_stack(self):
+        dut = DUT(2)
+        kcsrs = dut.master.rt_controller.kcsrs    
+
+        def get_fifo_level():
+            for i in range(8):
+                yield from kcsrs.counter_update.write(1)
+                print((yield from kcsrs.counter.read()))
+
+        run_simulation(dut, get_fifo_level(),
+            {"sys": 8, "rtio": 5, "rtio_rx": 5})

From ad042de954017df7328eba0f5376067f9c0f521c Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Tue, 25 Oct 2016 12:41:16 +0800
Subject: [PATCH 039/134] drtio: fixes, basic TTL working in simulation

---
 artiq/gateware/drtio/core.py                 |  5 ++-
 artiq/gateware/drtio/rt_controller.py        | 24 ++++++------
 artiq/gateware/drtio/rt_packets.py           |  5 ++-
 artiq/test/gateware/drtio/test_full_stack.py | 40 ++++++++++++++++----
 4 files changed, 52 insertions(+), 22 deletions(-)

diff --git a/artiq/gateware/drtio/core.py b/artiq/gateware/drtio/core.py
index 95260f773..af467156d 100644
--- a/artiq/gateware/drtio/core.py
+++ b/artiq/gateware/drtio/core.py
@@ -45,7 +45,7 @@ class DRTIOSatellite(Module):
 
 
 class DRTIOMaster(Module):
-    def __init__(self, transceiver):
+    def __init__(self, transceiver, channel_count=1024, fine_ts_width=3):
         self.submodules.link_layer = link_layer.LinkLayer(
             transceiver.encoder, transceiver.decoders)
         self.comb += [
@@ -53,7 +53,8 @@ class DRTIOMaster(Module):
             self.link_layer.rx_ready.eq(transceiver.rx_ready)
         ]
         self.submodules.rt_packets = rt_packets.RTPacketMaster(self.link_layer)
-        self.submodules.rt_controller = rt_controller.RTController(self.rt_packets)
+        self.submodules.rt_controller = rt_controller.RTController(
+            self.rt_packets, channel_count, fine_ts_width)
 
     def get_csrs(self):
         return self.rt_controller.get_csrs()
diff --git a/artiq/gateware/drtio/rt_controller.py b/artiq/gateware/drtio/rt_controller.py
index a5826767d..bc9dbb2f3 100644
--- a/artiq/gateware/drtio/rt_controller.py
+++ b/artiq/gateware/drtio/rt_controller.py
@@ -25,16 +25,16 @@ class _KernelCSRs(AutoCSR):
 
         self.tsc_correction = CSRStorage(64)
         self.set_time = CSR()
-        self.underflow_margin = CSRStorage(16, reset=50)
+        self.underflow_margin = CSRStorage(16, reset=200)
 
-        self.get_fifo_space = CSR()
-        self.dbg_fifo_space = CSRStatus(16)
-        self.dbg_last_timestamp = CSRStatus(64)
-        self.reset_channel_status = CSR()
+        self.o_get_fifo_space = CSR()
+        self.o_dbg_fifo_space = CSRStatus(16)
+        self.o_dbg_last_timestamp = CSRStatus(64)
+        self.o_reset_channel_status = CSR()
 
 
 class RTController(Module):
-    def __init__(self, rt_packets, channel_count=1024):
+    def __init__(self, rt_packets, channel_count, fine_ts_width):
         self.kcsrs = _KernelCSRs()
 
         self.submodules.counter = RTIOCounter(64)
@@ -93,8 +93,8 @@ class RTController(Module):
 
         # TODO: collision, replace, busy
         cond_sequence_error = self.kcsrs.o_timestamp.storage < last_timestamps.dat_r
-        cond_underflow = (self.kcsrs.o_timestamp.storage - self.kcsrs.underflow_margin.storage
-                          < self.counter.value_sys)
+        cond_underflow = ((self.kcsrs.o_timestamp.storage - self.kcsrs.underflow_margin.storage
+                           >> fine_ts_width) < self.counter.value_sys)
         cond_fifo_emptied = ((last_timestamps.dat_r
                               < self.counter.value_sys - self.kcsrs.underflow_margin.storage)
                              & (last_timestamps.dat_r != 0))
@@ -109,7 +109,7 @@ class RTController(Module):
                     NextState("WRITE")
                 )
             ),
-            If(self.kcsrs.get_fifo_space.re,
+            If(self.kcsrs.o_get_fifo_space.re,
                 NextState("GET_FIFO_SPACE")
             )
         )
@@ -154,9 +154,9 @@ class RTController(Module):
         )
 
         self.comb += [
-            self.kcsrs.dbg_fifo_space.status.eq(fifo_spaces.dat_r),
-            self.kcsrs.dbg_last_timestamp.status.eq(last_timestamps.dat_r),
-            If(self.kcsrs.reset_channel_status.re,
+            self.kcsrs.o_dbg_fifo_space.status.eq(fifo_spaces.dat_r),
+            self.kcsrs.o_dbg_last_timestamp.status.eq(last_timestamps.dat_r),
+            If(self.kcsrs.o_reset_channel_status.re,
                 fifo_spaces.dat_w.eq(0),
                 fifo_spaces.we.eq(1),
                 last_timestamps.dat_w.eq(0),
diff --git a/artiq/gateware/drtio/rt_packets.py b/artiq/gateware/drtio/rt_packets.py
index 51ad16368..3a3818f48 100644
--- a/artiq/gateware/drtio/rt_packets.py
+++ b/artiq/gateware/drtio/rt_packets.py
@@ -495,7 +495,10 @@ class RTPacketMaster(Module):
         tx_fsm.act("FIFO_SPACE",
             tx_dp.send("fifo_space_request", channel=write_channel),
             tx_dp.stb.eq(1),
-            If(tx_dp.done, NextState("IDLE_WRITE"))
+            If(tx_dp.done,
+                wfifo.re.eq(1),
+                NextState("IDLE_WRITE")
+            )
         )
         tx_fsm.act("ECHO",
             tx_dp.send("echo_request"),
diff --git a/artiq/test/gateware/drtio/test_full_stack.py b/artiq/test/gateware/drtio/test_full_stack.py
index a86013b82..76cd42619 100644
--- a/artiq/test/gateware/drtio/test_full_stack.py
+++ b/artiq/test/gateware/drtio/test_full_stack.py
@@ -37,7 +37,7 @@ class DummyRXSynchronizer:
 class DUT(Module):
     def __init__(self, nwords):
         self.ttl = Signal()
-        self.transceivers = DummyTransceiverPair(2)
+        self.transceivers = DummyTransceiverPair(nwords)
         
         self.submodules.master = DRTIOMaster(self.transceivers.alice)
 
@@ -52,10 +52,36 @@ class TestFullStack(unittest.TestCase):
         dut = DUT(2)
         kcsrs = dut.master.rt_controller.kcsrs    
 
-        def get_fifo_level():
-            for i in range(8):
-                yield from kcsrs.counter_update.write(1)
-                print((yield from kcsrs.counter.read()))
+        def get_fifo_space():
+            yield from kcsrs.o_get_fifo_space.write(1)
+            yield
+            while (yield from kcsrs.o_status.read()) & 1:
+                yield
+            return (yield from kcsrs.o_dbg_fifo_space.read())
 
-        run_simulation(dut, get_fifo_level(),
-            {"sys": 8, "rtio": 5, "rtio_rx": 5})
+        def test():
+            print((yield from get_fifo_space()))
+            yield from kcsrs.o_timestamp.write(550)
+            yield from kcsrs.o_data.write(1)
+            yield from kcsrs.o_we.write(1)
+            yield
+            status = 1
+            while status:
+                status = yield from kcsrs.o_status.read()
+                print("status after write:", status)
+                yield
+            yield from kcsrs.o_timestamp.write(600)
+            yield from kcsrs.o_data.write(0)
+            yield from kcsrs.o_we.write(1)
+            yield
+            status = 1
+            while status:
+                status = yield from kcsrs.o_status.read()
+                print("status after write:", status)
+                yield
+            for i in range(40):
+                yield
+            #print((yield from get_fifo_space()))
+
+        run_simulation(dut, test(),
+            {"sys": 8, "rtio": 5, "rtio_rx": 5, "rio": 5, "rio_phy": 5}, vcd_name="foo.vcd")

From f763b519f4b37af2dad45697d5bdfac9751a319e Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Wed, 26 Oct 2016 00:33:21 +0800
Subject: [PATCH 040/134] drtio: fix channel selection

---
 artiq/gateware/drtio/iot.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/artiq/gateware/drtio/iot.py b/artiq/gateware/drtio/iot.py
index 85367adc3..d7ba41c78 100644
--- a/artiq/gateware/drtio/iot.py
+++ b/artiq/gateware/drtio/iot.py
@@ -46,9 +46,10 @@ class IOT(Module):
                     rt_packets.fifo_space.eq(channel.ofifo_depth - fifo.level))
 
             # FIFO write
-            self.comb += fifo.we.eq(rt_packets.write_stb)
+            self.comb += fifo.we.eq(rt_packets.write_stb
+                                    & (rt_packets.write_channel == n))
             self.sync += \
-                If(rt_packets.write_stb,
+                If(fifo.we,
                     If(rt_packets.write_overflow_ack,
                         rt_packets.write_overflow.eq(0)),
                     If(~fifo.writable, rt_packets.write_overflow.eq(1)),

From 22173b8c709f899b092342a4ccacb61bb6834de3 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Wed, 26 Oct 2016 00:35:22 +0800
Subject: [PATCH 041/134] drtio: full stack unittest

---
 artiq/test/gateware/drtio/test_full_stack.py | 96 +++++++++++++++-----
 1 file changed, 73 insertions(+), 23 deletions(-)

diff --git a/artiq/test/gateware/drtio/test_full_stack.py b/artiq/test/gateware/drtio/test_full_stack.py
index 76cd42619..ab6fd68e4 100644
--- a/artiq/test/gateware/drtio/test_full_stack.py
+++ b/artiq/test/gateware/drtio/test_full_stack.py
@@ -6,6 +6,7 @@ from migen import *
 from artiq.gateware.drtio import *
 from artiq.gateware import rtio
 from artiq.gateware.rtio.phy import ttl_simple
+from artiq.coredevice.exceptions import *
 
 
 class DummyTransceiverPair:
@@ -36,52 +37,101 @@ class DummyRXSynchronizer:
 
 class DUT(Module):
     def __init__(self, nwords):
-        self.ttl = Signal()
+        self.ttl0 = Signal()
+        self.ttl1 = Signal()
         self.transceivers = DummyTransceiverPair(nwords)
         
         self.submodules.master = DRTIOMaster(self.transceivers.alice)
 
         rx_synchronizer = DummyRXSynchronizer()
-        self.submodules.phy = ttl_simple.Output(self.ttl)
+        self.submodules.phy0 = ttl_simple.Output(self.ttl0)
+        self.submodules.phy1 = ttl_simple.Output(self.ttl1)
+        rtio_channels = [
+            rtio.Channel.from_phy(self.phy0),
+            rtio.Channel.from_phy(self.phy1)
+        ]
         self.submodules.satellite = DRTIOSatellite(
-            self.transceivers.bob, rx_synchronizer, [rtio.Channel.from_phy(self.phy)])
+            self.transceivers.bob, rx_synchronizer, rtio_channels)
         
 
 class TestFullStack(unittest.TestCase):
     def test_full_stack(self):
         dut = DUT(2)
-        kcsrs = dut.master.rt_controller.kcsrs    
+        kcsrs = dut.master.rt_controller.kcsrs
 
-        def get_fifo_space():
+        now = 0
+        def delay(dt):
+            nonlocal now
+            now += dt
+
+        def get_fifo_space(channel):
+            yield from kcsrs.chan_sel.write(channel)
             yield from kcsrs.o_get_fifo_space.write(1)
             yield
             while (yield from kcsrs.o_status.read()) & 1:
                 yield
             return (yield from kcsrs.o_dbg_fifo_space.read())
 
-        def test():
-            print((yield from get_fifo_space()))
-            yield from kcsrs.o_timestamp.write(550)
-            yield from kcsrs.o_data.write(1)
+        def write(channel, data):
+            yield from kcsrs.chan_sel.write(channel)
+            yield from kcsrs.o_timestamp.write(now)
+            yield from kcsrs.o_data.write(data)
             yield from kcsrs.o_we.write(1)
             yield
             status = 1
             while status:
                 status = yield from kcsrs.o_status.read()
-                print("status after write:", status)
+                if status & 2:
+                    yield from kcsrs.o_underflow_reset.write(1)
+                    raise RTIOUnderflow
+                if status & 4:
+                    yield from kcsrs.o_sequence_error_reset.write(1)
+                    raise RTIOSequenceError
                 yield
-            yield from kcsrs.o_timestamp.write(600)
-            yield from kcsrs.o_data.write(0)
-            yield from kcsrs.o_we.write(1)
-            yield
-            status = 1
-            while status:
-                status = yield from kcsrs.o_status.read()
-                print("status after write:", status)
-                yield
-            for i in range(40):
-                yield
-            #print((yield from get_fifo_space()))
 
-        run_simulation(dut, test(),
+        def test():
+            yield from get_fifo_space(0)
+            yield from get_fifo_space(1)
+
+            with self.assertRaises(RTIOUnderflow):
+                yield from write(0, 0)
+            
+            delay(200*8)
+            yield from write(0, 1)
+            delay(5*8)
+            yield from write(0, 0)
+            yield from write(1, 1)
+            delay(6*8)
+            yield from write(1, 0)
+
+            delay(-200*8)
+            with self.assertRaises(RTIOSequenceError):
+                yield from write(0, 1)
+            delay(200*8)
+
+            for _ in range(50):
+                yield
+
+        ttl_changes = []
+        @passive
+        def check_ttls():
+            cycle = 0
+            old_ttls = [0, 0]
+            while True:
+                ttls = [(yield dut.ttl0), (yield dut.ttl1)]
+                for n, (old_ttl, ttl) in enumerate(zip(old_ttls, ttls)):
+                    if ttl != old_ttl:
+                        ttl_changes.append((cycle, n))
+                old_ttls = ttls
+                yield
+                cycle += 1
+
+        run_simulation(dut, 
+            {"sys": test(), "rtio": check_ttls()},
             {"sys": 8, "rtio": 5, "rtio_rx": 5, "rio": 5, "rio_phy": 5}, vcd_name="foo.vcd")
+        self.assertEqual(ttl_changes, [
+            (203, 0),
+            (208, 0),
+            (208, 1),
+            (214, 1)
+        ])

From 7f8e53aa5cfa868d524f416b6a9f8e1ddd657bea Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Wed, 26 Oct 2016 11:48:47 +0800
Subject: [PATCH 042/134] drtio: more fixes and tests

---
 artiq/gateware/drtio/rt_controller.py        | 12 +++---
 artiq/test/gateware/drtio/test_full_stack.py | 39 +++++++++++++++++---
 2 files changed, 40 insertions(+), 11 deletions(-)

diff --git a/artiq/gateware/drtio/rt_controller.py b/artiq/gateware/drtio/rt_controller.py
index bc9dbb2f3..e0efafcd7 100644
--- a/artiq/gateware/drtio/rt_controller.py
+++ b/artiq/gateware/drtio/rt_controller.py
@@ -37,7 +37,7 @@ class RTController(Module):
     def __init__(self, rt_packets, channel_count, fine_ts_width):
         self.kcsrs = _KernelCSRs()
 
-        self.submodules.counter = RTIOCounter(64)
+        self.submodules.counter = RTIOCounter(64-fine_ts_width)
         self.sync += If(self.kcsrs.counter_update.re, 
                         self.kcsrs.counter.status.eq(self.counter.value_sys))
         tsc_correction = Signal(64)
@@ -93,10 +93,10 @@ class RTController(Module):
 
         # TODO: collision, replace, busy
         cond_sequence_error = self.kcsrs.o_timestamp.storage < last_timestamps.dat_r
-        cond_underflow = ((self.kcsrs.o_timestamp.storage - self.kcsrs.underflow_margin.storage
-                           >> fine_ts_width) < self.counter.value_sys)
-        cond_fifo_emptied = ((last_timestamps.dat_r
-                              < self.counter.value_sys - self.kcsrs.underflow_margin.storage)
+        cond_underflow = ((self.kcsrs.o_timestamp.storage[fine_ts_width:]
+                           - self.kcsrs.underflow_margin.storage[fine_ts_width:]) < self.counter.value_sys)
+        cond_fifo_emptied = ((last_timestamps.dat_r[fine_ts_width:]
+                              < self.counter.value_sys - self.kcsrs.underflow_margin.storage[fine_ts_width:])
                              & (last_timestamps.dat_r != 0))
 
         fsm.act("IDLE",
@@ -124,7 +124,7 @@ class RTController(Module):
                     fifo_spaces.dat_w.eq(fifo_spaces.dat_r - 1)
                 ),
                 last_timestamps.we.eq(1),
-                If(fifo_spaces.dat_r <= 1,
+                If(~cond_fifo_emptied & (fifo_spaces.dat_r <= 1),
                     NextState("GET_FIFO_SPACE")
                 ).Else(
                     NextState("IDLE")
diff --git a/artiq/test/gateware/drtio/test_full_stack.py b/artiq/test/gateware/drtio/test_full_stack.py
index ab6fd68e4..002b9bbc0 100644
--- a/artiq/test/gateware/drtio/test_full_stack.py
+++ b/artiq/test/gateware/drtio/test_full_stack.py
@@ -47,8 +47,8 @@ class DUT(Module):
         self.submodules.phy0 = ttl_simple.Output(self.ttl0)
         self.submodules.phy1 = ttl_simple.Output(self.ttl1)
         rtio_channels = [
-            rtio.Channel.from_phy(self.phy0),
-            rtio.Channel.from_phy(self.phy1)
+            rtio.Channel.from_phy(self.phy0, ofifo_depth=4),
+            rtio.Channel.from_phy(self.phy1, ofifo_depth=4)
         ]
         self.submodules.satellite = DRTIOSatellite(
             self.transceivers.bob, rx_synchronizer, rtio_channels)
@@ -79,6 +79,7 @@ class TestFullStack(unittest.TestCase):
             yield from kcsrs.o_we.write(1)
             yield
             status = 1
+            wlen = 0
             while status:
                 status = yield from kcsrs.o_status.read()
                 if status & 2:
@@ -88,6 +89,8 @@ class TestFullStack(unittest.TestCase):
                     yield from kcsrs.o_sequence_error_reset.write(1)
                     raise RTIOSequenceError
                 yield
+                wlen += 1
+            return wlen
 
         def test():
             yield from get_fifo_space(0)
@@ -109,9 +112,28 @@ class TestFullStack(unittest.TestCase):
                 yield from write(0, 1)
             delay(200*8)
 
-            for _ in range(50):
+            delay(200*8)
+            max_wlen = 0
+            for _ in range(3):
+                wlen = yield from write(0, 1)
+                max_wlen = max(max_wlen, wlen)
+                delay(40*8)
+                wlen = yield from write(0, 0)
+                max_wlen = max(max_wlen, wlen)
+                delay(40*8)
+            # check that some writes caused FIFO space requests
+            self.assertGreater(max_wlen, 5)
+
+            # wait for all TTL events to execute
+            for _ in range(40):
                 yield
 
+            # check "last timestamp passed" FIFO empty condition
+            delay(1000*8)
+            wlen = yield from write(0, 1)
+            self.assertEqual(wlen, 2)
+
+
         ttl_changes = []
         @passive
         def check_ttls():
@@ -128,10 +150,17 @@ class TestFullStack(unittest.TestCase):
 
         run_simulation(dut, 
             {"sys": test(), "rtio": check_ttls()},
-            {"sys": 8, "rtio": 5, "rtio_rx": 5, "rio": 5, "rio_phy": 5}, vcd_name="foo.vcd")
+            {"sys": 8, "rtio": 5, "rtio_rx": 5, "rio": 5, "rio_phy": 5})
         self.assertEqual(ttl_changes, [
             (203, 0),
             (208, 0),
             (208, 1),
-            (214, 1)
+            (214, 1),
+
+            (414, 0),
+            (454, 0),
+            (494, 0),
+            (534, 0),
+            (574, 0),
+            (614, 0)
         ])

From 45621934fd404fc5374c89b4d2de4824ca094bf7 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Wed, 26 Oct 2016 22:03:05 +0800
Subject: [PATCH 043/134] drtio: forward errors to CSR

---
 artiq/gateware/drtio/rt_controller.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/artiq/gateware/drtio/rt_controller.py b/artiq/gateware/drtio/rt_controller.py
index e0efafcd7..fca4d75f4 100644
--- a/artiq/gateware/drtio/rt_controller.py
+++ b/artiq/gateware/drtio/rt_controller.py
@@ -32,6 +32,9 @@ class _KernelCSRs(AutoCSR):
         self.o_dbg_last_timestamp = CSRStatus(64)
         self.o_reset_channel_status = CSR()
 
+        self.err_present = CSR()
+        self.err_code = CSRStatus(8)
+
 
 class RTController(Module):
     def __init__(self, rt_packets, channel_count, fine_ts_width):
@@ -164,5 +167,11 @@ class RTController(Module):
             )
         ]
 
+        self.comb += [
+            self.kcsrs.err_present.w.eq(rt_packets.error_not),
+            rt_packets.error_not_ack.eq(self.kcsrs.err_present.re),
+            self.kcsrs.err_code.status.eq(rt_packets.error_code)
+        ]
+
     def get_csrs(self):
         return self.kcsrs.get_csrs()

From 929a7650a8280462bf8c767bd27409a7973b07b2 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Wed, 26 Oct 2016 22:03:44 +0800
Subject: [PATCH 044/134] drtio: fixes

---
 artiq/gateware/drtio/iot.py           | 11 ++++++-----
 artiq/gateware/drtio/rt_controller.py |  5 ++---
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/artiq/gateware/drtio/iot.py b/artiq/gateware/drtio/iot.py
index d7ba41c78..21e362367 100644
--- a/artiq/gateware/drtio/iot.py
+++ b/artiq/gateware/drtio/iot.py
@@ -48,17 +48,18 @@ class IOT(Module):
             # FIFO write
             self.comb += fifo.we.eq(rt_packets.write_stb
                                     & (rt_packets.write_channel == n))
-            self.sync += \
+            self.sync += [
+                If(rt_packets.write_overflow_ack,
+                    rt_packets.write_overflow.eq(0)),
+                If(rt_packets.write_underflow_ack,
+                    rt_packets.write_underflow.eq(0)),
                 If(fifo.we,
-                    If(rt_packets.write_overflow_ack,
-                        rt_packets.write_overflow.eq(0)),
                     If(~fifo.writable, rt_packets.write_overflow.eq(1)),
-                    If(rt_packets.write_underflow_ack,
-                        rt_packets.write_underflow.eq(0)),
                     If(rt_packets.write_timestamp[max_fine_ts_width:] < (tsc + 4),
                         rt_packets.write_underflow.eq(1)
                     )
                 )
+            ]
             if data_width:
                 self.comb += fifo_in.data.eq(rt_packets.write_data)
             if address_width:
diff --git a/artiq/gateware/drtio/rt_controller.py b/artiq/gateware/drtio/rt_controller.py
index fca4d75f4..a7ba5f2b6 100644
--- a/artiq/gateware/drtio/rt_controller.py
+++ b/artiq/gateware/drtio/rt_controller.py
@@ -48,7 +48,7 @@ class RTController(Module):
         self.comb += [ 
             rt_packets.tsc_value.eq(
                 self.counter.value_rtio + tsc_correction),
-            self.kcsrs.set_time.r.eq(rt_packets.set_time_stb)
+            self.kcsrs.set_time.w.eq(rt_packets.set_time_stb)
         ]
         self.sync += [
             If(rt_packets.set_time_ack, rt_packets.set_time_stb.eq(0)),
@@ -98,8 +98,7 @@ class RTController(Module):
         cond_sequence_error = self.kcsrs.o_timestamp.storage < last_timestamps.dat_r
         cond_underflow = ((self.kcsrs.o_timestamp.storage[fine_ts_width:]
                            - self.kcsrs.underflow_margin.storage[fine_ts_width:]) < self.counter.value_sys)
-        cond_fifo_emptied = ((last_timestamps.dat_r[fine_ts_width:]
-                              < self.counter.value_sys - self.kcsrs.underflow_margin.storage[fine_ts_width:])
+        cond_fifo_emptied = ((last_timestamps.dat_r[fine_ts_width:] < self.counter.value_sys)
                              & (last_timestamps.dat_r != 0))
 
         fsm.act("IDLE",

From 9bbc6eb0efc574de6e8534383213e95f191ab781 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Wed, 26 Oct 2016 22:04:32 +0800
Subject: [PATCH 045/134] drtio: more full stack testing

---
 artiq/test/gateware/drtio/test_full_stack.py | 72 ++++++++++++++------
 1 file changed, 53 insertions(+), 19 deletions(-)

diff --git a/artiq/test/gateware/drtio/test_full_stack.py b/artiq/test/gateware/drtio/test_full_stack.py
index 002b9bbc0..e121aa716 100644
--- a/artiq/test/gateware/drtio/test_full_stack.py
+++ b/artiq/test/gateware/drtio/test_full_stack.py
@@ -59,6 +59,21 @@ class TestFullStack(unittest.TestCase):
         dut = DUT(2)
         kcsrs = dut.master.rt_controller.kcsrs
 
+        ttl_changes = []
+        correct_ttl_changes = [
+            (203, 0),
+            (208, 0),
+            (208, 1),
+            (214, 1),
+
+            (414, 0),
+            (454, 0),
+            (494, 0),
+            (534, 0),
+            (574, 0),
+            (614, 0)
+        ]
+
         now = 0
         def delay(dt):
             nonlocal now
@@ -92,13 +107,15 @@ class TestFullStack(unittest.TestCase):
                 wlen += 1
             return wlen
 
-        def test():
+        def test_init():
             yield from get_fifo_space(0)
             yield from get_fifo_space(1)
 
+        def test_underflow():
             with self.assertRaises(RTIOUnderflow):
                 yield from write(0, 0)
-            
+
+        def test_pulses():
             delay(200*8)
             yield from write(0, 1)
             delay(5*8)
@@ -107,11 +124,13 @@ class TestFullStack(unittest.TestCase):
             delay(6*8)
             yield from write(1, 0)
 
+        def test_sequence_error():
             delay(-200*8)
             with self.assertRaises(RTIOSequenceError):
                 yield from write(0, 1)
             delay(200*8)
 
+        def test_fifo_space():
             delay(200*8)
             max_wlen = 0
             for _ in range(3):
@@ -124,17 +143,44 @@ class TestFullStack(unittest.TestCase):
             # check that some writes caused FIFO space requests
             self.assertGreater(max_wlen, 5)
 
+        def test_fifo_emptied():
             # wait for all TTL events to execute
-            for _ in range(40):
+            while len(ttl_changes) < len(correct_ttl_changes):
                 yield
-
             # check "last timestamp passed" FIFO empty condition
             delay(1000*8)
             wlen = yield from write(0, 1)
             self.assertEqual(wlen, 2)
 
+        def test_tsc_error():
+            err_present = yield from kcsrs.err_present.read()
+            self.assertEqual(err_present, 0)
+            yield from kcsrs.tsc_correction.write(10000000)
+            yield from kcsrs.set_time.write(1)
+            for i in range(5):
+               yield
+            delay(10000)
+            yield from write(0, 1)
+            for i in range(10):
+               yield
+            err_present = yield from kcsrs.err_present.read()
+            err_code = yield from kcsrs.err_code.read()
+            self.assertEqual(err_present, 1)
+            self.assertEqual(err_code, 2)
+            yield from kcsrs.err_present.write(1)
+            yield
+            err_present = yield from kcsrs.err_present.read()
+            self.assertEqual(err_present, 0)
+
+        def test():
+            yield from test_init()
+            yield from test_underflow()
+            yield from test_pulses()
+            yield from test_sequence_error()
+            yield from test_fifo_space()
+            yield from test_fifo_emptied()
+            yield from test_tsc_error()
 
-        ttl_changes = []
         @passive
         def check_ttls():
             cycle = 0
@@ -150,17 +196,5 @@ class TestFullStack(unittest.TestCase):
 
         run_simulation(dut, 
             {"sys": test(), "rtio": check_ttls()},
-            {"sys": 8, "rtio": 5, "rtio_rx": 5, "rio": 5, "rio_phy": 5})
-        self.assertEqual(ttl_changes, [
-            (203, 0),
-            (208, 0),
-            (208, 1),
-            (214, 1),
-
-            (414, 0),
-            (454, 0),
-            (494, 0),
-            (534, 0),
-            (574, 0),
-            (614, 0)
-        ])
+            {"sys": 8, "rtio": 5, "rtio_rx": 5, "rio": 5, "rio_phy": 5}, vcd_name="foo.vcd")
+        self.assertEqual(ttl_changes, correct_ttl_changes)

From 4f6241283c2504977983c08821ebfe29a06e2620 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Sat, 29 Oct 2016 16:37:53 +0800
Subject: [PATCH 046/134] drtio: always use NoRetiming on MultiReg inputs

---
 artiq/gateware/drtio/link_layer.py    | 4 +++-
 artiq/gateware/drtio/rt_controller.py | 7 +++++--
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/artiq/gateware/drtio/link_layer.py b/artiq/gateware/drtio/link_layer.py
index 46626ab77..bf5f771ec 100644
--- a/artiq/gateware/drtio/link_layer.py
+++ b/artiq/gateware/drtio/link_layer.py
@@ -3,7 +3,7 @@ from operator import xor, or_
 
 from migen import *
 from migen.genlib.fsm import *
-from migen.genlib.cdc import MultiReg
+from migen.genlib.cdc import MultiReg, NoRetiming
 
 
 class Scrambler(Module):
@@ -248,7 +248,9 @@ class LinkLayer(Module):
         rx_remote_rx_ready = Signal()
         rx_link_init = Signal()
         self.specials += [
+            NoRetiming(rx.remote_rx_ready),
             MultiReg(rx.remote_rx_ready, rx_remote_rx_ready, "rtio"),
+            NoRetiming(rx.link_init),
             MultiReg(rx.link_init, rx_link_init, "rtio")
         ]
 
diff --git a/artiq/gateware/drtio/rt_controller.py b/artiq/gateware/drtio/rt_controller.py
index a7ba5f2b6..acee322ee 100644
--- a/artiq/gateware/drtio/rt_controller.py
+++ b/artiq/gateware/drtio/rt_controller.py
@@ -1,5 +1,5 @@
 from migen import *
-from migen.genlib.cdc import MultiReg
+from migen.genlib.cdc import MultiReg, NoRetiming
 
 from misoc.interconnect.csr import *
 
@@ -44,7 +44,10 @@ class RTController(Module):
         self.sync += If(self.kcsrs.counter_update.re, 
                         self.kcsrs.counter.status.eq(self.counter.value_sys))
         tsc_correction = Signal(64)
-        self.specials += MultiReg(self.kcsrs.tsc_correction.storage, tsc_correction)
+        self.specials += [
+            NoRetiming(self.kcsrs.tsc_correction.storage),
+            MultiReg(self.kcsrs.tsc_correction.storage, tsc_correction)
+        ]
         self.comb += [ 
             rt_packets.tsc_value.eq(
                 self.counter.value_rtio + tsc_correction),

From 95def81c03e8bb72f9276f04dfe4796add455a93 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Sat, 29 Oct 2016 17:05:30 +0800
Subject: [PATCH 047/134] drtio: squelch frame signals until link layer ready

---
 artiq/gateware/drtio/link_layer.py           | 16 ++++++++++++++--
 artiq/test/gateware/drtio/test_full_stack.py |  3 +++
 2 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/artiq/gateware/drtio/link_layer.py b/artiq/gateware/drtio/link_layer.py
index bf5f771ec..a1946e228 100644
--- a/artiq/gateware/drtio/link_layer.py
+++ b/artiq/gateware/drtio/link_layer.py
@@ -232,11 +232,23 @@ class LinkLayer(Module):
 
         # in rtio_rx clock domain
         self.rx_aux_stb = rx.aux_stb
-        self.rx_aux_frame = rx.aux_frame
+        self.rx_aux_frame = Signal()
         self.rx_aux_data = rx.aux_data
-        self.rx_rt_frame = rx.rt_frame
+        self.rx_rt_frame = Signal()
         self.rx_rt_data = rx.rt_data
 
+        ready_r = Signal()
+        ready_rx = Signal()
+        self.sync.rtio += ready_r.eq(self.ready)
+        self.specials += [
+            NoRetiming(ready_r),
+            MultiReg(ready_r, ready_rx, "rtio_rx")
+        ]
+        self.comb += [
+            self.rx_aux_frame.eq(rx.aux_frame & ready_rx),
+            self.rx_rt_frame.eq(rx.rt_frame & ready_rx),
+        ]
+
         # # #
 
         fsm = ClockDomainsRenamer("rtio")(
diff --git a/artiq/test/gateware/drtio/test_full_stack.py b/artiq/test/gateware/drtio/test_full_stack.py
index e121aa716..22d1eb068 100644
--- a/artiq/test/gateware/drtio/test_full_stack.py
+++ b/artiq/test/gateware/drtio/test_full_stack.py
@@ -173,6 +173,9 @@ class TestFullStack(unittest.TestCase):
             self.assertEqual(err_present, 0)
 
         def test():
+            while not (yield dut.master.link_layer.ready):
+                yield
+
             yield from test_init()
             yield from test_underflow()
             yield from test_pulses()

From 7c05dccf65804bbfd4df703e3d30153e35522456 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Sat, 29 Oct 2016 17:30:29 +0800
Subject: [PATCH 048/134] drtio: add support for 125MHz clock on
 GTX_1000BASE_BX10

---
 .../gateware/drtio/transceiver/gtx_7series.py | 25 +++++++++++++------
 1 file changed, 18 insertions(+), 7 deletions(-)

diff --git a/artiq/gateware/drtio/transceiver/gtx_7series.py b/artiq/gateware/drtio/transceiver/gtx_7series.py
index 70e54b8b0..469d3f96f 100644
--- a/artiq/gateware/drtio/transceiver/gtx_7series.py
+++ b/artiq/gateware/drtio/transceiver/gtx_7series.py
@@ -10,7 +10,10 @@ from artiq.gateware.drtio.transceiver.gtx_7series_init import *
 class GTX_1000BASE_BX10(Module):
     rtio_clk_freq = 62.5e6
 
-    def __init__(self, clock_pads, tx_pads, rx_pads, sys_clk_freq):
+    # The transceiver clock on clock_pads must be 62.5MHz
+    # when clock_div2=False, and 125MHz when clock_div2=True.
+    def __init__(self, clock_pads, tx_pads, rx_pads, sys_clk_freq,
+                 clock_div2=False):
         self.submodules.encoder = ClockDomainsRenamer("rtio")(
             Encoder(2, True))
         self.decoders = [ClockDomainsRenamer("rtio_rx")(
@@ -23,12 +26,20 @@ class GTX_1000BASE_BX10(Module):
         # # #
 
         refclk = Signal()
-        self.specials += Instance("IBUFDS_GTE2",
-            i_CEB=0,
-            i_I=clock_pads.p,
-            i_IB=clock_pads.n,
-            o_O=refclk
-        )
+        if clock_div2:
+            self.specials += Instance("IBUFDS_GTE2",
+                i_CEB=0,
+                i_I=clock_pads.p,
+                i_IB=clock_pads.n,
+                o_ODIV2=refclk
+            )
+        else:
+            self.specials += Instance("IBUFDS_GTE2",
+                i_CEB=0,
+                i_I=clock_pads.p,
+                i_IB=clock_pads.n,
+                o_O=refclk
+            )
 
         cplllock = Signal()
         # TX generates RTIO clock, init must be in system domain

From da5208e16017e63043bcd21ea0ffd3046ba19ab8 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Sat, 29 Oct 2016 17:31:15 +0800
Subject: [PATCH 049/134] drtio: add master gateware target

---
 artiq/gateware/targets/kc705_drtio_master.py  | 71 +++++++++++++++++++
 .../gateware/targets/kc705_drtio_satellite.py |  0
 2 files changed, 71 insertions(+)
 create mode 100755 artiq/gateware/targets/kc705_drtio_master.py
 mode change 100644 => 100755 artiq/gateware/targets/kc705_drtio_satellite.py

diff --git a/artiq/gateware/targets/kc705_drtio_master.py b/artiq/gateware/targets/kc705_drtio_master.py
new file mode 100755
index 000000000..ecbf7b7ff
--- /dev/null
+++ b/artiq/gateware/targets/kc705_drtio_master.py
@@ -0,0 +1,71 @@
+#!/usr/bin/env python3.5
+
+import argparse
+
+from migen import *
+from migen.fhdl.specials import Keep
+
+from misoc.targets.kc705 import MiniSoC, soc_kc705_args, soc_kc705_argdict
+from misoc.integration.builder import builder_args, builder_argdict
+
+from artiq.gateware.soc import AMPSoC, build_artiq_soc
+from artiq.gateware.drtio.transceiver import gtx_7series
+from artiq.gateware.drtio import DRTIOMaster
+from artiq import __version__ as artiq_version
+
+
+class Master(MiniSoC, AMPSoC):
+    mem_map = {
+        "timer_kernel":  0x10000000, # (shadow @0x90000000)
+        "drtio":         0x20000000, # (shadow @0xa0000000)
+        "mailbox":       0x70000000  # (shadow @0xf0000000)
+    }
+    mem_map.update(MiniSoC.mem_map)
+
+    def __init__(self, **kwargs):
+        MiniSoC.__init__(self,
+                         cpu_type="or1k",
+                         sdram_controller_type="minicon",
+                         l2_size=128*1024,
+                         with_timer=False,
+                         ident=artiq_version,
+                         **kwargs)
+        AMPSoC.__init__(self)
+
+        platform = self.platform
+
+        self.comb += platform.request("sfp_tx_disable_n").eq(1)
+        self.submodules.transceiver = gtx_7series.GTX_1000BASE_BX10(
+            clock_pads=platform.request("sgmii_clock"),
+            tx_pads=platform.request("sfp_tx"),
+            rx_pads=platform.request("sfp_rx"),
+            sys_clk_freq=self.clk_freq,
+            clock_div2=True)
+        self.submodules.drtio = DRTIOMaster(self.transceiver)
+        self.register_kernel_cpu_csrdevice("drtio")
+
+        self.specials += [
+            Keep(self.ethphy.crg.cd_eth_rx.clk),
+            Keep(self.ethphy.crg.cd_eth_tx.clk),
+        ]
+
+        platform.add_period_constraint(self.ethphy.crg.cd_eth_rx.clk, 8.)
+        platform.add_period_constraint(self.ethphy.crg.cd_eth_tx.clk, 8.)
+        platform.add_false_path_constraints(
+            self.ethphy.crg.cd_eth_rx.clk,
+            self.ethphy.crg.cd_eth_tx.clk)
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="ARTIQ with DRTIO on KC705 - Master")
+    builder_args(parser)
+    soc_kc705_args(parser)
+    args = parser.parse_args()
+
+    soc = Master(**soc_kc705_argdict(args))
+    build_artiq_soc(soc, builder_argdict(args))
+
+
+if __name__ == "__main__":
+    main()
diff --git a/artiq/gateware/targets/kc705_drtio_satellite.py b/artiq/gateware/targets/kc705_drtio_satellite.py
old mode 100644
new mode 100755

From 9aa94e1a2d6cda70be080cd5df90d0e6e745f07b Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Mon, 31 Oct 2016 00:53:01 +0800
Subject: [PATCH 050/134] adapt to migen/misoc changes

---
 artiq/gateware/drtio/link_layer.py           | 12 +++++-------
 artiq/gateware/drtio/rt_controller.py        |  8 +++-----
 artiq/gateware/drtio/rt_packets.py           |  6 +++---
 artiq/gateware/rtio/cdc.py                   |  6 ++----
 artiq/gateware/targets/kc705.py              | 18 +++---------------
 artiq/gateware/targets/kc705_drtio_master.py | 12 ------------
 6 files changed, 16 insertions(+), 46 deletions(-)

diff --git a/artiq/gateware/drtio/link_layer.py b/artiq/gateware/drtio/link_layer.py
index a1946e228..82019092a 100644
--- a/artiq/gateware/drtio/link_layer.py
+++ b/artiq/gateware/drtio/link_layer.py
@@ -3,7 +3,7 @@ from operator import xor, or_
 
 from migen import *
 from migen.genlib.fsm import *
-from migen.genlib.cdc import MultiReg, NoRetiming
+from migen.genlib.cdc import MultiReg
 
 
 class Scrambler(Module):
@@ -240,10 +240,8 @@ class LinkLayer(Module):
         ready_r = Signal()
         ready_rx = Signal()
         self.sync.rtio += ready_r.eq(self.ready)
-        self.specials += [
-            NoRetiming(ready_r),
-            MultiReg(ready_r, ready_rx, "rtio_rx")
-        ]
+        ready_r.attr.add("no_retiming")
+        self.specials += MultiReg(ready_r, ready_rx, "rtio_rx")
         self.comb += [
             self.rx_aux_frame.eq(rx.aux_frame & ready_rx),
             self.rx_rt_frame.eq(rx.rt_frame & ready_rx),
@@ -259,10 +257,10 @@ class LinkLayer(Module):
 
         rx_remote_rx_ready = Signal()
         rx_link_init = Signal()
+        rx.remote_rx_ready.attr.add("no_retiming")
+        rx.link_init.attr.add("no_retiming")
         self.specials += [
-            NoRetiming(rx.remote_rx_ready),
             MultiReg(rx.remote_rx_ready, rx_remote_rx_ready, "rtio"),
-            NoRetiming(rx.link_init),
             MultiReg(rx.link_init, rx_link_init, "rtio")
         ]
 
diff --git a/artiq/gateware/drtio/rt_controller.py b/artiq/gateware/drtio/rt_controller.py
index acee322ee..753b04f7b 100644
--- a/artiq/gateware/drtio/rt_controller.py
+++ b/artiq/gateware/drtio/rt_controller.py
@@ -1,5 +1,5 @@
 from migen import *
-from migen.genlib.cdc import MultiReg, NoRetiming
+from migen.genlib.cdc import MultiReg
 
 from misoc.interconnect.csr import *
 
@@ -44,10 +44,8 @@ class RTController(Module):
         self.sync += If(self.kcsrs.counter_update.re, 
                         self.kcsrs.counter.status.eq(self.counter.value_sys))
         tsc_correction = Signal(64)
-        self.specials += [
-            NoRetiming(self.kcsrs.tsc_correction.storage),
-            MultiReg(self.kcsrs.tsc_correction.storage, tsc_correction)
-        ]
+        self.kcsrs.tsc_correction.storage.attr.add("no_retiming")
+        self.specials += MultiReg(self.kcsrs.tsc_correction.storage, tsc_correction)
         self.comb += [ 
             rt_packets.tsc_value.eq(
                 self.counter.value_rtio + tsc_correction),
diff --git a/artiq/gateware/drtio/rt_packets.py b/artiq/gateware/drtio/rt_packets.py
index 3a3818f48..f756fb70a 100644
--- a/artiq/gateware/drtio/rt_packets.py
+++ b/artiq/gateware/drtio/rt_packets.py
@@ -3,7 +3,7 @@ from types import SimpleNamespace
 from migen import *
 from migen.genlib.fsm import *
 from migen.genlib.fifo import AsyncFIFO
-from migen.genlib.cdc import PulseSynchronizer, NoRetiming
+from migen.genlib.cdc import PulseSynchronizer
 
 
 def layout_len(l):
@@ -334,7 +334,7 @@ class _CrossDomainRequest(Module):
         ]
         if req_data is not None:
             req_data_r = Signal.like(req_data)
-            self.specials += NoRetiming(req_data_r)
+            req_data_r.attr.add("no_retiming")
             self.sync += If(req_stb, req_data_r.eq(req_data))
         dsync += [
             If(request.o, srv_stb.eq(1)),
@@ -350,7 +350,7 @@ class _CrossDomainNotification(Module):
                  emi_stb, emi_data,
                  rec_stb, rec_ack, rec_data):
         emi_data_r = Signal.like(emi_data)
-        self.specials += NoRetiming(emi_data_r)
+        emi_data_r.attr.add("no_retiming")
         dsync = getattr(self.sync, domain)
         dsync += If(emi_stb, emi_data_r.eq(emi_data))
 
diff --git a/artiq/gateware/rtio/cdc.py b/artiq/gateware/rtio/cdc.py
index 7b31eedcd..bc561ebbf 100644
--- a/artiq/gateware/rtio/cdc.py
+++ b/artiq/gateware/rtio/cdc.py
@@ -18,10 +18,8 @@ class GrayCodeTransfer(Module):
         self.sync.rtio += value_gray_rtio.eq(self.i ^ self.i[1:])
         # transfer to system clock domain
         value_gray_sys = Signal(width)
-        self.specials += [
-            NoRetiming(value_gray_rtio),
-            MultiReg(value_gray_rtio, value_gray_sys)
-        ]
+        value_gray_rtio.attr.add("no_retiming")
+        self.specials += MultiReg(value_gray_rtio, value_gray_sys)
         # convert back to binary
         value_sys = Signal(width)
         self.comb += value_sys[-1].eq(value_gray_sys[-1])
diff --git a/artiq/gateware/targets/kc705.py b/artiq/gateware/targets/kc705.py
index 7209a2b3b..ea38273d4 100755
--- a/artiq/gateware/targets/kc705.py
+++ b/artiq/gateware/targets/kc705.py
@@ -8,7 +8,6 @@ from migen.genlib.cdc import MultiReg
 from migen.build.generic_platform import *
 from migen.build.xilinx.vivado import XilinxVivadoToolchain
 from migen.build.xilinx.ise import XilinxISEToolchain
-from migen.fhdl.specials import Keep
 
 from misoc.interconnect.csr import *
 from misoc.interconnect import wishbone
@@ -147,22 +146,11 @@ class _NIST_Ions(MiniSoC, AMPSoC):
         self.submodules.rtio_moninj = rtio.MonInj(rtio_channels)
         self.csr_devices.append("rtio_moninj")
 
-        self.specials += [
-            Keep(self.rtio.cd_rsys.clk),
-            Keep(self.rtio_crg.cd_rtio.clk),
-            Keep(self.ethphy.crg.cd_eth_rx.clk),
-            Keep(self.ethphy.crg.cd_eth_tx.clk),
-        ]
-
-        self.platform.add_period_constraint(self.rtio.cd_rsys.clk, 8.)
+        self.rtio_crg.cd_rtio.clk.attr.add("keep")
         self.platform.add_period_constraint(self.rtio_crg.cd_rtio.clk, 8.)
-        self.platform.add_period_constraint(self.ethphy.crg.cd_eth_rx.clk, 8.)
-        self.platform.add_period_constraint(self.ethphy.crg.cd_eth_tx.clk, 8.)
         self.platform.add_false_path_constraints(
-            self.rtio.cd_rsys.clk,
-            self.rtio_crg.cd_rtio.clk,
-            self.ethphy.crg.cd_eth_rx.clk,
-            self.ethphy.crg.cd_eth_tx.clk)
+            self.crg.cd_sys.clk,
+            self.rtio_crg.cd_rtio.clk)
 
         self.submodules.rtio_analyzer = rtio.Analyzer(self.rtio,
             self.get_native_sdram_if())
diff --git a/artiq/gateware/targets/kc705_drtio_master.py b/artiq/gateware/targets/kc705_drtio_master.py
index ecbf7b7ff..93118d70e 100755
--- a/artiq/gateware/targets/kc705_drtio_master.py
+++ b/artiq/gateware/targets/kc705_drtio_master.py
@@ -3,7 +3,6 @@
 import argparse
 
 from migen import *
-from migen.fhdl.specials import Keep
 
 from misoc.targets.kc705 import MiniSoC, soc_kc705_args, soc_kc705_argdict
 from misoc.integration.builder import builder_args, builder_argdict
@@ -44,17 +43,6 @@ class Master(MiniSoC, AMPSoC):
         self.submodules.drtio = DRTIOMaster(self.transceiver)
         self.register_kernel_cpu_csrdevice("drtio")
 
-        self.specials += [
-            Keep(self.ethphy.crg.cd_eth_rx.clk),
-            Keep(self.ethphy.crg.cd_eth_tx.clk),
-        ]
-
-        platform.add_period_constraint(self.ethphy.crg.cd_eth_rx.clk, 8.)
-        platform.add_period_constraint(self.ethphy.crg.cd_eth_tx.clk, 8.)
-        platform.add_false_path_constraints(
-            self.ethphy.crg.cd_eth_rx.clk,
-            self.ethphy.crg.cd_eth_tx.clk)
-
 
 def main():
     parser = argparse.ArgumentParser(

From 07ad00c1cae93af817adbd3d4c95d292cf192e33 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Mon, 31 Oct 2016 18:09:36 +0800
Subject: [PATCH 051/134] drtio: split kernel/system CSRs

---
 artiq/gateware/drtio/core.py                 |  3 ++
 artiq/gateware/drtio/rt_controller.py        | 56 +++++++++++++-------
 artiq/gateware/soc.py                        |  5 +-
 artiq/gateware/targets/kc705_drtio_master.py |  5 +-
 artiq/test/gateware/drtio/test_full_stack.py | 26 +++++----
 5 files changed, 62 insertions(+), 33 deletions(-)

diff --git a/artiq/gateware/drtio/core.py b/artiq/gateware/drtio/core.py
index af467156d..45428e514 100644
--- a/artiq/gateware/drtio/core.py
+++ b/artiq/gateware/drtio/core.py
@@ -56,5 +56,8 @@ class DRTIOMaster(Module):
         self.submodules.rt_controller = rt_controller.RTController(
             self.rt_packets, channel_count, fine_ts_width)
 
+    def get_kernel_csrs(self):
+        return self.rt_controller.get_kernel_csrs()
+
     def get_csrs(self):
         return self.rt_controller.get_csrs()
diff --git a/artiq/gateware/drtio/rt_controller.py b/artiq/gateware/drtio/rt_controller.py
index 753b04f7b..003d12951 100644
--- a/artiq/gateware/drtio/rt_controller.py
+++ b/artiq/gateware/drtio/rt_controller.py
@@ -23,6 +23,12 @@ class _KernelCSRs(AutoCSR):
         self.counter = CSRStatus(64)
         self.counter_update = CSR()
 
+
+class _CSRs(AutoCSR):
+    def __init__(self):
+        self.chan_sel_override = CSRStorage(16)
+        self.chan_sel_override_en = CSRStorage()
+
         self.tsc_correction = CSRStorage(64)
         self.set_time = CSR()
         self.underflow_margin = CSRStorage(16, reset=200)
@@ -31,6 +37,7 @@ class _KernelCSRs(AutoCSR):
         self.o_dbg_fifo_space = CSRStatus(16)
         self.o_dbg_last_timestamp = CSRStatus(64)
         self.o_reset_channel_status = CSR()
+        self.o_wait = CSRStatus()
 
         self.err_present = CSR()
         self.err_code = CSRStatus(8)
@@ -39,21 +46,28 @@ class _KernelCSRs(AutoCSR):
 class RTController(Module):
     def __init__(self, rt_packets, channel_count, fine_ts_width):
         self.kcsrs = _KernelCSRs()
+        self.csrs = _CSRs()
+
+        chan_sel = Signal(16)
+        self.comb += chan_sel.eq(
+            Mux(self.csrs.chan_sel_override_en.storage,
+                self.csrs.chan_sel_override.storage,
+                self.kcsrs.chan_sel.storage))
 
         self.submodules.counter = RTIOCounter(64-fine_ts_width)
         self.sync += If(self.kcsrs.counter_update.re, 
                         self.kcsrs.counter.status.eq(self.counter.value_sys))
         tsc_correction = Signal(64)
-        self.kcsrs.tsc_correction.storage.attr.add("no_retiming")
-        self.specials += MultiReg(self.kcsrs.tsc_correction.storage, tsc_correction)
+        self.csrs.tsc_correction.storage.attr.add("no_retiming")
+        self.specials += MultiReg(self.csrs.tsc_correction.storage, tsc_correction)
         self.comb += [ 
             rt_packets.tsc_value.eq(
                 self.counter.value_rtio + tsc_correction),
-            self.kcsrs.set_time.w.eq(rt_packets.set_time_stb)
+            self.csrs.set_time.w.eq(rt_packets.set_time_stb)
         ]
         self.sync += [
             If(rt_packets.set_time_ack, rt_packets.set_time_stb.eq(0)),
-            If(self.kcsrs.set_time.re, rt_packets.set_time_stb.eq(1))
+            If(self.csrs.set_time.re, rt_packets.set_time_stb.eq(1))
         ]
 
         fifo_spaces_mem = Memory(16, channel_count)
@@ -65,10 +79,10 @@ class RTController(Module):
 
         rt_packets_fifo_request = Signal()
         self.comb += [
-            fifo_spaces.adr.eq(self.kcsrs.chan_sel.storage),
-            last_timestamps.adr.eq(self.kcsrs.chan_sel.storage),
+            fifo_spaces.adr.eq(chan_sel),
+            last_timestamps.adr.eq(chan_sel),
             last_timestamps.dat_w.eq(self.kcsrs.o_timestamp.storage),
-            rt_packets.write_channel.eq(self.kcsrs.chan_sel.storage),
+            rt_packets.write_channel.eq(chan_sel),
             rt_packets.write_address.eq(self.kcsrs.o_address.storage),
             rt_packets.write_data.eq(self.kcsrs.o_data.storage),
             If(rt_packets_fifo_request,
@@ -84,8 +98,11 @@ class RTController(Module):
         status_wait = Signal()
         status_underflow = Signal()
         status_sequence_error = Signal()
-        self.comb += self.kcsrs.o_status.status.eq(Cat(
-            status_wait, status_underflow, status_sequence_error))
+        self.comb += [
+            self.kcsrs.o_status.status.eq(Cat(
+                status_wait, status_underflow, status_sequence_error)),
+            self.csrs.o_wait.status.eq(status_wait)
+        ]
         sequence_error_set = Signal()
         underflow_set = Signal()
         self.sync += [
@@ -98,7 +115,7 @@ class RTController(Module):
         # TODO: collision, replace, busy
         cond_sequence_error = self.kcsrs.o_timestamp.storage < last_timestamps.dat_r
         cond_underflow = ((self.kcsrs.o_timestamp.storage[fine_ts_width:]
-                           - self.kcsrs.underflow_margin.storage[fine_ts_width:]) < self.counter.value_sys)
+                           - self.csrs.underflow_margin.storage[fine_ts_width:]) < self.counter.value_sys)
         cond_fifo_emptied = ((last_timestamps.dat_r[fine_ts_width:] < self.counter.value_sys)
                              & (last_timestamps.dat_r != 0))
 
@@ -112,7 +129,7 @@ class RTController(Module):
                     NextState("WRITE")
                 )
             ),
-            If(self.kcsrs.o_get_fifo_space.re,
+            If(self.csrs.o_get_fifo_space.re,
                 NextState("GET_FIFO_SPACE")
             )
         )
@@ -157,9 +174,9 @@ class RTController(Module):
         )
 
         self.comb += [
-            self.kcsrs.o_dbg_fifo_space.status.eq(fifo_spaces.dat_r),
-            self.kcsrs.o_dbg_last_timestamp.status.eq(last_timestamps.dat_r),
-            If(self.kcsrs.o_reset_channel_status.re,
+            self.csrs.o_dbg_fifo_space.status.eq(fifo_spaces.dat_r),
+            self.csrs.o_dbg_last_timestamp.status.eq(last_timestamps.dat_r),
+            If(self.csrs.o_reset_channel_status.re,
                 fifo_spaces.dat_w.eq(0),
                 fifo_spaces.we.eq(1),
                 last_timestamps.dat_w.eq(0),
@@ -168,10 +185,13 @@ class RTController(Module):
         ]
 
         self.comb += [
-            self.kcsrs.err_present.w.eq(rt_packets.error_not),
-            rt_packets.error_not_ack.eq(self.kcsrs.err_present.re),
-            self.kcsrs.err_code.status.eq(rt_packets.error_code)
+            self.csrs.err_present.w.eq(rt_packets.error_not),
+            rt_packets.error_not_ack.eq(self.csrs.err_present.re),
+            self.csrs.err_code.status.eq(rt_packets.error_code)
         ]
 
-    def get_csrs(self):
+    def get_kernel_csrs(self):
         return self.kcsrs.get_csrs()
+
+    def get_csrs(self):
+        return self.csrs.get_csrs()
diff --git a/artiq/gateware/soc.py b/artiq/gateware/soc.py
index cd567b305..b02eeb726 100644
--- a/artiq/gateware/soc.py
+++ b/artiq/gateware/soc.py
@@ -39,8 +39,9 @@ class AMPSoC:
         self.submodules.timer_kernel = timer.Timer()
         self.register_kernel_cpu_csrdevice("timer_kernel")
 
-    def register_kernel_cpu_csrdevice(self, name):
-        csrs = getattr(self, name).get_csrs()
+    def register_kernel_cpu_csrdevice(self, name, csrs=None):
+        if csrs is None:
+            csrs = getattr(self, name).get_csrs()
         bank = wishbone.CSRBank(csrs)
         self.submodules += bank
         self.kernel_cpu.add_wb_slave(mem_decoder(self.mem_map[name]),
diff --git a/artiq/gateware/targets/kc705_drtio_master.py b/artiq/gateware/targets/kc705_drtio_master.py
index 93118d70e..b3d973544 100755
--- a/artiq/gateware/targets/kc705_drtio_master.py
+++ b/artiq/gateware/targets/kc705_drtio_master.py
@@ -16,7 +16,7 @@ from artiq import __version__ as artiq_version
 class Master(MiniSoC, AMPSoC):
     mem_map = {
         "timer_kernel":  0x10000000, # (shadow @0x90000000)
-        "drtio":         0x20000000, # (shadow @0xa0000000)
+        "kdrtio":        0x20000000, # (shadow @0xa0000000)
         "mailbox":       0x70000000  # (shadow @0xf0000000)
     }
     mem_map.update(MiniSoC.mem_map)
@@ -41,7 +41,8 @@ class Master(MiniSoC, AMPSoC):
             sys_clk_freq=self.clk_freq,
             clock_div2=True)
         self.submodules.drtio = DRTIOMaster(self.transceiver)
-        self.register_kernel_cpu_csrdevice("drtio")
+        self.register_kernel_cpu_csrdevice("kdrtio", self.drtio.get_kernel_csrs())
+        self.csr_devices.append("drtio")
 
 
 def main():
diff --git a/artiq/test/gateware/drtio/test_full_stack.py b/artiq/test/gateware/drtio/test_full_stack.py
index 22d1eb068..c27437aa7 100644
--- a/artiq/test/gateware/drtio/test_full_stack.py
+++ b/artiq/test/gateware/drtio/test_full_stack.py
@@ -58,6 +58,7 @@ class TestFullStack(unittest.TestCase):
     def test_full_stack(self):
         dut = DUT(2)
         kcsrs = dut.master.rt_controller.kcsrs
+        csrs = dut.master.rt_controller.csrs
 
         ttl_changes = []
         correct_ttl_changes = [
@@ -80,12 +81,15 @@ class TestFullStack(unittest.TestCase):
             now += dt
 
         def get_fifo_space(channel):
-            yield from kcsrs.chan_sel.write(channel)
-            yield from kcsrs.o_get_fifo_space.write(1)
+            yield from csrs.chan_sel_override_en.write(1)
+            yield from csrs.chan_sel_override.write(channel)
+            yield from csrs.o_get_fifo_space.write(1)
             yield
-            while (yield from kcsrs.o_status.read()) & 1:
+            while (yield from csrs.o_wait.read()):
                 yield
-            return (yield from kcsrs.o_dbg_fifo_space.read())
+            r = (yield from csrs.o_dbg_fifo_space.read())
+            yield from csrs.chan_sel_override_en.write(0)
+            return r
 
         def write(channel, data):
             yield from kcsrs.chan_sel.write(channel)
@@ -153,23 +157,23 @@ class TestFullStack(unittest.TestCase):
             self.assertEqual(wlen, 2)
 
         def test_tsc_error():
-            err_present = yield from kcsrs.err_present.read()
+            err_present = yield from csrs.err_present.read()
             self.assertEqual(err_present, 0)
-            yield from kcsrs.tsc_correction.write(10000000)
-            yield from kcsrs.set_time.write(1)
+            yield from csrs.tsc_correction.write(10000000)
+            yield from csrs.set_time.write(1)
             for i in range(5):
                yield
             delay(10000)
             yield from write(0, 1)
             for i in range(10):
                yield
-            err_present = yield from kcsrs.err_present.read()
-            err_code = yield from kcsrs.err_code.read()
+            err_present = yield from csrs.err_present.read()
+            err_code = yield from csrs.err_code.read()
             self.assertEqual(err_present, 1)
             self.assertEqual(err_code, 2)
-            yield from kcsrs.err_present.write(1)
+            yield from csrs.err_present.write(1)
             yield
-            err_present = yield from kcsrs.err_present.read()
+            err_present = yield from csrs.err_present.read()
             self.assertEqual(err_present, 0)
 
         def test():

From 0c1a76d6685a214472119d1e2d8ec888511aa3a6 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Tue, 1 Nov 2016 00:30:16 +0800
Subject: [PATCH 052/134] unify rtio/drtio kernel interface

---
 artiq/gateware/drtio/rt_controller.py | 21 ++-------------
 artiq/gateware/rtio/core.py           | 37 ++-------------------------
 artiq/gateware/rtio/kernel_csrs.py    | 27 +++++++++++++++++++
 3 files changed, 31 insertions(+), 54 deletions(-)
 create mode 100644 artiq/gateware/rtio/kernel_csrs.py

diff --git a/artiq/gateware/drtio/rt_controller.py b/artiq/gateware/drtio/rt_controller.py
index 003d12951..d718393da 100644
--- a/artiq/gateware/drtio/rt_controller.py
+++ b/artiq/gateware/drtio/rt_controller.py
@@ -4,24 +4,7 @@ from migen.genlib.cdc import MultiReg
 from misoc.interconnect.csr import *
 
 from artiq.gateware.rtio.cdc import RTIOCounter
-
-
-class _KernelCSRs(AutoCSR):
-    def __init__(self):
-        # chan_sel must be written at least 1 cycle before we
-        # and held stable until the transaction is complete.
-        # timestamp must be written at least 1 cycle before we.
-        self.chan_sel = CSRStorage(16)
-        self.o_data = CSRStorage(64)
-        self.o_address = CSRStorage(16)
-        self.o_timestamp = CSRStorage(64)
-        self.o_we = CSR()
-        self.o_status = CSRStatus(3)
-        self.o_underflow_reset = CSR()
-        self.o_sequence_error_reset = CSR()
-
-        self.counter = CSRStatus(64)
-        self.counter_update = CSR()
+from artiq.gateware.rtio.kernel_csrs import KernelCSRs
 
 
 class _CSRs(AutoCSR):
@@ -45,7 +28,7 @@ class _CSRs(AutoCSR):
 
 class RTController(Module):
     def __init__(self, rt_packets, channel_count, fine_ts_width):
-        self.kcsrs = _KernelCSRs()
+        self.kcsrs = KernelCSRs()
         self.csrs = _CSRs()
 
         chan_sel = Signal(16)
diff --git a/artiq/gateware/rtio/core.py b/artiq/gateware/rtio/core.py
index a64bace11..43e9dea5d 100644
--- a/artiq/gateware/rtio/core.py
+++ b/artiq/gateware/rtio/core.py
@@ -5,9 +5,9 @@ from migen import *
 from migen.genlib.record import Record
 from migen.genlib.fifo import AsyncFIFO
 from migen.genlib.resetsync import AsyncResetSynchronizer
-from misoc.interconnect.csr import *
 
 from artiq.gateware.rtio import rtlink
+from artiq.gateware.rtio.kernel_csrs import KernelCSRs
 from artiq.gateware.rtio.cdc import *
 
 
@@ -265,36 +265,6 @@ class LogChannel:
         self.overrides = []
 
 
-class _KernelCSRs(AutoCSR):
-    def __init__(self, chan_sel_width,
-                 data_width, address_width, full_ts_width):
-        self.reset = CSRStorage(reset=1)
-        self.reset_phy = CSRStorage(reset=1)
-        self.chan_sel = CSRStorage(chan_sel_width)
-
-        if data_width:
-            self.o_data = CSRStorage(data_width)
-        if address_width:
-            self.o_address = CSRStorage(address_width)
-        self.o_timestamp = CSRStorage(full_ts_width)
-        self.o_we = CSR()
-        self.o_status = CSRStatus(5)
-        self.o_underflow_reset = CSR()
-        self.o_sequence_error_reset = CSR()
-        self.o_collision_reset = CSR()
-        self.o_busy_reset = CSR()
-
-        if data_width:
-            self.i_data = CSRStatus(data_width)
-        self.i_timestamp = CSRStatus(full_ts_width)
-        self.i_re = CSR()
-        self.i_status = CSRStatus(2)
-        self.i_overflow_reset = CSR()
-
-        self.counter = CSRStatus(full_ts_width)
-        self.counter_update = CSR()
-
-
 class RTIO(Module):
     def __init__(self, channels, full_ts_width=63, guard_io_cycles=20):
         data_width = max(rtlink.get_data_width(c.interface)
@@ -308,10 +278,7 @@ class RTIO(Module):
         self.address_width = address_width
         self.fine_ts_width = fine_ts_width
 
-        # CSRs
-        self.kcsrs = _KernelCSRs(bits_for(len(channels)-1),
-                                 data_width, address_width,
-                                 full_ts_width)
+        self.kcsrs = KernelCSRs()
 
         # Clocking/Reset
         # Create rsys, rio and rio_phy domains based on sys and rtio
diff --git a/artiq/gateware/rtio/kernel_csrs.py b/artiq/gateware/rtio/kernel_csrs.py
new file mode 100644
index 000000000..d897a7566
--- /dev/null
+++ b/artiq/gateware/rtio/kernel_csrs.py
@@ -0,0 +1,27 @@
+from misoc.interconnect.csr import *
+
+
+class KernelCSRs(AutoCSR):
+    def __init__(self):
+        self.reset = CSRStorage(reset=1)
+        self.reset_phy = CSRStorage(reset=1)
+        self.chan_sel = CSRStorage(16)
+
+        self.o_data = CSRStorage(32)
+        self.o_address = CSRStorage(16)
+        self.o_timestamp = CSRStorage(64)
+        self.o_we = CSR()
+        self.o_status = CSRStatus(5)
+        self.o_underflow_reset = CSR()
+        self.o_sequence_error_reset = CSR()
+        self.o_collision_reset = CSR()
+        self.o_busy_reset = CSR()
+
+        self.i_data = CSRStatus(32)
+        self.i_timestamp = CSRStatus(64)
+        self.i_re = CSR()
+        self.i_status = CSRStatus(2)
+        self.i_overflow_reset = CSR()
+
+        self.counter = CSRStatus(64)
+        self.counter_update = CSR()

From a6ae25479617c994d0d245627faa551b690fb1ae Mon Sep 17 00:00:00 2001
From: whitequark <whitequark@whitequark.org>
Date: Tue, 1 Nov 2016 16:01:24 +0000
Subject: [PATCH 053/134] Si5324: update to free run from XA/XB, with CKIN1
 having priority.

---
 artiq/gateware/targets/kc705_drtio_satellite.py | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/artiq/gateware/targets/kc705_drtio_satellite.py b/artiq/gateware/targets/kc705_drtio_satellite.py
index dbb4b8958..f2dcd9913 100755
--- a/artiq/gateware/targets/kc705_drtio_satellite.py
+++ b/artiq/gateware/targets/kc705_drtio_satellite.py
@@ -18,18 +18,22 @@ def get_i2c_program(sys_clk_freq):
     # into registers. They have to be mapped; see the datasheet.
     # DSPLLsim reports the logical parameters in the design summary, not
     # the physical register values (but those are present separately).
-    N1_HS  = 0   # 4
-    NC1_LS = 19  # 20
-    N2_HS  = 1   # 5
-    N2_LS  = 511 # 512
-    N31    = 31  # 32
+    N1_HS  = 6     # 10
+    NC1_LS = 7     # 8
+    N2_HS  = 6     # 10
+    N2_LS  = 20111 # 20112
+    N31    = 2513  # 2514
+    N32    = 4596  # 4597
 
     i2c_sequence = [
         # PCA9548: select channel 7
         [(0x74 << 1), 1 << 7],
         # Si5324: configure
+        [(0x68 << 1), 0,   0b01010000],        # FREE_RUN=1
+        [(0x68 << 1), 1,   0b11100100],        # CK_PRIOR2=1 CK_PRIOR1=0
         [(0x68 << 1), 2,   0b0010 | (4 << 4)], # BWSEL=4
         [(0x68 << 1), 3,   0b0101 | 0x10],     # SQ_ICAL=1
+        [(0x68 << 1), 4,   0b10010010],        # AUTOSEL_REG=b10
         [(0x68 << 1), 6,            0x07],     # SFOUT1_REG=b111
         [(0x68 << 1), 25,  (N1_HS  << 5 ) & 0xff],
         [(0x68 << 1), 31,  (NC1_LS >> 16) & 0xff],

From 1ed3278783840619cf0d45c97205f47f8d8cea39 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Wed, 2 Nov 2016 10:53:54 +0800
Subject: [PATCH 054/134] remove stale TODO

---
 artiq/gateware/targets/kc705_drtio_satellite.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/artiq/gateware/targets/kc705_drtio_satellite.py b/artiq/gateware/targets/kc705_drtio_satellite.py
index f2dcd9913..b902289ff 100755
--- a/artiq/gateware/targets/kc705_drtio_satellite.py
+++ b/artiq/gateware/targets/kc705_drtio_satellite.py
@@ -12,7 +12,6 @@ from artiq.gateware.drtio.transceiver import gtx_7series
 from artiq.gateware.drtio import DRTIOSatellite
 
 
-# TODO: program Si5324 free-run mode, with automatic switching
 def get_i2c_program(sys_clk_freq):
     # NOTE: the logical parameters DO NOT MAP to physical values written
     # into registers. They have to be mapped; see the datasheet.

From bee9774bd5093176da3a55332cfdd755cbb219f6 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Wed, 2 Nov 2016 13:09:13 +0800
Subject: [PATCH 055/134] drtio: add link layer status CSR

---
 artiq/gateware/drtio/core.py       |  2 +-
 artiq/gateware/drtio/link_layer.py | 17 +++++++++++++++--
 2 files changed, 16 insertions(+), 3 deletions(-)

diff --git a/artiq/gateware/drtio/core.py b/artiq/gateware/drtio/core.py
index 45428e514..078bff751 100644
--- a/artiq/gateware/drtio/core.py
+++ b/artiq/gateware/drtio/core.py
@@ -60,4 +60,4 @@ class DRTIOMaster(Module):
         return self.rt_controller.get_kernel_csrs()
 
     def get_csrs(self):
-        return self.rt_controller.get_csrs()
+        return self.link_layer.get_csrs() + self.rt_controller.get_csrs()
diff --git a/artiq/gateware/drtio/link_layer.py b/artiq/gateware/drtio/link_layer.py
index 82019092a..a34e4dcad 100644
--- a/artiq/gateware/drtio/link_layer.py
+++ b/artiq/gateware/drtio/link_layer.py
@@ -3,7 +3,9 @@ from operator import xor, or_
 
 from migen import *
 from migen.genlib.fsm import *
-from migen.genlib.cdc import MultiReg
+from migen.genlib.cdc import MultiReg, BusSynchronizer
+
+from misoc.interconnect.csr import *
 
 
 class Scrambler(Module):
@@ -209,8 +211,10 @@ class LinkLayerRX(Module):
         ]
 
 
-class LinkLayer(Module):
+class LinkLayer(Module, AutoCSR):
     def __init__(self, encoder, decoders):
+        self.link_status = CSRStatus(3)
+
         # control signals, in rtio clock domain
         self.reset = Signal()
         self.ready = Signal()
@@ -264,18 +268,25 @@ class LinkLayer(Module):
             MultiReg(rx.link_init, rx_link_init, "rtio")
         ]
 
+        link_status = BusSynchronizer(3, "rtio", "sys")
+        self.submodules += link_status
+        self.comb += self.link_status.status.eq(link_status.o)
+
         fsm.act("RESET_RX",
+            link_status.i.eq(0),
             tx.link_init.eq(1),
             self.rx_reset.eq(1),
             NextState("WAIT_LOCAL_RX_READY")
         )
         fsm.act("WAIT_LOCAL_RX_READY",
+            link_status.i.eq(1),
             tx.link_init.eq(1),
             If(self.rx_ready,
                 NextState("WAIT_REMOTE_RX_READY")
             )
         )
         fsm.act("WAIT_REMOTE_RX_READY",
+            link_status.i.eq(2),
             tx.link_init.eq(1),
             tx.signal_rx_ready.eq(1),
             If(rx_remote_rx_ready,
@@ -283,9 +294,11 @@ class LinkLayer(Module):
             )
         )
         fsm.act("WAIT_REMOTE_LINK_UP",
+            link_status.i.eq(3),
             If(~rx_link_init, NextState("READY"))
         )
         fsm.act("READY",
+            link_status.i.eq(4),
             If(rx_link_init, NextState("RESET_RX")),
             self.ready.eq(1)
         )

From 00100148f11475453eb9ca97b2fa3f5572804953 Mon Sep 17 00:00:00 2001
From: whitequark <whitequark@whitequark.org>
Date: Wed, 2 Nov 2016 07:08:44 +0000
Subject: [PATCH 056/134] Si5324: actually write value of N32 into registers.

---
 artiq/gateware/targets/kc705_drtio_satellite.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/artiq/gateware/targets/kc705_drtio_satellite.py b/artiq/gateware/targets/kc705_drtio_satellite.py
index b902289ff..cc0fd9ee4 100755
--- a/artiq/gateware/targets/kc705_drtio_satellite.py
+++ b/artiq/gateware/targets/kc705_drtio_satellite.py
@@ -45,6 +45,9 @@ def get_i2c_program(sys_clk_freq):
         [(0x68 << 1), 43,  (N31    >> 16) & 0xff],
         [(0x68 << 1), 44,  (N31    >> 8)  & 0xff],
         [(0x68 << 1), 45,  (N31)          & 0xff],
+        [(0x68 << 1), 46,  (N32    >> 16) & 0xff],
+        [(0x68 << 1), 47,  (N32    >> 8)  & 0xff],
+        [(0x68 << 1), 48,  (N32)          & 0xff],
         [(0x68 << 1), 137,          0x01],     # FASTLOCK=1
         [(0x68 << 1), 136,          0x40],     # ICAL=1
     ]

From ba58a8affd6f935ee2bb99da17d965edf7b17651 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Wed, 2 Nov 2016 18:30:22 +0800
Subject: [PATCH 057/134] drtio/gtx_7series: paranoid reset deglitching

---
 artiq/gateware/drtio/transceiver/gtx_7series.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/artiq/gateware/drtio/transceiver/gtx_7series.py b/artiq/gateware/drtio/transceiver/gtx_7series.py
index 469d3f96f..b768ae470 100644
--- a/artiq/gateware/drtio/transceiver/gtx_7series.py
+++ b/artiq/gateware/drtio/transceiver/gtx_7series.py
@@ -162,15 +162,21 @@ class GTX_1000BASE_BX10(Module):
                 o_GTXTXN=tx_pads.n,
             )
 
+        tx_reset_deglitched = Signal()
+        tx_reset_deglitched.attr.add("no_retiming")
+        self.sync += tx_reset_deglitched.eq(~tx_init.done)
         self.clock_domains.cd_rtio = ClockDomain()
         self.specials += [
             Instance("BUFG", i_I=txoutclk, o_O=self.cd_rtio.clk),
-            AsyncResetSynchronizer(self.cd_rtio, ~tx_init.done)
+            AsyncResetSynchronizer(self.cd_rtio, tx_reset_deglitched)
         ]
+        rx_reset_deglitched = Signal()
+        rx_reset_deglitched.attr.add("no_retiming")
+        self.sync.rtio += rx_reset_deglitched.eq(~rx_init.done)
         self.clock_domains.cd_rtio_rx = ClockDomain()
         self.specials += [
             Instance("BUFG", i_I=rxoutclk, o_O=self.cd_rtio_rx.clk),
-            AsyncResetSynchronizer(self.cd_rtio_rx, ~rx_init.done)
+            AsyncResetSynchronizer(self.cd_rtio_rx, rx_reset_deglitched)
         ]
 
         self.comb += [

From a4ba34bb2c5dad8beda0aceacc869347f2450b42 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Thu, 3 Nov 2016 20:13:31 +0800
Subject: [PATCH 058/134] drtio: cleanup test_full_stack

---
 artiq/test/gateware/drtio/test_full_stack.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/artiq/test/gateware/drtio/test_full_stack.py b/artiq/test/gateware/drtio/test_full_stack.py
index c27437aa7..b7cd67833 100644
--- a/artiq/test/gateware/drtio/test_full_stack.py
+++ b/artiq/test/gateware/drtio/test_full_stack.py
@@ -203,5 +203,5 @@ class TestFullStack(unittest.TestCase):
 
         run_simulation(dut, 
             {"sys": test(), "rtio": check_ttls()},
-            {"sys": 8, "rtio": 5, "rtio_rx": 5, "rio": 5, "rio_phy": 5}, vcd_name="foo.vcd")
+            {"sys": 8, "rtio": 5, "rtio_rx": 5, "rio": 5, "rio_phy": 5})
         self.assertEqual(ttl_changes, correct_ttl_changes)

From 1d027ffa95904f14de4d580037f0d647b69578a4 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Thu, 3 Nov 2016 20:13:50 +0800
Subject: [PATCH 059/134] drtio: fix gtx_7series comma alignment

---
 artiq/gateware/drtio/transceiver/gtx_7series.py      | 2 +-
 artiq/gateware/drtio/transceiver/gtx_7series_init.py | 6 +++++-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/artiq/gateware/drtio/transceiver/gtx_7series.py b/artiq/gateware/drtio/transceiver/gtx_7series.py
index b768ae470..4d2f66e13 100644
--- a/artiq/gateware/drtio/transceiver/gtx_7series.py
+++ b/artiq/gateware/drtio/transceiver/gtx_7series.py
@@ -185,7 +185,7 @@ class GTX_1000BASE_BX10(Module):
             self.decoders[1].input.eq(rxdata[10:])
         ]
 
-        clock_aligner = BruteforceClockAligner(0b0011111000, self.rtio_clk_freq)
+        clock_aligner = BruteforceClockAligner(0b0001111100, self.rtio_clk_freq)
         self.submodules += clock_aligner
         self.comb += [
             clock_aligner.rxdata.eq(rxdata),
diff --git a/artiq/gateware/drtio/transceiver/gtx_7series_init.py b/artiq/gateware/drtio/transceiver/gtx_7series_init.py
index c46e3eef0..cf502ec24 100644
--- a/artiq/gateware/drtio/transceiver/gtx_7series_init.py
+++ b/artiq/gateware/drtio/transceiver/gtx_7series_init.py
@@ -118,7 +118,7 @@ class GTXInit(Module):
 
 
 # Changes the phase of the transceiver RX clock to align the comma to
-# the MSBs of RXDATA, fixing the latency.
+# the LSBs of RXDATA, fixing the latency.
 #
 # This is implemented by repeatedly resetting the transceiver until it
 # gives out the correct phase. Each reset gives a random phase.
@@ -130,6 +130,9 @@ class GTXInit(Module):
 #  * RXSLIDE_MODE=PMA cannot be used with the RX buffer bypassed.
 # Those design flaws make RXSLIDE_MODE=PMA yet another broken and useless
 # transceiver "feature".
+#
+# Warning: Xilinx transceivers are LSB first, and comma needs to be flipped
+# compared to the usual 8b10b binary representation.
 class BruteforceClockAligner(Module):
     def __init__(self, comma, rtio_clk_freq, check_period=6e-3, ready_time=50e-3):
         self.rxdata = Signal(20)
@@ -156,6 +159,7 @@ class BruteforceClockAligner(Module):
         comma_n = ~comma & 0b1111111111
         comma_seen_rxclk = Signal()
         comma_seen = Signal()
+        comma_seen_rxclk.attr.add("no_retiming")
         self.specials += MultiReg(comma_seen_rxclk, comma_seen)
         comma_seen_reset = PulseSynchronizer("rtio", "rtio_rx")
         self.submodules += comma_seen_reset

From 6a758372610ff8b01003ebfab3f9519d94cde747 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Thu, 3 Nov 2016 20:15:04 +0800
Subject: [PATCH 060/134] drtio: fix link_layer remote RX ready detection

---
 artiq/gateware/drtio/link_layer.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/artiq/gateware/drtio/link_layer.py b/artiq/gateware/drtio/link_layer.py
index a34e4dcad..0a1cca493 100644
--- a/artiq/gateware/drtio/link_layer.py
+++ b/artiq/gateware/drtio/link_layer.py
@@ -202,7 +202,7 @@ class LinkLayerRX(Module):
                 ).Elif(decoders[0].d != K(28, 7),
                     self.remote_rx_ready.eq(0)
                 ),
-                If(decoders[0].d == K(30, 7),
+                If(decoders[1].d == K(30, 7),
                     self.remote_rx_ready.eq(1)
                 ) if len(decoders) > 1 else None
             ).Else(

From f76aa249ce36cfb113cc4a7b5c9ac5b2e63f3f04 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Fri, 4 Nov 2016 15:16:48 +0800
Subject: [PATCH 061/134] drtio: squelch 7series RXSynchronizer outputs when
 MMCM is unlocked

---
 artiq/gateware/drtio/transceiver/gtx_7series.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/artiq/gateware/drtio/transceiver/gtx_7series.py b/artiq/gateware/drtio/transceiver/gtx_7series.py
index 4d2f66e13..0780982ca 100644
--- a/artiq/gateware/drtio/transceiver/gtx_7series.py
+++ b/artiq/gateware/drtio/transceiver/gtx_7series.py
@@ -212,10 +212,11 @@ class RXSynchronizer(Module, AutoCSR):
         self.phase_shift = CSR()
         self.phase_shift_done = CSRStatus()
 
-        self.clock_domains.cd_rtio_delayed = ClockDomain(reset_less=True)
+        self.clock_domains.cd_rtio_delayed = ClockDomain()
 
         mmcm_output = Signal()
         mmcm_fb = Signal()
+        mmcm_locked = Signal()
         # maximize VCO frequency to maximize phase shift resolution
         mmcm_mult = 1200e6//rtio_clk_freq
         self.specials += [
@@ -242,7 +243,8 @@ class RXSynchronizer(Module, AutoCSR):
                 i_PSINCDEC=self.phase_shift.r,
                 o_PSDONE=self.phase_shift_done.status,
             ),
-            Instance("BUFR", i_I=mmcm_output, o_O=self.cd_rtio_delayed.clk)
+            Instance("BUFR", i_I=mmcm_output, o_O=self.cd_rtio_delayed.clk),
+            AsyncResetSynchronizer(self.cd_rtio_delayed, ~mmcm_locked)
         ]
 
     def resync(self, signal):

From 747da3da15318d50f152d13d9ea1525c611634e8 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Fri, 4 Nov 2016 15:17:19 +0800
Subject: [PATCH 062/134] drtio: differentiate local and remote unknown packet
 type errors

---
 artiq/gateware/drtio/rt_packets.py           | 11 ++++++-----
 artiq/test/gateware/drtio/test_full_stack.py |  2 +-
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/artiq/gateware/drtio/rt_packets.py b/artiq/gateware/drtio/rt_packets.py
index f756fb70a..84243e903 100644
--- a/artiq/gateware/drtio/rt_packets.py
+++ b/artiq/gateware/drtio/rt_packets.py
@@ -50,12 +50,13 @@ def get_s2m_layouts(alignment):
 
 
 error_codes = {
-    "unknown_type": 0,
+    "unknown_type_local": 0,
+    "unknown_type_remote": 1,
     # The transmitter is normally responsible for avoiding
     # overflows and underflows. Those error reports are only
     # for diagnosing internal ARTIQ bugs.
-    "write_overflow": 1,
-    "write_underflow": 2
+    "write_overflow": 2,
+    "write_underflow": 3
 }
 
 
@@ -254,7 +255,7 @@ class RTPacketSatellite(Module):
                             NextState("FIFO_SPACE"),
                         "default": [
                             err_set.eq(1),
-                            NextValue(err_code, error_codes["unknown_type"])]
+                            NextValue(err_code, error_codes["unknown_type_remote"])]
                     })
                 )
             )
@@ -528,7 +529,7 @@ class RTPacketMaster(Module):
                         rx_plm.types["fifo_space_reply"]: NextState("FIFO_SPACE"),
                         "default": [
                             error_not.eq(1),
-                            error_code.eq(error_codes["unknown_type"])
+                            error_code.eq(error_codes["unknown_type_local"])
                         ]
                     })
                 )
diff --git a/artiq/test/gateware/drtio/test_full_stack.py b/artiq/test/gateware/drtio/test_full_stack.py
index b7cd67833..23d95da2e 100644
--- a/artiq/test/gateware/drtio/test_full_stack.py
+++ b/artiq/test/gateware/drtio/test_full_stack.py
@@ -170,7 +170,7 @@ class TestFullStack(unittest.TestCase):
             err_present = yield from csrs.err_present.read()
             err_code = yield from csrs.err_code.read()
             self.assertEqual(err_present, 1)
-            self.assertEqual(err_code, 2)
+            self.assertEqual(err_code, 3)
             yield from csrs.err_present.write(1)
             yield
             err_present = yield from csrs.err_present.read()

From 3da1cce783fa0e95d956568a4eb54fa5f4159ba7 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Fri, 4 Nov 2016 17:53:42 +0800
Subject: [PATCH 063/134] drtio: add packet counters

---
 artiq/gateware/drtio/rt_controller.py | 13 ++++++++++
 artiq/gateware/drtio/rt_packets.py    | 36 +++++++++++++++++++++++++++
 2 files changed, 49 insertions(+)

diff --git a/artiq/gateware/drtio/rt_controller.py b/artiq/gateware/drtio/rt_controller.py
index d718393da..28d00d1d2 100644
--- a/artiq/gateware/drtio/rt_controller.py
+++ b/artiq/gateware/drtio/rt_controller.py
@@ -25,6 +25,10 @@ class _CSRs(AutoCSR):
         self.err_present = CSR()
         self.err_code = CSRStatus(8)
 
+        self.dbg_update_packet_cnt = CSR()
+        self.dbg_packet_cnt_tx = CSRStatus(32)
+        self.dbg_packet_cnt_rx = CSRStatus(32)
+
 
 class RTController(Module):
     def __init__(self, rt_packets, channel_count, fine_ts_width):
@@ -156,6 +160,7 @@ class RTController(Module):
             )
         )
 
+        # channel state access
         self.comb += [
             self.csrs.o_dbg_fifo_space.status.eq(fifo_spaces.dat_r),
             self.csrs.o_dbg_last_timestamp.status.eq(last_timestamps.dat_r),
@@ -167,12 +172,20 @@ class RTController(Module):
             )
         ]
 
+        # errors
         self.comb += [
             self.csrs.err_present.w.eq(rt_packets.error_not),
             rt_packets.error_not_ack.eq(self.csrs.err_present.re),
             self.csrs.err_code.status.eq(rt_packets.error_code)
         ]
 
+        # packet counters
+        self.sync += \
+            If(self.csrs.dbg_update_packet_cnt.re,
+                self.csrs.dbg_packet_cnt_tx.status.eq(rt_packets.packet_cnt_tx),
+                self.csrs.dbg_packet_cnt_rx.status.eq(rt_packets.packet_cnt_rx)
+            )
+
     def get_kernel_csrs(self):
         return self.kcsrs.get_csrs()
 
diff --git a/artiq/gateware/drtio/rt_packets.py b/artiq/gateware/drtio/rt_packets.py
index 84243e903..183bc913f 100644
--- a/artiq/gateware/drtio/rt_packets.py
+++ b/artiq/gateware/drtio/rt_packets.py
@@ -5,6 +5,8 @@ from migen.genlib.fsm import *
 from migen.genlib.fifo import AsyncFIFO
 from migen.genlib.cdc import PulseSynchronizer
 
+from artiq.gateware.rtio.cdc import GrayCodeTransfer
+
 
 def layout_len(l):
     return sum(e[1] for e in l)
@@ -404,6 +406,10 @@ class RTPacketMaster(Module):
         self.error_not_ack = Signal()
         self.error_code = Signal(8)
 
+        # packet counters
+        self.packet_cnt_tx = Signal(32)
+        self.packet_cnt_rx = Signal(32)
+
         # # #
 
         # CDC
@@ -545,3 +551,33 @@ class RTPacketMaster(Module):
             fifo_space.eq(rx_dp.packet_as["fifo_space_reply"].space),
             NextState("INPUT")
         )
+
+        # packet counters
+        tx_frame_r = Signal()
+        packet_cnt_tx = Signal(32)
+        self.sync.rtio += [
+            tx_frame_r.eq(link_layer.tx_rt_frame),
+            If(link_layer.tx_rt_frame & ~tx_frame_r,
+                packet_cnt_tx.eq(packet_cnt_tx + 1))
+        ]
+        cdc_packet_cnt_tx = GrayCodeTransfer(32)
+        self.submodules += cdc_packet_cnt_tx
+        self.comb += [
+            cdc_packet_cnt_tx.i.eq(packet_cnt_tx),
+            self.packet_cnt_tx.eq(cdc_packet_cnt_tx.o)
+        ]
+
+        rx_frame_r = Signal()
+        packet_cnt_rx = Signal(32)
+        self.sync.rtio_rx += [
+            rx_frame_r.eq(link_layer.rx_rt_frame),
+            If(link_layer.rx_rt_frame & ~rx_frame_r,
+                packet_cnt_rx.eq(packet_cnt_rx + 1))
+        ]
+        cdc_packet_cnt_rx = ClockDomainsRenamer({"rtio": "rtio_rx"})(
+            GrayCodeTransfer(32))
+        self.submodules += cdc_packet_cnt_rx
+        self.comb += [
+            cdc_packet_cnt_rx.i.eq(packet_cnt_rx),
+            self.packet_cnt_rx.eq(cdc_packet_cnt_rx.o)
+        ]

From 1145a193dd3dab23264c591127bfced8233f399a Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Fri, 4 Nov 2016 18:36:43 +0800
Subject: [PATCH 064/134] drtio: fix ack of echo and set_time requests

---
 artiq/gateware/drtio/rt_packets.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/artiq/gateware/drtio/rt_packets.py b/artiq/gateware/drtio/rt_packets.py
index 183bc913f..648c9919f 100644
--- a/artiq/gateware/drtio/rt_packets.py
+++ b/artiq/gateware/drtio/rt_packets.py
@@ -510,12 +510,18 @@ class RTPacketMaster(Module):
         tx_fsm.act("ECHO",
             tx_dp.send("echo_request"),
             tx_dp.stb.eq(1),
-            If(tx_dp.done, NextState("IDLE_WRITE"))
+            If(tx_dp.done,
+                echo_ack.eq(1),
+                NextState("IDLE_WRITE")
+            )
         )
         tx_fsm.act("SET_TIME",
             tx_dp.send("set_time", timestamp=tsc_value),
             tx_dp.stb.eq(1),
-            If(tx_dp.done, NextState("IDLE_WRITE"))
+            If(tx_dp.done,
+                set_time_ack.eq(1),
+                NextState("IDLE_WRITE")
+            )
         )
 
         # RX FSM

From df7294792ccd12521316d9756934f0fcc0d9f307 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Fri, 4 Nov 2016 19:38:24 +0800
Subject: [PATCH 065/134] drtio: break some RT features into manager, add echo
 request CSR

---
 artiq/gateware/drtio/core.py                 |  5 +-
 artiq/gateware/drtio/rt_controller.py        | 57 ++++++++++++--------
 artiq/test/gateware/drtio/test_full_stack.py | 11 ++--
 3 files changed, 46 insertions(+), 27 deletions(-)

diff --git a/artiq/gateware/drtio/core.py b/artiq/gateware/drtio/core.py
index 078bff751..ca852726d 100644
--- a/artiq/gateware/drtio/core.py
+++ b/artiq/gateware/drtio/core.py
@@ -55,9 +55,12 @@ class DRTIOMaster(Module):
         self.submodules.rt_packets = rt_packets.RTPacketMaster(self.link_layer)
         self.submodules.rt_controller = rt_controller.RTController(
             self.rt_packets, channel_count, fine_ts_width)
+        self.submodules.rt_manager = rt_controller.RTManager(self.rt_packets)
 
     def get_kernel_csrs(self):
         return self.rt_controller.get_kernel_csrs()
 
     def get_csrs(self):
-        return self.link_layer.get_csrs() + self.rt_controller.get_csrs()
+        return (self.link_layer.get_csrs() +
+                self.rt_controller.get_csrs() +
+                self.rt_manager.get_csrs())
diff --git a/artiq/gateware/drtio/rt_controller.py b/artiq/gateware/drtio/rt_controller.py
index 28d00d1d2..4c4a95c27 100644
--- a/artiq/gateware/drtio/rt_controller.py
+++ b/artiq/gateware/drtio/rt_controller.py
@@ -22,25 +22,20 @@ class _CSRs(AutoCSR):
         self.o_reset_channel_status = CSR()
         self.o_wait = CSRStatus()
 
-        self.err_present = CSR()
-        self.err_code = CSRStatus(8)
-
-        self.dbg_update_packet_cnt = CSR()
-        self.dbg_packet_cnt_tx = CSRStatus(32)
-        self.dbg_packet_cnt_rx = CSRStatus(32)
-
 
 class RTController(Module):
     def __init__(self, rt_packets, channel_count, fine_ts_width):
         self.kcsrs = KernelCSRs()
         self.csrs = _CSRs()
 
+        # channel selection
         chan_sel = Signal(16)
         self.comb += chan_sel.eq(
             Mux(self.csrs.chan_sel_override_en.storage,
                 self.csrs.chan_sel_override.storage,
                 self.kcsrs.chan_sel.storage))
 
+        # master RTIO counter and counter synchronization
         self.submodules.counter = RTIOCounter(64-fine_ts_width)
         self.sync += If(self.kcsrs.counter_update.re, 
                         self.kcsrs.counter.status.eq(self.counter.value_sys))
@@ -57,6 +52,7 @@ class RTController(Module):
             If(self.csrs.set_time.re, rt_packets.set_time_stb.eq(1))
         ]
 
+        # remote channel status cache
         fifo_spaces_mem = Memory(16, channel_count)
         fifo_spaces = fifo_spaces_mem.get_port(write_capable=True)
         self.specials += fifo_spaces_mem, fifo_spaces
@@ -64,6 +60,7 @@ class RTController(Module):
         last_timestamps = last_timestamps_mem.get_port(write_capable=True)
         self.specials += last_timestamps_mem, last_timestamps
 
+        # common packet fields
         rt_packets_fifo_request = Signal()
         self.comb += [
             fifo_spaces.adr.eq(chan_sel),
@@ -172,22 +169,40 @@ class RTController(Module):
             )
         ]
 
-        # errors
-        self.comb += [
-            self.csrs.err_present.w.eq(rt_packets.error_not),
-            rt_packets.error_not_ack.eq(self.csrs.err_present.re),
-            self.csrs.err_code.status.eq(rt_packets.error_code)
-        ]
-
-        # packet counters
-        self.sync += \
-            If(self.csrs.dbg_update_packet_cnt.re,
-                self.csrs.dbg_packet_cnt_tx.status.eq(rt_packets.packet_cnt_tx),
-                self.csrs.dbg_packet_cnt_rx.status.eq(rt_packets.packet_cnt_rx)
-            )
-
     def get_kernel_csrs(self):
         return self.kcsrs.get_csrs()
 
     def get_csrs(self):
         return self.csrs.get_csrs()
+
+
+class RTManager(Module, AutoCSR):
+    def __init__(self, rt_packets):
+        self.request_echo = CSR()
+
+        self.err_present = CSR()
+        self.err_code = CSRStatus(8)
+
+        self.update_packet_cnt = CSR()
+        self.packet_cnt_tx = CSRStatus(32)
+        self.packet_cnt_rx = CSRStatus(32)
+
+        # # #
+
+        self.comb += self.request_echo.w.eq(rt_packets.echo_stb)
+        self.sync += [
+            If(rt_packets.echo_ack, rt_packets.echo_stb.eq(0)),
+            If(self.request_echo.re, rt_packets.echo_stb.eq(1))
+        ]
+
+        self.comb += [
+            self.err_present.w.eq(rt_packets.error_not),
+            rt_packets.error_not_ack.eq(self.err_present.re),
+            self.err_code.status.eq(rt_packets.error_code)
+        ]
+
+        self.sync += \
+            If(self.update_packet_cnt.re,
+                self.packet_cnt_tx.status.eq(rt_packets.packet_cnt_tx),
+                self.packet_cnt_rx.status.eq(rt_packets.packet_cnt_rx)
+            )
diff --git a/artiq/test/gateware/drtio/test_full_stack.py b/artiq/test/gateware/drtio/test_full_stack.py
index 23d95da2e..191c63671 100644
--- a/artiq/test/gateware/drtio/test_full_stack.py
+++ b/artiq/test/gateware/drtio/test_full_stack.py
@@ -59,6 +59,7 @@ class TestFullStack(unittest.TestCase):
         dut = DUT(2)
         kcsrs = dut.master.rt_controller.kcsrs
         csrs = dut.master.rt_controller.csrs
+        mgr = dut.master.rt_manager
 
         ttl_changes = []
         correct_ttl_changes = [
@@ -157,7 +158,7 @@ class TestFullStack(unittest.TestCase):
             self.assertEqual(wlen, 2)
 
         def test_tsc_error():
-            err_present = yield from csrs.err_present.read()
+            err_present = yield from mgr.err_present.read()
             self.assertEqual(err_present, 0)
             yield from csrs.tsc_correction.write(10000000)
             yield from csrs.set_time.write(1)
@@ -167,13 +168,13 @@ class TestFullStack(unittest.TestCase):
             yield from write(0, 1)
             for i in range(10):
                yield
-            err_present = yield from csrs.err_present.read()
-            err_code = yield from csrs.err_code.read()
+            err_present = yield from mgr.err_present.read()
+            err_code = yield from mgr.err_code.read()
             self.assertEqual(err_present, 1)
             self.assertEqual(err_code, 3)
-            yield from csrs.err_present.write(1)
+            yield from mgr.err_present.write(1)
             yield
-            err_present = yield from csrs.err_present.read()
+            err_present = yield from mgr.err_present.read()
             self.assertEqual(err_present, 0)
 
         def test():

From 5019b03f106aee7609415c288c1609416b21a8c4 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Fri, 4 Nov 2016 22:24:35 +0800
Subject: [PATCH 066/134] drtio: add echo and packet count test

---
 artiq/test/gateware/drtio/test_full_stack.py | 33 ++++++++++++++++++--
 1 file changed, 30 insertions(+), 3 deletions(-)

diff --git a/artiq/test/gateware/drtio/test_full_stack.py b/artiq/test/gateware/drtio/test_full_stack.py
index 191c63671..ae0ce7290 100644
--- a/artiq/test/gateware/drtio/test_full_stack.py
+++ b/artiq/test/gateware/drtio/test_full_stack.py
@@ -55,7 +55,9 @@ class DUT(Module):
         
 
 class TestFullStack(unittest.TestCase):
-    def test_full_stack(self):
+    clocks = {"sys": 8, "rtio": 5, "rtio_rx": 5, "rio": 5, "rio_phy": 5}
+
+    def test_controller(self):
         dut = DUT(2)
         kcsrs = dut.master.rt_controller.kcsrs
         csrs = dut.master.rt_controller.csrs
@@ -203,6 +205,31 @@ class TestFullStack(unittest.TestCase):
                 cycle += 1
 
         run_simulation(dut, 
-            {"sys": test(), "rtio": check_ttls()},
-            {"sys": 8, "rtio": 5, "rtio_rx": 5, "rio": 5, "rio_phy": 5})
+            {"sys": test(), "rtio": check_ttls()}, self.clocks)
         self.assertEqual(ttl_changes, correct_ttl_changes)
+
+    def test_echo(self):
+        dut = DUT(2)
+        csrs = dut.master.rt_controller.csrs
+        mgr = dut.master.rt_manager
+
+        def test():
+            while not (yield dut.master.link_layer.ready):
+                yield
+
+            yield from mgr.update_packet_cnt.write(1)
+            yield
+            self.assertEqual((yield from mgr.packet_cnt_tx.read()), 0)
+            self.assertEqual((yield from mgr.packet_cnt_rx.read()), 0)
+
+            yield from mgr.request_echo.write(1)
+
+            for i in range(15):
+                yield
+
+            yield from mgr.update_packet_cnt.write(1)
+            yield
+            self.assertEqual((yield from mgr.packet_cnt_tx.read()), 1)
+            self.assertEqual((yield from mgr.packet_cnt_rx.read()), 1)
+
+        run_simulation(dut, test(), self.clocks)

From de4712373712a139e952f8c6610df18a17a23aee Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Sat, 5 Nov 2016 00:24:00 +0800
Subject: [PATCH 067/134] drtio: connect RST and LOCKED on 7series
 RXSynchronizer MMCM

---
 artiq/gateware/drtio/transceiver/gtx_7series.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/artiq/gateware/drtio/transceiver/gtx_7series.py b/artiq/gateware/drtio/transceiver/gtx_7series.py
index 0780982ca..61e9287a7 100644
--- a/artiq/gateware/drtio/transceiver/gtx_7series.py
+++ b/artiq/gateware/drtio/transceiver/gtx_7series.py
@@ -223,6 +223,7 @@ class RXSynchronizer(Module, AutoCSR):
             Instance("MMCME2_ADV",
                 p_CLKIN1_PERIOD=1e9/rtio_clk_freq,
                 i_CLKIN1=ClockSignal("rtio_rx"),
+                i_RST=ResetSignal("rtio_rx"),
                 i_CLKINSEL=1,  # yes, 1=CLKIN1 0=CLKIN2
 
                 p_CLKFBOUT_MULT_F=mmcm_mult,
@@ -237,6 +238,7 @@ class RXSynchronizer(Module, AutoCSR):
 
                 p_CLKOUT0_USE_FINE_PS="TRUE",
                 o_CLKOUT0=mmcm_output,
+                o_LOCKED=mmcm_locked,
 
                 i_PSCLK=ClockSignal(),
                 i_PSEN=self.phase_shift.re,

From de065b7578f131ad868d509268063557cc0f90c0 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Sat, 5 Nov 2016 23:48:15 +0800
Subject: [PATCH 068/134] kc705_drtio_satellite: set output dir

---
 artiq/gateware/targets/kc705_drtio_satellite.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/artiq/gateware/targets/kc705_drtio_satellite.py b/artiq/gateware/targets/kc705_drtio_satellite.py
index cc0fd9ee4..32795634f 100755
--- a/artiq/gateware/targets/kc705_drtio_satellite.py
+++ b/artiq/gateware/targets/kc705_drtio_satellite.py
@@ -160,10 +160,13 @@ def main():
     parser = argparse.ArgumentParser(description="KC705 DRTIO satellite")
     parser.add_argument("--toolchain", default="vivado",
                         help="FPGA toolchain to use: ise, vivado")
+    parser.add_argument("--output-dir", default="drtiosat_kc705",
+                        help="output directory for generated "
+                             "source files and binaries")
     args = parser.parse_args()
 
     top = Satellite(args.toolchain)
-    top.build()
+    top.build(build_dir=args.output_dir)
 
 if __name__ == "__main__":
     main()

From 47b9868c68c4d97884db4c03b5a49e02cf964964 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Sat, 5 Nov 2016 23:48:29 +0800
Subject: [PATCH 069/134] kc705_drtio_master: pretend drtio is rtio

---
 artiq/gateware/targets/kc705_drtio_master.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/artiq/gateware/targets/kc705_drtio_master.py b/artiq/gateware/targets/kc705_drtio_master.py
index b3d973544..fe48909de 100755
--- a/artiq/gateware/targets/kc705_drtio_master.py
+++ b/artiq/gateware/targets/kc705_drtio_master.py
@@ -16,7 +16,7 @@ from artiq import __version__ as artiq_version
 class Master(MiniSoC, AMPSoC):
     mem_map = {
         "timer_kernel":  0x10000000, # (shadow @0x90000000)
-        "kdrtio":        0x20000000, # (shadow @0xa0000000)
+        "rtio":          0x20000000, # (shadow @0xa0000000)
         "mailbox":       0x70000000  # (shadow @0xf0000000)
     }
     mem_map.update(MiniSoC.mem_map)
@@ -41,7 +41,7 @@ class Master(MiniSoC, AMPSoC):
             sys_clk_freq=self.clk_freq,
             clock_div2=True)
         self.submodules.drtio = DRTIOMaster(self.transceiver)
-        self.register_kernel_cpu_csrdevice("kdrtio", self.drtio.get_kernel_csrs())
+        self.register_kernel_cpu_csrdevice("rtio", self.drtio.get_kernel_csrs())
         self.csr_devices.append("drtio")
 
 

From 266ae292d9a92897986513aa744452a321624d4e Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Sun, 6 Nov 2016 23:52:27 +0800
Subject: [PATCH 070/134] runtime: support configurations without moninj, log
 or dds

---
 artiq/runtime.rs/libksupport/api.rs | 7 +++++--
 artiq/runtime.rs/src/lib.rs         | 2 ++
 artiq/runtime/rtio.c                | 2 ++
 3 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/artiq/runtime.rs/libksupport/api.rs b/artiq/runtime.rs/libksupport/api.rs
index aa6483bbd..81c4d75df 100644
--- a/artiq/runtime.rs/libksupport/api.rs
+++ b/artiq/runtime.rs/libksupport/api.rs
@@ -105,13 +105,16 @@ static mut API: &'static [(&'static str, *const ())] = &[
     api!(rtio_input_timestamp),
     api!(rtio_input_data),
 
-// #if ((defined CONFIG_RTIO_DDS_COUNT) && (CONFIG_RTIO_DDS_COUNT > 0))
+    #[cfg(rtio_dds_count)]
     api!(dds_init),
+    #[cfg(rtio_dds_count)]
     api!(dds_init_sync),
+    #[cfg(rtio_dds_count)]
     api!(dds_batch_enter),
+    #[cfg(rtio_dds_count)]
     api!(dds_batch_exit),
+    #[cfg(rtio_dds_count)]
     api!(dds_set),
-// #endif
 
     api!(i2c_init),
     api!(i2c_start),
diff --git a/artiq/runtime.rs/src/lib.rs b/artiq/runtime.rs/src/lib.rs
index e661c427f..48efd5d0d 100644
--- a/artiq/runtime.rs/src/lib.rs
+++ b/artiq/runtime.rs/src/lib.rs
@@ -75,6 +75,7 @@ mod rpc_proto;
 
 mod kernel;
 mod session;
+#[cfg(has_rtio_moninj)]
 mod moninj;
 #[cfg(has_rtio_analyzer)]
 mod analyzer;
@@ -109,6 +110,7 @@ pub unsafe extern fn rust_main() {
 
         let mut scheduler = sched::Scheduler::new();
         scheduler.spawner().spawn(16384, session::thread);
+        #[cfg(has_rtio_moninj)]
         scheduler.spawner().spawn(4096, moninj::thread);
         #[cfg(has_rtio_analyzer)]
         scheduler.spawner().spawn(4096, analyzer::thread);
diff --git a/artiq/runtime/rtio.c b/artiq/runtime/rtio.c
index 09f71d9d4..ac5168224 100644
--- a/artiq/runtime/rtio.c
+++ b/artiq/runtime/rtio.c
@@ -124,6 +124,7 @@ unsigned int rtio_input_data(int channel)
 
 void rtio_log_va(long long int timestamp, const char *fmt, va_list args)
 {
+#ifdef CONFIG_RTIO_LOG_CHANNEL
     // This executes on the kernel CPU's stack, which is specifically designed
     // for allocation of this kind of massive buffers.
     int   len = vsnprintf(NULL, 0, fmt, args);
@@ -152,6 +153,7 @@ void rtio_log_va(long long int timestamp, const char *fmt, va_list args)
             i = 0;
         }
     }
+#endif
 }
 
 void rtio_log(long long int timestamp, const char *fmt, ...)

From 78f18a12eb3903d4712c90284199b530211e0cef Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Mon, 7 Nov 2016 19:34:34 +0800
Subject: [PATCH 071/134] runtime: basic DRTIO init code

---
 artiq/runtime.rs/src/drtio.rs | 67 +++++++++++++++++++++++++++++++++++
 artiq/runtime.rs/src/lib.rs   |  7 ++++
 2 files changed, 74 insertions(+)
 create mode 100644 artiq/runtime.rs/src/drtio.rs

diff --git a/artiq/runtime.rs/src/drtio.rs b/artiq/runtime.rs/src/drtio.rs
new file mode 100644
index 000000000..8ee5dfc23
--- /dev/null
+++ b/artiq/runtime.rs/src/drtio.rs
@@ -0,0 +1,67 @@
+use board::csr;
+use sched::{Waiter, Spawner};
+
+fn drtio_link_is_up() -> bool {
+    unsafe {
+        csr::drtio::link_status_read() == 4
+    }
+}
+
+fn drtio_sync_tsc() {
+    unsafe {
+        csr::drtio::set_time_write(1);
+        while csr::drtio::set_time_read() == 1 {}
+    }
+}
+
+fn drtio_init_channel(channel: u16) {
+    unsafe {
+        csr::drtio::chan_sel_override_write(channel);
+        csr::drtio::chan_sel_override_en_write(1);
+        
+        csr::drtio::o_reset_channel_status_write(1);
+        csr::drtio::o_get_fifo_space_write(1);
+        while csr::drtio::o_wait_read() == 1 {}  // TODO: timeout
+        info!("FIFO space on channel {} is {}", channel, csr::drtio::o_dbg_fifo_space_read());
+
+        csr::drtio::chan_sel_override_en_write(0);
+    }
+}
+
+pub fn link_thread(waiter: Waiter, _spawner: Spawner) {
+    loop {
+        waiter.until(drtio_link_is_up).unwrap();
+        info!("link is up");
+
+        drtio_sync_tsc();
+        info!("TSC synced");
+        for channel in 0..16 {
+            drtio_init_channel(channel);
+        }
+        info!("link initialization completed");
+
+        waiter.until(|| !drtio_link_is_up()).unwrap();
+        info!("link is down");
+    }
+}
+
+fn drtio_error_present() -> bool {
+    unsafe {
+        csr::drtio::err_present_read() != 0
+    }
+}
+
+fn drtio_get_error() -> u8 {
+    unsafe {
+        let err = csr::drtio::err_code_read();
+        csr::drtio::err_present_write(1);
+        err
+    }
+}
+
+pub fn error_thread(waiter: Waiter, _spawner: Spawner) {
+    loop {
+        waiter.until(drtio_error_present).unwrap();
+        error!("DRTIO error {}", drtio_get_error());   
+    }
+}
diff --git a/artiq/runtime.rs/src/lib.rs b/artiq/runtime.rs/src/lib.rs
index 48efd5d0d..101df55b2 100644
--- a/artiq/runtime.rs/src/lib.rs
+++ b/artiq/runtime.rs/src/lib.rs
@@ -79,6 +79,8 @@ mod session;
 mod moninj;
 #[cfg(has_rtio_analyzer)]
 mod analyzer;
+#[cfg(has_drtio)]
+mod drtio;
 
 extern {
     fn network_init();
@@ -114,6 +116,11 @@ pub unsafe extern fn rust_main() {
         scheduler.spawner().spawn(4096, moninj::thread);
         #[cfg(has_rtio_analyzer)]
         scheduler.spawner().spawn(4096, analyzer::thread);
+        #[cfg(has_drtio)]
+        {
+            scheduler.spawner().spawn(4096, drtio::link_thread);
+            scheduler.spawner().spawn(4096, drtio::error_thread);
+        }
 
         loop {
             scheduler.run();

From 8c86920364c593e4894ffe8318d7bce7485e8b94 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Tue, 8 Nov 2016 16:11:10 +0800
Subject: [PATCH 072/134] drtio: add examples

---
 artiq/examples/drtio/device_db.pyon           | 52 +++++++++++++++++++
 .../drtio/repository/blink_forever.py         | 14 +++++
 2 files changed, 66 insertions(+)
 create mode 100644 artiq/examples/drtio/device_db.pyon
 create mode 100644 artiq/examples/drtio/repository/blink_forever.py

diff --git a/artiq/examples/drtio/device_db.pyon b/artiq/examples/drtio/device_db.pyon
new file mode 100644
index 000000000..850950165
--- /dev/null
+++ b/artiq/examples/drtio/device_db.pyon
@@ -0,0 +1,52 @@
+# This is an example device database that needs to be adapted to your setup.
+# The RTIO channel numbers here are for NIST CLOCK on KC705.
+# The list of devices here is not exhaustive.
+
+{
+    "comm": {
+        "type": "local",
+        "module": "artiq.coredevice.comm_tcp",
+        "class": "Comm",
+        "arguments": {"host": "kc705.lab.m-labs.hk"}
+    },
+    "core": {
+        "type": "local",
+        "module": "artiq.coredevice.core",
+        "class": "Core",
+        "arguments": {"ref_period": 16e-9}
+    },
+    "core_cache": {
+        "type": "local",
+        "module": "artiq.coredevice.cache",
+        "class": "CoreCache"
+    },
+
+    "rled0": {
+        "type": "local",
+        "module": "artiq.coredevice.ttl",
+        "class": "TTLOut",
+        "arguments": {"channel": 0},
+    },
+    "rled1": {
+        "type": "local",
+        "module": "artiq.coredevice.ttl",
+        "class": "TTLOut",
+        "arguments": {"channel": 1},
+        "comment": "Hello World"
+    },
+
+    "rsmap": {
+        "type": "local",
+        "module": "artiq.coredevice.ttl",
+        "class": "TTLOut",
+        "arguments": {"channel": 8}
+    },
+    "rsman": {
+        "type": "local",
+        "module": "artiq.coredevice.ttl",
+        "class": "TTLOut",
+        "arguments": {"channel": 9}
+    },
+
+
+}
diff --git a/artiq/examples/drtio/repository/blink_forever.py b/artiq/examples/drtio/repository/blink_forever.py
new file mode 100644
index 000000000..0d6fa8ff8
--- /dev/null
+++ b/artiq/examples/drtio/repository/blink_forever.py
@@ -0,0 +1,14 @@
+from artiq.experiment import *
+
+
+class BlinkForever(EnvExperiment):
+    def build(self):
+        self.setattr_device("core")
+        self.setattr_device("rled0")
+
+    @kernel
+    def run(self):
+        self.core.reset()
+        while True:
+            self.rled0.pulse(100*ms)
+            delay(100*ms)

From bcb5053fb60ddba9a794762421faf1ee3c151d88 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Tue, 8 Nov 2016 16:40:50 +0800
Subject: [PATCH 073/134] drtio: fix master TSC KCSR readout

---
 artiq/gateware/drtio/rt_controller.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/artiq/gateware/drtio/rt_controller.py b/artiq/gateware/drtio/rt_controller.py
index 4c4a95c27..e301443df 100644
--- a/artiq/gateware/drtio/rt_controller.py
+++ b/artiq/gateware/drtio/rt_controller.py
@@ -38,7 +38,7 @@ class RTController(Module):
         # master RTIO counter and counter synchronization
         self.submodules.counter = RTIOCounter(64-fine_ts_width)
         self.sync += If(self.kcsrs.counter_update.re, 
-                        self.kcsrs.counter.status.eq(self.counter.value_sys))
+            self.kcsrs.counter.status.eq(self.counter.value_sys << fine_ts_width))
         tsc_correction = Signal(64)
         self.csrs.tsc_correction.storage.attr.add("no_retiming")
         self.specials += MultiReg(self.csrs.tsc_correction.storage, tsc_correction)

From 95acc9b9d45aa0dd3cddeeef2f935a6769c75b15 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Tue, 8 Nov 2016 16:52:40 +0800
Subject: [PATCH 074/134] drtio: allow specifying 7series RXSynchronizer
 initial phase

---
 artiq/gateware/drtio/transceiver/gtx_7series.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/artiq/gateware/drtio/transceiver/gtx_7series.py b/artiq/gateware/drtio/transceiver/gtx_7series.py
index 61e9287a7..8653dbe33 100644
--- a/artiq/gateware/drtio/transceiver/gtx_7series.py
+++ b/artiq/gateware/drtio/transceiver/gtx_7series.py
@@ -208,7 +208,7 @@ class RXSynchronizer(Module, AutoCSR):
     Xilinx scriptures (when existent) and should be constant for a given design
     placement.
     """
-    def __init__(self, rtio_clk_freq):
+    def __init__(self, rtio_clk_freq, initial_phase=0.0):
         self.phase_shift = CSR()
         self.phase_shift_done = CSRStatus()
 
@@ -228,6 +228,7 @@ class RXSynchronizer(Module, AutoCSR):
 
                 p_CLKFBOUT_MULT_F=mmcm_mult,
                 p_CLKOUT0_DIVIDE_F=mmcm_mult,
+                p_CLKOUT0_PHASE=intial_phase,
                 p_DIVCLK_DIVIDE=1,
 
                 # According to Xilinx, there is no guarantee of input/output

From d547c5d922428a9cffe7b0e2f6c9e5799c5717ec Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Wed, 9 Nov 2016 14:44:01 +0800
Subject: [PATCH 075/134] drtio: fix example ref_period

---
 artiq/examples/drtio/device_db.pyon | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/artiq/examples/drtio/device_db.pyon b/artiq/examples/drtio/device_db.pyon
index 850950165..5db8d5e20 100644
--- a/artiq/examples/drtio/device_db.pyon
+++ b/artiq/examples/drtio/device_db.pyon
@@ -13,7 +13,7 @@
         "type": "local",
         "module": "artiq.coredevice.core",
         "class": "Core",
-        "arguments": {"ref_period": 16e-9}
+        "arguments": {"ref_period": 2e-9}
     },
     "core_cache": {
         "type": "local",

From 60e748eabe2fafe16f43ad75fe619f87af368332 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Wed, 9 Nov 2016 14:53:45 +0800
Subject: [PATCH 076/134] drtio: better LED demo

---
 artiq/examples/drtio/device_db.pyon           | 37 ++++++++++++++++++-
 .../drtio/repository/blink_forever.py         | 15 ++++++--
 2 files changed, 47 insertions(+), 5 deletions(-)

diff --git a/artiq/examples/drtio/device_db.pyon b/artiq/examples/drtio/device_db.pyon
index 5db8d5e20..7696aacce 100644
--- a/artiq/examples/drtio/device_db.pyon
+++ b/artiq/examples/drtio/device_db.pyon
@@ -32,7 +32,42 @@
         "module": "artiq.coredevice.ttl",
         "class": "TTLOut",
         "arguments": {"channel": 1},
-        "comment": "Hello World"
+    },
+    "rled2": {
+        "type": "local",
+        "module": "artiq.coredevice.ttl",
+        "class": "TTLOut",
+        "arguments": {"channel": 2},
+    },
+    "rled3": {
+        "type": "local",
+        "module": "artiq.coredevice.ttl",
+        "class": "TTLOut",
+        "arguments": {"channel": 3},
+    },
+    "rled4": {
+        "type": "local",
+        "module": "artiq.coredevice.ttl",
+        "class": "TTLOut",
+        "arguments": {"channel": 4},
+    },
+    "rled5": {
+        "type": "local",
+        "module": "artiq.coredevice.ttl",
+        "class": "TTLOut",
+        "arguments": {"channel": 5},
+    },
+    "rled6": {
+        "type": "local",
+        "module": "artiq.coredevice.ttl",
+        "class": "TTLOut",
+        "arguments": {"channel": 6},
+    },
+    "rled7": {
+        "type": "local",
+        "module": "artiq.coredevice.ttl",
+        "class": "TTLOut",
+        "arguments": {"channel": 7},
     },
 
     "rsmap": {
diff --git a/artiq/examples/drtio/repository/blink_forever.py b/artiq/examples/drtio/repository/blink_forever.py
index 0d6fa8ff8..589d389cb 100644
--- a/artiq/examples/drtio/repository/blink_forever.py
+++ b/artiq/examples/drtio/repository/blink_forever.py
@@ -4,11 +4,18 @@ from artiq.experiment import *
 class BlinkForever(EnvExperiment):
     def build(self):
         self.setattr_device("core")
-        self.setattr_device("rled0")
+        self.leds = [self.get_device("rled" + str(i)) for i in range(8)]
 
     @kernel
     def run(self):
-        self.core.reset()
+        #self.core.reset()
+        self.core.break_realtime()
+
         while True:
-            self.rled0.pulse(100*ms)
-            delay(100*ms)
+            for led in self.leds:
+                led.pulse(250*ms)
+            t = now_mu()
+            for led in self.leds:
+                at_mu(t)
+                led.pulse(500*ms)
+            delay(250*ms)

From c92ccd3b5bce76c908cbab820083aa9de8149220 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Wed, 9 Nov 2016 15:29:15 +0800
Subject: [PATCH 077/134] drtio: add pulse rate example

---
 artiq/examples/drtio/repository/pulse_rate.py | 26 +++++++++++++++++++
 1 file changed, 26 insertions(+)
 create mode 100644 artiq/examples/drtio/repository/pulse_rate.py

diff --git a/artiq/examples/drtio/repository/pulse_rate.py b/artiq/examples/drtio/repository/pulse_rate.py
new file mode 100644
index 000000000..091a30786
--- /dev/null
+++ b/artiq/examples/drtio/repository/pulse_rate.py
@@ -0,0 +1,26 @@
+from artiq.experiment import *
+
+
+class PulseRate(EnvExperiment):
+    def build(self):
+        self.setattr_device("core")
+        self.setattr_device("rsmap")
+
+    @kernel
+    def run(self):
+        #self.core.reset()
+        self.core.break_realtime()
+
+        dt = seconds_to_mu(300*ns)
+        while True:
+            for i in range(10000):
+                try:
+                    self.rsmap.pulse_mu(dt)
+                    delay_mu(dt)
+                except RTIOUnderflow:
+                    dt += 1
+                    self.core.break_realtime()
+                    break
+            else:
+                print(mu_to_seconds(dt))
+                return

From 863934c4fab346d8b1a805d9eafc7550db4731b0 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Wed, 9 Nov 2016 22:03:47 +0800
Subject: [PATCH 078/134] drtio: more reliable link layer init

---
 artiq/gateware/drtio/core.py                 |  9 ++--
 artiq/gateware/drtio/link_layer.py           | 45 +++++++++++++++-----
 artiq/runtime.rs/src/drtio.rs                |  2 +-
 artiq/test/gateware/drtio/test_full_stack.py |  6 ++-
 4 files changed, 44 insertions(+), 18 deletions(-)

diff --git a/artiq/gateware/drtio/core.py b/artiq/gateware/drtio/core.py
index ca852726d..71334a13f 100644
--- a/artiq/gateware/drtio/core.py
+++ b/artiq/gateware/drtio/core.py
@@ -6,9 +6,10 @@ from artiq.gateware.drtio import link_layer, rt_packets, iot, rt_controller
 
 
 class DRTIOSatellite(Module):
-    def __init__(self, transceiver, rx_synchronizer, channels, fine_ts_width=3, full_ts_width=63):
+    def __init__(self, transceiver, rx_synchronizer, channels, fine_ts_width=3, full_ts_width=63,
+                 ll_rx_ready_confirm=1000):
         self.submodules.link_layer = link_layer.LinkLayer(
-            transceiver.encoder, transceiver.decoders)
+            transceiver.encoder, transceiver.decoders, ll_rx_ready_confirm)
         self.comb += [
             transceiver.rx_reset.eq(self.link_layer.rx_reset),
             self.link_layer.rx_ready.eq(transceiver.rx_ready)
@@ -45,9 +46,9 @@ class DRTIOSatellite(Module):
 
 
 class DRTIOMaster(Module):
-    def __init__(self, transceiver, channel_count=1024, fine_ts_width=3):
+    def __init__(self, transceiver, channel_count=1024, fine_ts_width=3, ll_rx_ready_confirm=1000):
         self.submodules.link_layer = link_layer.LinkLayer(
-            transceiver.encoder, transceiver.decoders)
+            transceiver.encoder, transceiver.decoders, ll_rx_ready_confirm)
         self.comb += [
             transceiver.rx_reset.eq(self.link_layer.rx_reset),
             self.link_layer.rx_ready.eq(transceiver.rx_ready)
diff --git a/artiq/gateware/drtio/link_layer.py b/artiq/gateware/drtio/link_layer.py
index 0a1cca493..59f286732 100644
--- a/artiq/gateware/drtio/link_layer.py
+++ b/artiq/gateware/drtio/link_layer.py
@@ -4,6 +4,7 @@ from operator import xor, or_
 from migen import *
 from migen.genlib.fsm import *
 from migen.genlib.cdc import MultiReg, BusSynchronizer
+from migen.genlib.misc import WaitTimer
 
 from misoc.interconnect.csr import *
 
@@ -212,7 +213,7 @@ class LinkLayerRX(Module):
 
 
 class LinkLayer(Module, AutoCSR):
-    def __init__(self, encoder, decoders):
+    def __init__(self, encoder, decoders, rx_ready_confirm_cycles):
         self.link_status = CSRStatus(3)
 
         # control signals, in rtio clock domain
@@ -272,6 +273,12 @@ class LinkLayer(Module, AutoCSR):
         self.submodules += link_status
         self.comb += self.link_status.status.eq(link_status.o)
 
+        wait_confirm = ClockDomainsRenamer("rtio")(
+            WaitTimer(rx_ready_confirm_cycles))
+        self.submodules += wait_confirm
+        signal_rx_ready_margin = ClockDomainsRenamer("rtio")(WaitTimer(15))
+        self.submodules += signal_rx_ready_margin
+
         fsm.act("RESET_RX",
             link_status.i.eq(0),
             tx.link_init.eq(1),
@@ -281,24 +288,40 @@ class LinkLayer(Module, AutoCSR):
         fsm.act("WAIT_LOCAL_RX_READY",
             link_status.i.eq(1),
             tx.link_init.eq(1),
-            If(self.rx_ready,
-                NextState("WAIT_REMOTE_RX_READY")
-            )
+            If(self.rx_ready, NextState("CONFIRM_LOCAL_RX_READY"))
         )
-        fsm.act("WAIT_REMOTE_RX_READY",
+        fsm.act("CONFIRM_LOCAL_RX_READY",
             link_status.i.eq(2),
             tx.link_init.eq(1),
+            wait_confirm.wait.eq(1),
+            If(wait_confirm.done, NextState("WAIT_REMOTE_RX_READY")),
+            If(~rx_link_init, NextState("RESET_RX"))
+        )
+        fsm.act("WAIT_REMOTE_RX_READY",
+            link_status.i.eq(3),
+            tx.link_init.eq(1),
             tx.signal_rx_ready.eq(1),
-            If(rx_remote_rx_ready,
-                NextState("WAIT_REMOTE_LINK_UP")
-            )
+            If(rx_remote_rx_ready, NextState("ENSURE_SIGNAL_RX_READY"))
+        )
+        # If the transceiver transmits one character per RTIO cycle,
+        # we may be unlucky and signal_rx_ready will transmit a comma
+        # on the first cycle instead of a "RX ready" character.
+        # Further, we need to ensure the rx.remote_rx_ready
+        # gets through MultiReg to rx_remote_rx_ready at the receiver.
+        # So transmit the "RX ready" pattern for several cycles.
+        fsm.act("ENSURE_SIGNAL_RX_READY",
+            link_status.i.eq(3),
+            tx.link_init.eq(1),
+            tx.signal_rx_ready.eq(1),
+            signal_rx_ready_margin.wait.eq(1),
+            If(signal_rx_ready_margin.done, NextState("WAIT_REMOTE_LINK_UP"))
         )
         fsm.act("WAIT_REMOTE_LINK_UP",
-            link_status.i.eq(3),
+            link_status.i.eq(4),
             If(~rx_link_init, NextState("READY"))
         )
         fsm.act("READY",
-            link_status.i.eq(4),
-            If(rx_link_init, NextState("RESET_RX")),
+            link_status.i.eq(5),
+            If(rx_link_init, NextState("RESET_RX")),  # TODO: remove this, link deinit should be detected at upper layer
             self.ready.eq(1)
         )
diff --git a/artiq/runtime.rs/src/drtio.rs b/artiq/runtime.rs/src/drtio.rs
index 8ee5dfc23..214295697 100644
--- a/artiq/runtime.rs/src/drtio.rs
+++ b/artiq/runtime.rs/src/drtio.rs
@@ -3,7 +3,7 @@ use sched::{Waiter, Spawner};
 
 fn drtio_link_is_up() -> bool {
     unsafe {
-        csr::drtio::link_status_read() == 4
+        csr::drtio::link_status_read() == 5
     }
 }
 
diff --git a/artiq/test/gateware/drtio/test_full_stack.py b/artiq/test/gateware/drtio/test_full_stack.py
index ae0ce7290..eea0c2256 100644
--- a/artiq/test/gateware/drtio/test_full_stack.py
+++ b/artiq/test/gateware/drtio/test_full_stack.py
@@ -41,7 +41,8 @@ class DUT(Module):
         self.ttl1 = Signal()
         self.transceivers = DummyTransceiverPair(nwords)
         
-        self.submodules.master = DRTIOMaster(self.transceivers.alice)
+        self.submodules.master = DRTIOMaster(self.transceivers.alice,
+                                             ll_rx_ready_confirm=15)
 
         rx_synchronizer = DummyRXSynchronizer()
         self.submodules.phy0 = ttl_simple.Output(self.ttl0)
@@ -51,7 +52,8 @@ class DUT(Module):
             rtio.Channel.from_phy(self.phy1, ofifo_depth=4)
         ]
         self.submodules.satellite = DRTIOSatellite(
-            self.transceivers.bob, rx_synchronizer, rtio_channels)
+            self.transceivers.bob, rx_synchronizer, rtio_channels,
+            ll_rx_ready_confirm=15)
         
 
 class TestFullStack(unittest.TestCase):

From 8a48d6d66e6b308d9ae26c1a80f8f9aca2495ca9 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Wed, 9 Nov 2016 22:15:42 +0800
Subject: [PATCH 079/134] drtio: fix typo

---
 artiq/gateware/drtio/transceiver/gtx_7series.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/artiq/gateware/drtio/transceiver/gtx_7series.py b/artiq/gateware/drtio/transceiver/gtx_7series.py
index 8653dbe33..00166e11f 100644
--- a/artiq/gateware/drtio/transceiver/gtx_7series.py
+++ b/artiq/gateware/drtio/transceiver/gtx_7series.py
@@ -228,7 +228,7 @@ class RXSynchronizer(Module, AutoCSR):
 
                 p_CLKFBOUT_MULT_F=mmcm_mult,
                 p_CLKOUT0_DIVIDE_F=mmcm_mult,
-                p_CLKOUT0_PHASE=intial_phase,
+                p_CLKOUT0_PHASE=initial_phase,
                 p_DIVCLK_DIVIDE=1,
 
                 # According to Xilinx, there is no guarantee of input/output

From f2f131e0fba7ee290e1a1bfce39055383df79c49 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Mon, 14 Nov 2016 00:04:53 +0800
Subject: [PATCH 080/134] drtio: add aux receiver (untested)

---
 artiq/gateware/drtio/aux_controller.py | 117 +++++++++++++++++++++++++
 1 file changed, 117 insertions(+)
 create mode 100644 artiq/gateware/drtio/aux_controller.py

diff --git a/artiq/gateware/drtio/aux_controller.py b/artiq/gateware/drtio/aux_controller.py
new file mode 100644
index 000000000..27a1118fe
--- /dev/null
+++ b/artiq/gateware/drtio/aux_controller.py
@@ -0,0 +1,117 @@
+from migen import *
+from migen.fhdl.simplify import FullMemoryWE
+
+from misoc.interconnect.csr import *
+from misoc.interconnect import stream 
+from misoc.interconnect import wishbone
+
+
+max_packet = 1024
+
+class Transmitter(Module, AutoCSR):
+    def __init__(self, link_layer, min_mem_dw):
+        # TODO
+
+
+class Receiver(Module, AutoCSR):
+    def __init__(self, link_layer, min_mem_dw):
+        self.aux_rx_length = CSRStatus(bits_for(max_packet))
+        self.aux_rx_present = CSR()
+        self.aux_rx_error = CSR()
+
+        ll_dw = len(link_layer.rx_aux_data)
+        mem_dw = max(min_mem_dw, ll_dw)
+        self.specials.mem = Memory(mem_dw, max_packet//(mem_dw//8))
+
+        converter = stream.Converter(ll_dw, mem_dw)
+        self.submodules += converter
+
+        self.sync.rtio_rx += [
+            converter.sink.stb.eq(link_layer.rx_aux_frame),
+            converter.sink.data.eq(link_layer.rx_aux_data)
+        ]
+        self.comb += converter.sink.eop.eq(~link_layer.rx_aux_frame)
+
+        mem_port = self.mem.get_port(write_capable=True, clock_domain="rtio_rx")
+        self.specials += mem_port
+
+        frame_counter_nbits = bits_for(max_packet) - log2_int(mem_dw//8)
+        frame_counter = Signal(frame_counter_nbits)
+        self.comb += [
+            mem_port.adr.eq(frame_counter),
+            mem_port.dat_w.eq(link_layer.rx_aux_data)
+        ]
+
+        frame_counter.attr.add("no_retiming")
+        frame_counter_sys = Signal(frame_counter_nbits)
+        self.specials += MultiReg(frame_counter, frame_counter_sys)
+        self.comb += aux_rx_length.status.eq(frame_counter_sys << log2_int(mem_dw//8))
+
+        signal_frame = PulseSynchronizer("rtio_rx", "sys")
+        frame_ack = PulseSynchronizer("sys", "rtio_rx")
+        signal_error = PulseSynchronizer("rtio_rx", "sys")
+        self.submodules += signal_frame, signal_error
+        self.sync += [
+            If(self.aux_rx_present.re, self.aux_rx_present.w.eq(0)),
+            If(signal_frame.o, self.aux_rx_present.w.eq(1)),
+            If(self.aux_rx_error.re, self.aux_rx_error.w.eq(0)),
+            If(signal_error.o, self.aux_rx_error.eq(1))
+        ]
+        self.comb += frame_ack.i.eq(self.aux_rx_present.re)
+
+        fsm = ClockDomainsRenamer("rtio_rx")(FSM(reset_state="IDLE"))
+        self.submodules += fsm
+
+        rx_aux_frame_r = Signal()
+        self.sync.rtio_rx += rx_aux_frame_r.eq(link_layer.rx_aux_frame)
+
+        fsm.act("IDLE",
+            If(link_layer.rx_aux_frame & ~rx_aux_frame_r,
+                NextValue(frame_counter, frame_counter + 1),
+                mem_port.we.eq(1),
+                NextState("FRAME")
+            ).Else(
+                NextValue(frame_counter, 0)
+            )
+        )
+        fsm.act("FRAME",
+            If(link_layer.rx_aux_frame,
+                NextValue(frame_counter, frame_counter + 1),
+                mem_port.we.eq(1),
+                If(frame_counter == max_packet,
+                    mem_port.we.eq(0),
+                    signal_error.i.eq(1),
+                    NextState("IDLE")  # remainder of the frame discarded
+                )
+            ).Else(
+                signal_frame.i.eq(1),
+                NextState("WAIT_ACK")
+            )
+        )
+        fsm.act("WAIT_ACK",
+            If(frame_ack.o,
+                NextValue(frame_counter, 0), 
+                NextState("IDLE")
+            ),
+            If(link_layer.rx_aux_frame, signal_error.i.eq(1))
+        )
+
+
+class AuxController(Module):
+    def __init__(self, link_layer):
+        self.bus = wishbone.Interface()
+        self.submodules.transmitter = Transmitter(link_layer, len(self.bus.dat_w))
+        self.submodules.receiver = Receiver(link_layer, len(self.bus.dat_w))
+
+        # TODO: FullMemoryWE should be applied by migen.build
+        tx_sdram_if = FullMemoryWE()(self.transmitter.mem, read_only=False)
+        rx_sdram_if = wishbone.SRAM(self.receiver.mem, read_only=True)
+        wsb = log2_int(len(self.bus.dat_w)//8)
+        decoder = wishbone.Decoder(self.bus,
+            [(lambda a: a[log2_int(max_packet)-wsb] == 0, tx_sdram_if)
+             (lambda a: a[log2_int(max_packet)-wsb] == 1, rx_sdram_if)],
+            register=True)
+        self.submodules += tx_sdram_if, rx_sdram_if, decoder
+
+    def get_csrs(self):
+        return self.transmitter.get_csrs() + self.receiver.get_csrs()

From a4d92716dae4bd36daf800049f426585ad772ed3 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Mon, 14 Nov 2016 17:18:54 +0800
Subject: [PATCH 081/134] drtio: fix aux receiver, add aux transmitter

---
 artiq/gateware/drtio/aux_controller.py | 126 ++++++++++++++++++++++---
 1 file changed, 112 insertions(+), 14 deletions(-)

diff --git a/artiq/gateware/drtio/aux_controller.py b/artiq/gateware/drtio/aux_controller.py
index 27a1118fe..f9f6f5bca 100644
--- a/artiq/gateware/drtio/aux_controller.py
+++ b/artiq/gateware/drtio/aux_controller.py
@@ -8,9 +8,89 @@ from misoc.interconnect import wishbone
 
 max_packet = 1024
 
+
 class Transmitter(Module, AutoCSR):
     def __init__(self, link_layer, min_mem_dw):
-        # TODO
+        self.aux_tx_length = CSRStorage(bits_for(max_packet))
+        self.aux_tx = CSR()
+
+        ll_dw = len(link_layer.tx_aux_data)
+        mem_dw = max(min_mem_dw, ll_dw)
+        self.specials.mem = Memory(mem_dw, max_packet//(mem_dw//8))
+
+        converter = stream.Converter(mem_dw, ll_dw)
+        self.submodules += converter
+
+        # when continuously fed, the Converter outputs data continuously
+        self.comb += [
+            converter.source.ack.eq(link_layer.tx_aux_ack),
+            link_layer.tx_aux_frame.eq(converter.source.stb),
+            link_layer.tx_aux_data.eq(converter.source.data)
+        ]
+
+        seen_eop_rst = Signal()
+        frame_r = Signal()
+        seen_eop = Signal()
+        self.sync.rtio += [
+            If(link_layer.tx_aux_ack,
+                frame_r.eq(link_layer.tx_aux_frame),
+                If(frame_r & ~link_layer.tx_aux_frame, seen_eop.eq(1))
+            ),
+            If(seen_eop_rst, seen_eop.eq(0))
+        ]
+
+        mem_port = self.mem.get_port(clock_domain="rtio")
+        self.specials += mem_port
+
+        self.aux_tx_length.storage.attr.add("no_retiming")
+        tx_length = Signal(bits_for(max_packet))
+        self.specials += MultiReg(self.aux_tx_length.storage, tx_length, "rtio")
+
+        frame_counter_nbits = bits_for(max_packet) - log2_int(mem_dw//8)
+        frame_counter = Signal(frame_counter_nbits)
+        frame_counter_next = Signal(frame_counter_nbits)
+        frame_counter_ce = Signal()
+        frame_counter_rst = Signal(),
+        self.comb += [
+            frame_counter_next.eq(frame_counter),
+            If(frame_counter_rst,
+                frame_counter_next.eq(0)
+            ).Elif(frame_counter_ce,
+                frame_counter_next.eq(frame_counter + 1)
+            ),
+            mem_port.adr.eq(frame_counter_next),
+            converter.sink.data.eq(mem_port.dat_r)
+        ]
+        self.sync.rtio += frame_counter.eq(frame_counter_next)
+
+        start_tx = PulseSynchronizer("sys", "rtio")
+        tx_done = PulseSynchronizer("rtio", "sys")
+        self.submodules += start_tx, tx_done
+        self.comb += start_tx.i.eq(self.aux_tx.re)
+        self.sync += [
+            If(tx_done.o, self.aux_tx_w.eq(0)),
+            If(self.aux_tx.re, self.aux_tx.w.eq(1))
+        ]
+
+        fsm = ClockDomainsRenamer("rtio")(FSM(reset_state="IDLE"))
+        self.submodules += fsm
+
+        fsm.act("IDLE",
+            frame_counter_rst.eq(1),
+            seen_eop_rst.eq(1),
+            If(start_tx.o, NextState("TRANSMIT"))
+        )
+        fsm.act("TRANSMIT",
+            converter.sink.stb.eq(1),
+            frame_counter_ce.eq(1),
+            If(frame_counter_next == tx_length, NextState("WAIT_INTERFRAME"))
+        )
+        fsm.act("WAIT_INTERFRAME",
+            If(seen_eop,
+                tx_done.i.eq(1),
+                NextState("IDLE")
+            )
+        )
 
 
 class Receiver(Module, AutoCSR):
@@ -26,11 +106,12 @@ class Receiver(Module, AutoCSR):
         converter = stream.Converter(ll_dw, mem_dw)
         self.submodules += converter
 
+        # when continuously drained, the Converter accepts data continuously
         self.sync.rtio_rx += [
-            converter.sink.stb.eq(link_layer.rx_aux_frame),
+            converter.sink.stb.eq(converter.rx_aux_stb),
             converter.sink.data.eq(link_layer.rx_aux_data)
         ]
-        self.comb += converter.sink.eop.eq(~link_layer.rx_aux_frame)
+        self.comb += converter.sink.eop.eq(link_layer.rx_aux_stb & ~link_layer.rx_aux_frame)
 
         mem_port = self.mem.get_port(write_capable=True, clock_domain="rtio_rx")
         self.specials += mem_port
@@ -39,7 +120,8 @@ class Receiver(Module, AutoCSR):
         frame_counter = Signal(frame_counter_nbits)
         self.comb += [
             mem_port.adr.eq(frame_counter),
-            mem_port.dat_w.eq(link_layer.rx_aux_data)
+            mem_port.dat_w.eq(converter.source.data),
+            converter.source.ack.eq(1)
         ]
 
         frame_counter.attr.add("no_retiming")
@@ -62,38 +144,54 @@ class Receiver(Module, AutoCSR):
         fsm = ClockDomainsRenamer("rtio_rx")(FSM(reset_state="IDLE"))
         self.submodules += fsm
 
-        rx_aux_frame_r = Signal()
-        self.sync.rtio_rx += rx_aux_frame_r.eq(link_layer.rx_aux_frame)
+        sop = Signal()
+        self.sync.rtio_rx += \
+            If(converter.source.stb,
+                If(converter.source.eop,
+                    sop.eq(1)
+                ).Else(
+                    sop.eq(0)
+                )
+            )
 
         fsm.act("IDLE",
-            If(link_layer.rx_aux_frame & ~rx_aux_frame_r,
+            If(converter.source.stb & sop,
                 NextValue(frame_counter, frame_counter + 1),
                 mem_port.we.eq(1),
-                NextState("FRAME")
+                If(converter.source.eop,
+                    NextState("SIGNAL_FRAME")
+                ).Else(
+                    NextState("FRAME")
+                )
             ).Else(
                 NextValue(frame_counter, 0)
             )
         )
         fsm.act("FRAME",
-            If(link_layer.rx_aux_frame,
+            If(converter.source.stb,
                 NextValue(frame_counter, frame_counter + 1),
                 mem_port.we.eq(1),
                 If(frame_counter == max_packet,
                     mem_port.we.eq(0),
                     signal_error.i.eq(1),
-                    NextState("IDLE")  # remainder of the frame discarded
+                    NextState("IDLE")  # discard the rest of the frame
+                ),
+                If(converter.source.eop,
+                    NextState("SIGNAL_FRAME")
                 )
-            ).Else(
-                signal_frame.i.eq(1),
-                NextState("WAIT_ACK")
             )
         )
+        fsm.act("SIGNAL_FRAME",
+            signal_frame.i.eq(1),
+            NextState("WAIT_ACK"),
+            If(converter.source.stb, signal_error.i.eq(1))
+        )
         fsm.act("WAIT_ACK",
             If(frame_ack.o,
                 NextValue(frame_counter, 0), 
                 NextState("IDLE")
             ),
-            If(link_layer.rx_aux_frame, signal_error.i.eq(1))
+            If(converter.source.stb, signal_error.i.eq(1))
         )
 
 

From 84bd962ed54e73a82733ab16c45a695fc01cd637 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Mon, 14 Nov 2016 17:20:47 +0800
Subject: [PATCH 082/134] drtio: integrate aux controller

---
 artiq/gateware/drtio/core.py | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/artiq/gateware/drtio/core.py b/artiq/gateware/drtio/core.py
index 71334a13f..51d8cd391 100644
--- a/artiq/gateware/drtio/core.py
+++ b/artiq/gateware/drtio/core.py
@@ -2,7 +2,7 @@ from types import SimpleNamespace
 
 from migen import *
 
-from artiq.gateware.drtio import link_layer, rt_packets, iot, rt_controller
+from artiq.gateware.drtio import link_layer, rt_packets, iot, rt_controller, aux_controller
 
 
 class DRTIOSatellite(Module):
@@ -44,6 +44,12 @@ class DRTIOSatellite(Module):
             self.cd_rio_phy.rst.eq(ResetSignal("rtio", allow_reset_less=True)),
         ]
 
+        self.submodules.aux_controller = aux_controller.AuxController(
+            self.link_layer)
+
+    def get_csrs(self):
+        return self.aux_controller.get_csrs()
+
 
 class DRTIOMaster(Module):
     def __init__(self, transceiver, channel_count=1024, fine_ts_width=3, ll_rx_ready_confirm=1000):
@@ -58,10 +64,14 @@ class DRTIOMaster(Module):
             self.rt_packets, channel_count, fine_ts_width)
         self.submodules.rt_manager = rt_controller.RTManager(self.rt_packets)
 
+        self.submodules.aux_controller = aux_controller.AuxController(
+            self.link_layer)
+
     def get_kernel_csrs(self):
         return self.rt_controller.get_kernel_csrs()
 
     def get_csrs(self):
         return (self.link_layer.get_csrs() +
                 self.rt_controller.get_csrs() +
-                self.rt_manager.get_csrs())
+                self.rt_manager.get_csrs() +
+                self.aux_controller.get_csrs())

From e1394db861c59d32e6b58bb14e52150984ee757a Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Mon, 14 Nov 2016 17:26:30 +0800
Subject: [PATCH 083/134] drtio: aux controller minor fixes

---
 artiq/gateware/drtio/aux_controller.py | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/artiq/gateware/drtio/aux_controller.py b/artiq/gateware/drtio/aux_controller.py
index f9f6f5bca..e8fd87e70 100644
--- a/artiq/gateware/drtio/aux_controller.py
+++ b/artiq/gateware/drtio/aux_controller.py
@@ -1,5 +1,6 @@
 from migen import *
 from migen.fhdl.simplify import FullMemoryWE
+from migen.genlib.cdc import MultiReg, PulseSynchronizer
 
 from misoc.interconnect.csr import *
 from misoc.interconnect import stream 
@@ -50,7 +51,7 @@ class Transmitter(Module, AutoCSR):
         frame_counter = Signal(frame_counter_nbits)
         frame_counter_next = Signal(frame_counter_nbits)
         frame_counter_ce = Signal()
-        frame_counter_rst = Signal(),
+        frame_counter_rst = Signal()
         self.comb += [
             frame_counter_next.eq(frame_counter),
             If(frame_counter_rst,
@@ -68,7 +69,7 @@ class Transmitter(Module, AutoCSR):
         self.submodules += start_tx, tx_done
         self.comb += start_tx.i.eq(self.aux_tx.re)
         self.sync += [
-            If(tx_done.o, self.aux_tx_w.eq(0)),
+            If(tx_done.o, self.aux_tx.w.eq(0)),
             If(self.aux_tx.re, self.aux_tx.w.eq(1))
         ]
 
@@ -108,7 +109,7 @@ class Receiver(Module, AutoCSR):
 
         # when continuously drained, the Converter accepts data continuously
         self.sync.rtio_rx += [
-            converter.sink.stb.eq(converter.rx_aux_stb),
+            converter.sink.stb.eq(link_layer.rx_aux_stb),
             converter.sink.data.eq(link_layer.rx_aux_data)
         ]
         self.comb += converter.sink.eop.eq(link_layer.rx_aux_stb & ~link_layer.rx_aux_frame)
@@ -127,7 +128,7 @@ class Receiver(Module, AutoCSR):
         frame_counter.attr.add("no_retiming")
         frame_counter_sys = Signal(frame_counter_nbits)
         self.specials += MultiReg(frame_counter, frame_counter_sys)
-        self.comb += aux_rx_length.status.eq(frame_counter_sys << log2_int(mem_dw//8))
+        self.comb += self.aux_rx_length.status.eq(frame_counter_sys << log2_int(mem_dw//8))
 
         signal_frame = PulseSynchronizer("rtio_rx", "sys")
         frame_ack = PulseSynchronizer("sys", "rtio_rx")
@@ -137,7 +138,7 @@ class Receiver(Module, AutoCSR):
             If(self.aux_rx_present.re, self.aux_rx_present.w.eq(0)),
             If(signal_frame.o, self.aux_rx_present.w.eq(1)),
             If(self.aux_rx_error.re, self.aux_rx_error.w.eq(0)),
-            If(signal_error.o, self.aux_rx_error.eq(1))
+            If(signal_error.o, self.aux_rx_error.w.eq(1))
         ]
         self.comb += frame_ack.i.eq(self.aux_rx_present.re)
 
@@ -202,12 +203,12 @@ class AuxController(Module):
         self.submodules.receiver = Receiver(link_layer, len(self.bus.dat_w))
 
         # TODO: FullMemoryWE should be applied by migen.build
-        tx_sdram_if = FullMemoryWE()(self.transmitter.mem, read_only=False)
+        tx_sdram_if = FullMemoryWE()(wishbone.SRAM(self.transmitter.mem, read_only=False))
         rx_sdram_if = wishbone.SRAM(self.receiver.mem, read_only=True)
         wsb = log2_int(len(self.bus.dat_w)//8)
         decoder = wishbone.Decoder(self.bus,
-            [(lambda a: a[log2_int(max_packet)-wsb] == 0, tx_sdram_if)
-             (lambda a: a[log2_int(max_packet)-wsb] == 1, rx_sdram_if)],
+            [(lambda a: a[log2_int(max_packet)-wsb] == 0, tx_sdram_if.bus),
+             (lambda a: a[log2_int(max_packet)-wsb] == 1, rx_sdram_if.bus)],
             register=True)
         self.submodules += tx_sdram_if, rx_sdram_if, decoder
 

From 6c9965b4441fcfd96a050047fcf6a9caa04843c7 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Tue, 15 Nov 2016 12:02:41 +0800
Subject: [PATCH 084/134] drtio: aux controller fixes

---
 artiq/gateware/drtio/aux_controller.py | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/artiq/gateware/drtio/aux_controller.py b/artiq/gateware/drtio/aux_controller.py
index e8fd87e70..c9a7840ba 100644
--- a/artiq/gateware/drtio/aux_controller.py
+++ b/artiq/gateware/drtio/aux_controller.py
@@ -12,11 +12,12 @@ max_packet = 1024
 
 class Transmitter(Module, AutoCSR):
     def __init__(self, link_layer, min_mem_dw):
-        self.aux_tx_length = CSRStorage(bits_for(max_packet))
-        self.aux_tx = CSR()
-
         ll_dw = len(link_layer.tx_aux_data)
         mem_dw = max(min_mem_dw, ll_dw)
+
+        self.aux_tx_length = CSRStorage(bits_for(max_packet),
+                                        alignment_bits=log2_int(mem_dw//8))
+        self.aux_tx = CSR()
         self.specials.mem = Memory(mem_dw, max_packet//(mem_dw//8))
 
         converter = stream.Converter(mem_dw, ll_dw)
@@ -83,7 +84,9 @@ class Transmitter(Module, AutoCSR):
         )
         fsm.act("TRANSMIT",
             converter.sink.stb.eq(1),
-            frame_counter_ce.eq(1),
+            If(converter.sink.ack,
+                frame_counter_ce.eq(1)
+            ),
             If(frame_counter_next == tx_length, NextState("WAIT_INTERFRAME"))
         )
         fsm.act("WAIT_INTERFRAME",
@@ -109,7 +112,7 @@ class Receiver(Module, AutoCSR):
 
         # when continuously drained, the Converter accepts data continuously
         self.sync.rtio_rx += [
-            converter.sink.stb.eq(link_layer.rx_aux_stb),
+            converter.sink.stb.eq(link_layer.rx_aux_stb & link_layer.rx_aux_frame),
             converter.sink.data.eq(link_layer.rx_aux_data)
         ]
         self.comb += converter.sink.eop.eq(link_layer.rx_aux_stb & ~link_layer.rx_aux_frame)
@@ -145,7 +148,7 @@ class Receiver(Module, AutoCSR):
         fsm = ClockDomainsRenamer("rtio_rx")(FSM(reset_state="IDLE"))
         self.submodules += fsm
 
-        sop = Signal()
+        sop = Signal(reset=1)
         self.sync.rtio_rx += \
             If(converter.source.stb,
                 If(converter.source.eop,

From 7fa9a4efc3892fc9726b3e85ffab78a0c63b0885 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Tue, 15 Nov 2016 12:02:53 +0800
Subject: [PATCH 085/134] drtio: aux controller unittest WIP

---
 .../gateware/drtio/test_aux_controller.py     | 67 +++++++++++++++++++
 1 file changed, 67 insertions(+)
 create mode 100644 artiq/test/gateware/drtio/test_aux_controller.py

diff --git a/artiq/test/gateware/drtio/test_aux_controller.py b/artiq/test/gateware/drtio/test_aux_controller.py
new file mode 100644
index 000000000..808fbc391
--- /dev/null
+++ b/artiq/test/gateware/drtio/test_aux_controller.py
@@ -0,0 +1,67 @@
+import unittest
+from types import SimpleNamespace
+
+from migen import *
+
+from artiq.gateware.drtio.link_layer import *
+from artiq.gateware.drtio.aux_controller import *
+
+
+class Loopback(Module):
+    def __init__(self, nwords):
+        ks = [Signal() for k in range(nwords)]
+        ds = [Signal(8) for d in range(nwords)]
+        encoder = SimpleNamespace(k=ks, d=ds)
+        decoders = [SimpleNamespace(k=k, d=d) for k, d in zip(ks, ds)]
+        self.submodules.tx = LinkLayerTX(encoder)
+        self.submodules.rx = LinkLayerRX(decoders)
+
+        self.ready = Signal()
+
+        self.tx_aux_frame = self.tx.aux_frame
+        self.tx_aux_data = self.tx.aux_data
+        self.tx_aux_ack = self.tx.aux_ack
+        self.tx_rt_frame = self.tx.rt_frame
+        self.tx_rt_data = self.tx.rt_data
+
+        self.rx_aux_stb = self.rx.aux_stb
+        self.rx_aux_frame = self.rx.aux_frame & self.ready
+        self.rx_aux_data = self.rx.aux_data
+        self.rx_rt_frame = self.rx.rt_frame & self.ready
+        self.rx_rt_data = self.rx.rt_data
+
+
+class TB(Module):
+    def __init__(self, nwords):
+        self.submodules.link_layer = Loopback(nwords)
+        self.submodules.aux_controller = ClockDomainsRenamer(
+            {"rtio": "sys", "rtio_rx": "sys"})(AuxController(self.link_layer))
+
+
+class TestAuxController(unittest.TestCase):
+    def test_aux_controller(self):
+        dut = TB(4)
+
+        def gen():
+            yield dut.link_layer.tx.link_init.eq(1)
+            yield
+            yield
+            yield dut.link_layer.tx.link_init.eq(0)
+            while not (yield dut.link_layer.rx.link_init):
+                yield
+            while (yield dut.link_layer.rx.link_init):
+                yield
+            yield dut.link_layer.ready.eq(1)
+            yield
+
+            yield from dut.aux_controller.bus.write(0, 0x42)
+            yield from dut.aux_controller.bus.write(1, 0x23)
+            yield from dut.aux_controller.transmitter.aux_tx_length.write(8)
+            yield from dut.aux_controller.transmitter.aux_tx.write(1)
+            for i in range(40):
+                yield
+            print(hex((yield from dut.aux_controller.bus.read(256))))
+            print(hex((yield from dut.aux_controller.bus.read(257))))
+            print((yield from dut.aux_controller.receiver.aux_rx_length.read()))
+
+        run_simulation(dut, gen(), vcd_name="foo.vcd")

From 140bb0ecee8481a1e178a662e04d83f4b3ab1ef8 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Wed, 16 Nov 2016 19:44:03 +0800
Subject: [PATCH 086/134] drtio: aux controller fixes

---
 artiq/gateware/drtio/aux_controller.py | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/artiq/gateware/drtio/aux_controller.py b/artiq/gateware/drtio/aux_controller.py
index c9a7840ba..8755fedf2 100644
--- a/artiq/gateware/drtio/aux_controller.py
+++ b/artiq/gateware/drtio/aux_controller.py
@@ -111,11 +111,17 @@ class Receiver(Module, AutoCSR):
         self.submodules += converter
 
         # when continuously drained, the Converter accepts data continuously
+        frame_r = Signal()
         self.sync.rtio_rx += [
-            converter.sink.stb.eq(link_layer.rx_aux_stb & link_layer.rx_aux_frame),
-            converter.sink.data.eq(link_layer.rx_aux_data)
+            If(link_layer.rx_aux_stb,
+                frame_r.eq(link_layer.rx_aux_frame),
+                converter.sink.data.eq(link_layer.rx_aux_data)
+            )
+        ]
+        self.comb += [
+            converter.sink.stb.eq(link_layer.rx_aux_stb & frame_r),
+            converter.sink.eop.eq(converter.sink.stb & ~link_layer.rx_aux_frame)
         ]
-        self.comb += converter.sink.eop.eq(link_layer.rx_aux_stb & ~link_layer.rx_aux_frame)
 
         mem_port = self.mem.get_port(write_capable=True, clock_domain="rtio_rx")
         self.specials += mem_port
@@ -136,7 +142,7 @@ class Receiver(Module, AutoCSR):
         signal_frame = PulseSynchronizer("rtio_rx", "sys")
         frame_ack = PulseSynchronizer("sys", "rtio_rx")
         signal_error = PulseSynchronizer("rtio_rx", "sys")
-        self.submodules += signal_frame, signal_error
+        self.submodules += signal_frame, frame_ack, signal_error
         self.sync += [
             If(self.aux_rx_present.re, self.aux_rx_present.w.eq(0)),
             If(signal_frame.o, self.aux_rx_present.w.eq(1)),

From 09363e1da8807bb910b137724209574c4a6e4214 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Wed, 16 Nov 2016 19:45:28 +0800
Subject: [PATCH 087/134] drtio: aux controller unittest

---
 .../gateware/drtio/test_aux_controller.py     | 54 +++++++++++++++----
 1 file changed, 44 insertions(+), 10 deletions(-)

diff --git a/artiq/test/gateware/drtio/test_aux_controller.py b/artiq/test/gateware/drtio/test_aux_controller.py
index 808fbc391..d9a3ef467 100644
--- a/artiq/test/gateware/drtio/test_aux_controller.py
+++ b/artiq/test/gateware/drtio/test_aux_controller.py
@@ -1,4 +1,5 @@
 import unittest
+import random
 from types import SimpleNamespace
 
 from migen import *
@@ -42,7 +43,7 @@ class TestAuxController(unittest.TestCase):
     def test_aux_controller(self):
         dut = TB(4)
 
-        def gen():
+        def link_init():
             yield dut.link_layer.tx.link_init.eq(1)
             yield
             yield
@@ -52,16 +53,49 @@ class TestAuxController(unittest.TestCase):
             while (yield dut.link_layer.rx.link_init):
                 yield
             yield dut.link_layer.ready.eq(1)
-            yield
 
-            yield from dut.aux_controller.bus.write(0, 0x42)
-            yield from dut.aux_controller.bus.write(1, 0x23)
-            yield from dut.aux_controller.transmitter.aux_tx_length.write(8)
+        def send_packet(packet):
+            for i, d in enumerate(packet):
+                yield from dut.aux_controller.bus.write(i, d)
+            yield from dut.aux_controller.transmitter.aux_tx_length.write(len(packet)*4)
             yield from dut.aux_controller.transmitter.aux_tx.write(1)
-            for i in range(40):
+            yield
+            while (yield from dut.aux_controller.transmitter.aux_tx.read()):
                 yield
-            print(hex((yield from dut.aux_controller.bus.read(256))))
-            print(hex((yield from dut.aux_controller.bus.read(257))))
-            print((yield from dut.aux_controller.receiver.aux_rx_length.read()))
 
-        run_simulation(dut, gen(), vcd_name="foo.vcd")
+        def receive_packet():
+            while not (yield from dut.aux_controller.receiver.aux_rx_present.read()):
+                yield
+            length = yield from dut.aux_controller.receiver.aux_rx_length.read()
+            r = []
+            for i in range(length//4):
+                r.append((yield from dut.aux_controller.bus.read(256+i)))
+            yield from dut.aux_controller.receiver.aux_rx_present.write(1)
+            return r
+
+        prng = random.Random(0)
+
+        def send_and_check_packet():
+            data = [prng.randrange(2**32-1) for _ in range(prng.randrange(1, 16))]
+            yield from send_packet(data)
+            received = yield from receive_packet()
+            self.assertEqual(data, received)
+
+        def sim():
+            yield from link_init()
+            for i in range(8):
+                yield from send_and_check_packet()
+
+        @passive
+        def rt_traffic():
+            while True:
+                while prng.randrange(4):
+                    yield
+                yield dut.link_layer.tx_rt_frame.eq(1)
+                yield
+                while prng.randrange(4):
+                    yield
+                yield dut.link_layer.tx_rt_frame.eq(0)
+                yield
+
+        run_simulation(dut, [sim(), rt_traffic()])

From bb047aabe99652f857931ea3537560310d40cffe Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Thu, 17 Nov 2016 22:32:39 +0800
Subject: [PATCH 088/134] drtio: simpler link layer

---
 artiq/gateware/drtio/core.py                  |   9 +-
 artiq/gateware/drtio/link_layer.py            | 343 ++++++++----------
 .../gateware/drtio/transceiver/gtx_7series.py |   2 +-
 artiq/runtime.rs/src/drtio.rs                 |   7 +-
 .../gateware/drtio/test_aux_controller.py     |   8 +-
 artiq/test/gateware/drtio/test_full_stack.py  |   6 +-
 artiq/test/gateware/drtio/test_link_layer.py  |  41 +--
 7 files changed, 173 insertions(+), 243 deletions(-)

diff --git a/artiq/gateware/drtio/core.py b/artiq/gateware/drtio/core.py
index 51d8cd391..a51d22629 100644
--- a/artiq/gateware/drtio/core.py
+++ b/artiq/gateware/drtio/core.py
@@ -6,10 +6,9 @@ from artiq.gateware.drtio import link_layer, rt_packets, iot, rt_controller, aux
 
 
 class DRTIOSatellite(Module):
-    def __init__(self, transceiver, rx_synchronizer, channels, fine_ts_width=3, full_ts_width=63,
-                 ll_rx_ready_confirm=1000):
+    def __init__(self, transceiver, rx_synchronizer, channels, fine_ts_width=3, full_ts_width=63):
         self.submodules.link_layer = link_layer.LinkLayer(
-            transceiver.encoder, transceiver.decoders, ll_rx_ready_confirm)
+            transceiver.encoder, transceiver.decoders)
         self.comb += [
             transceiver.rx_reset.eq(self.link_layer.rx_reset),
             self.link_layer.rx_ready.eq(transceiver.rx_ready)
@@ -52,9 +51,9 @@ class DRTIOSatellite(Module):
 
 
 class DRTIOMaster(Module):
-    def __init__(self, transceiver, channel_count=1024, fine_ts_width=3, ll_rx_ready_confirm=1000):
+    def __init__(self, transceiver, channel_count=1024, fine_ts_width=3):
         self.submodules.link_layer = link_layer.LinkLayer(
-            transceiver.encoder, transceiver.decoders, ll_rx_ready_confirm)
+            transceiver.encoder, transceiver.decoders)
         self.comb += [
             transceiver.rx_reset.eq(self.link_layer.rx_reset),
             self.link_layer.rx_ready.eq(transceiver.rx_ready)
diff --git a/artiq/gateware/drtio/link_layer.py b/artiq/gateware/drtio/link_layer.py
index 59f286732..24d325f9c 100644
--- a/artiq/gateware/drtio/link_layer.py
+++ b/artiq/gateware/drtio/link_layer.py
@@ -3,43 +3,102 @@ from operator import xor, or_
 
 from migen import *
 from migen.genlib.fsm import *
-from migen.genlib.cdc import MultiReg, BusSynchronizer
+from migen.genlib.cdc import MultiReg
 from migen.genlib.misc import WaitTimer
 
 from misoc.interconnect.csr import *
 
 
 class Scrambler(Module):
-    def __init__(self, n_io, n_state=23, taps=[17, 22]):
-        self.i = Signal(n_io)
-        self.o = Signal(n_io)
+    def __init__(self, n_io1, n_io2, n_state=23, taps=[17, 22]):
+        self.i1 = Signal(n_io1)
+        self.o1 = Signal(n_io1)
+        self.i2 = Signal(n_io2)
+        self.o2 = Signal(n_io2)
+        self.sel = Signal()
 
         # # #
 
         state = Signal(n_state, reset=1)
-        curval = [state[i] for i in range(n_state)]
-        for i in reversed(range(n_io)):
-            flip = reduce(xor, [curval[tap] for tap in taps])
-            self.sync += self.o[i].eq(flip ^ self.i[i])
-            curval.insert(0, flip)
-            curval.pop()
 
-        self.sync += state.eq(Cat(*curval[:n_state]))
+        stmts1 = []
+        stmts2 = []
+        for stmts, si, so in ((stmts1, self.i1, self.o1),
+                              (stmts2, self.i2, self.o2)):
+            curval = [state[i] for i in range(n_state)]
+            for i in reversed(range(len(si))):
+                out = si[i] ^ reduce(xor, [curval[tap] for tap in taps])
+                stmts += [so[i].eq(out)]
+                curval.insert(0, out)
+                curval.pop()
+
+            stmts += [state.eq(Cat(*curval[:n_state]))]
+
+        self.sync += If(self.sel, stmts2).Else(stmts1)
+
+
+class Descrambler(Module):
+    def __init__(self, n_io1, n_io2, n_state=23, taps=[17, 22]):
+        self.i1 = Signal(n_io1)
+        self.o1 = Signal(n_io1)
+        self.i2 = Signal(n_io2)
+        self.o2 = Signal(n_io2)
+        self.sel = Signal()
+
+        # # #
+
+        state = Signal(n_state, reset=1)
+
+        stmts1 = []
+        stmts2 = []
+        for stmts, si, so in ((stmts1, self.i1, self.o1),
+                              (stmts2, self.i2, self.o2)):
+            curval = [state[i] for i in range(n_state)]
+            for i in reversed(range(len(si))):
+                flip = reduce(xor, [curval[tap] for tap in taps])
+                stmts += [so[i].eq(si[i] ^ flip)]
+                curval.insert(0, si[i])
+                curval.pop()
+
+            stmts += [state.eq(Cat(*curval[:n_state]))]
+
+        self.sync += If(self.sel, stmts2).Else(stmts1)
 
 
 def K(x, y):
     return (y << 5) | x
 
 
+aux_coding_comma = [
+    K(28, 5),
+    K(28, 0),
+    K(28, 1),
+    K(28, 2),
+    K(23, 7),
+    K(27, 7),
+    K(29, 7),
+    K(30, 7),
+]
+
+
+aux_coding_nocomma = [
+    K(28, 0),
+    K(28, 2),
+    K(28, 3),
+    K(28, 4),
+    K(23, 7),
+    K(27, 7),
+    K(29, 7),
+    K(30, 7),
+]
+
+
 class LinkLayerTX(Module):
     def __init__(self, encoder):
         nwords = len(encoder.k)
         # nwords must be a power of 2
         assert nwords & (nwords - 1) == 0
 
-        self.link_init = Signal()
-        self.signal_rx_ready = Signal()
-
         self.aux_frame = Signal()
         self.aux_data = Signal(2*nwords)
         self.aux_ack = Signal()
@@ -49,93 +108,64 @@ class LinkLayerTX(Module):
 
         # # #
 
-        # Idle and auxiliary traffic use special characters excluding K.28.7,
-        # K.29.7 and K.30.7 in order to easily separate the link initialization
-        # phase (K.28.7 is additionally excluded as we cannot guarantee its
-        # non-repetition here).
+        # Idle and auxiliary traffic use special characters defined in the
+        # aux_coding_* tables.
+        # The first (or only) character uses aux_coding_comma which guarantees
+        # that commas appear regularly in the absence of traffic.
+        # The subsequent characters, if any (depending on the transceiver
+        # serialization ratio) use aux_coding_nocomma which does not contain
+        # commas. This permits aligning the comma to the first character at
+        # the receiver.
+        #
         # A set of 8 special characters is chosen using a 3-bit control word.
         # This control word is scrambled to reduce EMI. The control words have
         # the following meanings:
         #   100 idle/auxiliary framing
         #   0AB 2 bits of auxiliary data
-        aux_scrambler = ResetInserter()(CEInserter()(Scrambler(3*nwords)))
-        self.submodules += aux_scrambler
+        #
+        # RT traffic uses D characters and is also scrambled. The aux and RT
+        # scramblers are multiplicative and share the same state so that idle
+        # or aux traffic can synchronize the RT descrambler.
+
+        scrambler = Scrambler(3*nwords, 8*nwords)
+        self.submodules += scrambler
+
+        # scrambler input
         aux_data_ctl = []
         for i in range(nwords):
             aux_data_ctl.append(self.aux_data[i*2:i*2+2])
             aux_data_ctl.append(0)
         self.comb += [
             If(self.aux_frame,
-                aux_scrambler.i.eq(Cat(*aux_data_ctl))
+                scrambler.i1.eq(Cat(*aux_data_ctl))
             ).Else(
-                aux_scrambler.i.eq(Replicate(0b100, nwords))
+                scrambler.i1.eq(Replicate(0b100, nwords))
             ),
-            aux_scrambler.reset.eq(self.link_init),
-            aux_scrambler.ce.eq(~self.rt_frame),
+            scrambler.i2.eq(self.rt_data),
+            scrambler.sel.eq(self.rt_frame),
             self.aux_ack.eq(~self.rt_frame)
         ]
+
+        # compensate for scrambler latency
+        rt_frame_r = Signal()
+        self.sync += rt_frame_r.eq(self.rt_frame)
+
+        # scrambler output
         for i in range(nwords):
-            scrambled_ctl = aux_scrambler.o[i*3:i*3+3]
+            scrambled_ctl = scrambler.o1[i*3:i*3+3]
+            if i:
+                aux_coding = aux_coding_nocomma
+            else:
+                aux_coding = aux_coding_comma
             self.sync += [
                 encoder.k[i].eq(1),
-                If(scrambled_ctl == 7,
-                    encoder.d[i].eq(K(23, 7))
-                ).Else(
-                    encoder.d[i].eq(K(28, scrambled_ctl))
-                )
+                encoder.d[i].eq(Array(aux_coding)[scrambled_ctl])
             ]
-
-        # Real-time traffic uses data characters and is framed by the special
-        # characters of auxiliary traffic. RT traffic is also scrambled.
-        rt_scrambler = ResetInserter()(CEInserter()(Scrambler(8*nwords)))
-        self.submodules += rt_scrambler
-        self.comb += [
-            rt_scrambler.i.eq(self.rt_data),
-            rt_scrambler.reset.eq(self.link_init),
-            rt_scrambler.ce.eq(self.rt_frame)
-        ]
-        rt_frame_r = Signal()
-        self.sync += [
-            rt_frame_r.eq(self.rt_frame),
+        self.sync += \
             If(rt_frame_r,
                 [k.eq(0) for k in encoder.k],
-                [d.eq(rt_scrambler.o[i*8:i*8+8]) for i, d in enumerate(encoder.d)]
+                [d.eq(scrambler.o2[i*8:i*8+8]) for i, d in enumerate(encoder.d)]
             )
-        ]
-
-        # During link init, send a series of 1*K.28.7 (comma) + 31*K.29.7/K.30.7
-        # The receiving end configures its transceiver to also place the comma
-        # on its LSB, achieving fixed (or known) latency and alignment of
-        # packet starts.
-        # K.29.7 and K.30.7 are chosen to avoid comma alignment issues arising
-        # from K.28.7.
-        # K.30.7 is sent instead of K.29.7 to signal the alignment of the local
-        # receiver, thus the remote can end its link initialization pattern.
-        link_init_r = Signal()
-        link_init_counter = Signal(max=32//nwords)
-        self.sync += [
-            link_init_r.eq(self.link_init),
-            If(link_init_r,
-                link_init_counter.eq(link_init_counter + 1),
-                [k.eq(1) for k in encoder.k],
-                If(self.signal_rx_ready,
-                    [d.eq(K(30, 7)) for d in encoder.d[1:]]
-                ).Else(
-                    [d.eq(K(29, 7)) for d in encoder.d[1:]]
-                ),
-                If(link_init_counter == 0,
-                    encoder.d[0].eq(K(28, 7)),
-                ).Else(
-                    If(self.signal_rx_ready,
-                        encoder.d[0].eq(K(30, 7))
-                    ).Else(
-                        encoder.d[0].eq(K(29, 7))
-                    )
-                )
-            ).Else(
-                link_init_counter.eq(0)
-            )
-        ]
 
 
 class LinkLayerRX(Module):
@@ -144,9 +174,6 @@ class LinkLayerRX(Module):
         # nwords must be a power of 2
         assert nwords & (nwords - 1) == 0
 
-        self.link_init = Signal()
-        self.remote_rx_ready = Signal()
-
         self.aux_stb = Signal()
         self.aux_frame = Signal()
         self.aux_data = Signal(2*nwords)
@@ -156,65 +183,46 @@ class LinkLayerRX(Module):
 
         # # #
 
-        aux_descrambler = ResetInserter()(CEInserter()(Scrambler(3*nwords)))
-        rt_descrambler = ResetInserter()(CEInserter()(Scrambler(8*nwords)))
-        self.submodules += aux_descrambler, rt_descrambler
+        descrambler = Descrambler(3*nwords, 8*nwords)
+        self.submodules += descrambler
+
+        # scrambler input
+        all_decoded_aux = []
+        for i, d in enumerate(decoders):
+            decoded_aux = Signal(3)
+            all_decoded_aux.append(decoded_aux)
+
+            if i:
+                aux_coding = aux_coding_nocomma
+            else:
+                aux_coding = aux_coding_comma
+
+            cases = {code: decoded_aux.eq(i) for i, code in enumerate(aux_coding)}
+            self.comb += Case(d.d, cases).makedefault()
+
         self.comb += [
-            self.aux_frame.eq(~aux_descrambler.o[2]),
+            descrambler.i1.eq(Cat(*all_decoded_aux)),
+            descrambler.i2.eq(Cat(*[d.d for d in decoders])),
+            descrambler.sel.eq(~decoders[0].k)
+        ]
+
+        # scrambler output
+        self.comb += [
+            self.aux_frame.eq(~descrambler.o1[2]),
             self.aux_data.eq(
-                Cat(*[aux_descrambler.o[3*i:3*i+2] for i in range(nwords)])),
-            self.rt_data.eq(rt_descrambler.o),
-        ]
-
-        aux_stb_d = Signal()
-        rt_frame_d = Signal()
-        self.sync += [
-            self.aux_stb.eq(aux_stb_d),
-            self.rt_frame.eq(rt_frame_d)
-        ]
-
-        link_init_char = Signal()
-        self.comb += [
-            link_init_char.eq(
-                (decoders[0].d == K(28, 7)) |
-                (decoders[0].d == K(29, 7)) |
-                (decoders[0].d == K(30, 7))),
-            If(decoders[0].k,
-                If(link_init_char,
-                    aux_descrambler.reset.eq(1),
-                    rt_descrambler.reset.eq(1)
-                ).Else(
-                    aux_stb_d.eq(1)
-                ),
-                aux_descrambler.ce.eq(1)
-            ).Else(
-                rt_frame_d.eq(1),
-                rt_descrambler.ce.eq(1)
-            ),
-            aux_descrambler.i.eq(Cat(*[d.d[5:] for d in decoders])),
-            rt_descrambler.i.eq(Cat(*[d.d for d in decoders]))
+                Cat(*[descrambler.o1[3*i:3*i+2] for i in range(nwords)])),
+            self.rt_data.eq(descrambler.o2)
         ]
         self.sync += [
-            self.link_init.eq(0),
-            If(decoders[0].k,
-                If(link_init_char, self.link_init.eq(1)),
-                If(decoders[0].d == K(30, 7),
-                    self.remote_rx_ready.eq(1)
-                ).Elif(decoders[0].d != K(28, 7),
-                    self.remote_rx_ready.eq(0)
-                ),
-                If(decoders[1].d == K(30, 7),
-                    self.remote_rx_ready.eq(1)
-                ) if len(decoders) > 1 else None
-            ).Else(
-                self.remote_rx_ready.eq(0)
-            )
+            self.aux_stb.eq(decoders[0].k),
+            self.rt_frame.eq(~decoders[0].k)
         ]
 
 
+
 class LinkLayer(Module, AutoCSR):
-    def __init__(self, encoder, decoders, rx_ready_confirm_cycles):
-        self.link_status = CSRStatus(3)
+    def __init__(self, encoder, decoders):
+        self.link_status = CSRStatus()
 
         # control signals, in rtio clock domain
         self.reset = Signal()
@@ -242,6 +250,8 @@ class LinkLayer(Module, AutoCSR):
         self.rx_rt_frame = Signal()
         self.rx_rt_data = rx.rt_data
 
+        # # #
+
         ready_r = Signal()
         ready_rx = Signal()
         self.sync.rtio += ready_r.eq(self.ready)
@@ -251,8 +261,10 @@ class LinkLayer(Module, AutoCSR):
             self.rx_aux_frame.eq(rx.aux_frame & ready_rx),
             self.rx_rt_frame.eq(rx.rt_frame & ready_rx),
         ]
+        self.specials += MultiReg(ready_r, self.link_status.status)
 
-        # # #
+        wait_scrambler = WaitTimer(15)
+        self.submodules += wait_scrambler
 
         fsm = ClockDomainsRenamer("rtio")(
             ResetInserter()(FSM(reset_state="RESET_RX")))
@@ -260,68 +272,17 @@ class LinkLayer(Module, AutoCSR):
 
         self.comb += fsm.reset.eq(self.reset)
 
-        rx_remote_rx_ready = Signal()
-        rx_link_init = Signal()
-        rx.remote_rx_ready.attr.add("no_retiming")
-        rx.link_init.attr.add("no_retiming")
-        self.specials += [
-            MultiReg(rx.remote_rx_ready, rx_remote_rx_ready, "rtio"),
-            MultiReg(rx.link_init, rx_link_init, "rtio")
-        ]
-
-        link_status = BusSynchronizer(3, "rtio", "sys")
-        self.submodules += link_status
-        self.comb += self.link_status.status.eq(link_status.o)
-
-        wait_confirm = ClockDomainsRenamer("rtio")(
-            WaitTimer(rx_ready_confirm_cycles))
-        self.submodules += wait_confirm
-        signal_rx_ready_margin = ClockDomainsRenamer("rtio")(WaitTimer(15))
-        self.submodules += signal_rx_ready_margin
-
         fsm.act("RESET_RX",
-            link_status.i.eq(0),
-            tx.link_init.eq(1),
             self.rx_reset.eq(1),
-            NextState("WAIT_LOCAL_RX_READY")
+            NextState("WAIT_RX_READY")
         )
-        fsm.act("WAIT_LOCAL_RX_READY",
-            link_status.i.eq(1),
-            tx.link_init.eq(1),
-            If(self.rx_ready, NextState("CONFIRM_LOCAL_RX_READY"))
+        fsm.act("WAIT_RX_READY",
+            If(self.rx_ready, NextState("WAIT_SCRAMBLER_SYNC"))
         )
-        fsm.act("CONFIRM_LOCAL_RX_READY",
-            link_status.i.eq(2),
-            tx.link_init.eq(1),
-            wait_confirm.wait.eq(1),
-            If(wait_confirm.done, NextState("WAIT_REMOTE_RX_READY")),
-            If(~rx_link_init, NextState("RESET_RX"))
-        )
-        fsm.act("WAIT_REMOTE_RX_READY",
-            link_status.i.eq(3),
-            tx.link_init.eq(1),
-            tx.signal_rx_ready.eq(1),
-            If(rx_remote_rx_ready, NextState("ENSURE_SIGNAL_RX_READY"))
-        )
-        # If the transceiver transmits one character per RTIO cycle,
-        # we may be unlucky and signal_rx_ready will transmit a comma
-        # on the first cycle instead of a "RX ready" character.
-        # Further, we need to ensure the rx.remote_rx_ready
-        # gets through MultiReg to rx_remote_rx_ready at the receiver.
-        # So transmit the "RX ready" pattern for several cycles.
-        fsm.act("ENSURE_SIGNAL_RX_READY",
-            link_status.i.eq(3),
-            tx.link_init.eq(1),
-            tx.signal_rx_ready.eq(1),
-            signal_rx_ready_margin.wait.eq(1),
-            If(signal_rx_ready_margin.done, NextState("WAIT_REMOTE_LINK_UP"))
-        )
-        fsm.act("WAIT_REMOTE_LINK_UP",
-            link_status.i.eq(4),
-            If(~rx_link_init, NextState("READY"))
+        fsm.act("WAIT_SCRAMBLER_SYNC",
+            wait_scrambler.wait.eq(1),
+            If(wait_scrambler.done, NextState("READY")),
         )
         fsm.act("READY",
-            link_status.i.eq(5),
-            If(rx_link_init, NextState("RESET_RX")),  # TODO: remove this, link deinit should be detected at upper layer
             self.ready.eq(1)
         )
diff --git a/artiq/gateware/drtio/transceiver/gtx_7series.py b/artiq/gateware/drtio/transceiver/gtx_7series.py
index 00166e11f..68885bf08 100644
--- a/artiq/gateware/drtio/transceiver/gtx_7series.py
+++ b/artiq/gateware/drtio/transceiver/gtx_7series.py
@@ -185,7 +185,7 @@ class GTX_1000BASE_BX10(Module):
             self.decoders[1].input.eq(rxdata[10:])
         ]
 
-        clock_aligner = BruteforceClockAligner(0b0001111100, self.rtio_clk_freq)
+        clock_aligner = BruteforceClockAligner(0b0101111100, self.rtio_clk_freq)
         self.submodules += clock_aligner
         self.comb += [
             clock_aligner.rxdata.eq(rxdata),
diff --git a/artiq/runtime.rs/src/drtio.rs b/artiq/runtime.rs/src/drtio.rs
index 214295697..089dd55b4 100644
--- a/artiq/runtime.rs/src/drtio.rs
+++ b/artiq/runtime.rs/src/drtio.rs
@@ -3,7 +3,7 @@ use sched::{Waiter, Spawner};
 
 fn drtio_link_is_up() -> bool {
     unsafe {
-        csr::drtio::link_status_read() == 5
+        csr::drtio::link_status_read() == 1
     }
 }
 
@@ -31,7 +31,10 @@ fn drtio_init_channel(channel: u16) {
 pub fn link_thread(waiter: Waiter, _spawner: Spawner) {
     loop {
         waiter.until(drtio_link_is_up).unwrap();
-        info!("link is up");
+        info!("link RX is up");
+
+        waiter.sleep(300);
+        info!("wait for remote side done");
 
         drtio_sync_tsc();
         info!("TSC synced");
diff --git a/artiq/test/gateware/drtio/test_aux_controller.py b/artiq/test/gateware/drtio/test_aux_controller.py
index d9a3ef467..f07f5eb6d 100644
--- a/artiq/test/gateware/drtio/test_aux_controller.py
+++ b/artiq/test/gateware/drtio/test_aux_controller.py
@@ -44,13 +44,7 @@ class TestAuxController(unittest.TestCase):
         dut = TB(4)
 
         def link_init():
-            yield dut.link_layer.tx.link_init.eq(1)
-            yield
-            yield
-            yield dut.link_layer.tx.link_init.eq(0)
-            while not (yield dut.link_layer.rx.link_init):
-                yield
-            while (yield dut.link_layer.rx.link_init):
+            for i in range(8):
                 yield
             yield dut.link_layer.ready.eq(1)
 
diff --git a/artiq/test/gateware/drtio/test_full_stack.py b/artiq/test/gateware/drtio/test_full_stack.py
index eea0c2256..ae0ce7290 100644
--- a/artiq/test/gateware/drtio/test_full_stack.py
+++ b/artiq/test/gateware/drtio/test_full_stack.py
@@ -41,8 +41,7 @@ class DUT(Module):
         self.ttl1 = Signal()
         self.transceivers = DummyTransceiverPair(nwords)
         
-        self.submodules.master = DRTIOMaster(self.transceivers.alice,
-                                             ll_rx_ready_confirm=15)
+        self.submodules.master = DRTIOMaster(self.transceivers.alice)
 
         rx_synchronizer = DummyRXSynchronizer()
         self.submodules.phy0 = ttl_simple.Output(self.ttl0)
@@ -52,8 +51,7 @@ class DUT(Module):
             rtio.Channel.from_phy(self.phy1, ofifo_depth=4)
         ]
         self.submodules.satellite = DRTIOSatellite(
-            self.transceivers.bob, rx_synchronizer, rtio_channels,
-            ll_rx_ready_confirm=15)
+            self.transceivers.bob, rx_synchronizer, rtio_channels)
         
 
 class TestFullStack(unittest.TestCase):
diff --git a/artiq/test/gateware/drtio/test_link_layer.py b/artiq/test/gateware/drtio/test_link_layer.py
index 25a2cd2a0..3d161e670 100644
--- a/artiq/test/gateware/drtio/test_link_layer.py
+++ b/artiq/test/gateware/drtio/test_link_layer.py
@@ -22,15 +22,6 @@ def process(seq):
     return rseq
 
 
-class TestScrambler(unittest.TestCase):
-    def test_roundtrip(self):
-        seq = list(range(256))*3
-        scrambled_seq = process(seq)
-        descrambled_seq = process(scrambled_seq)
-        self.assertNotEqual(seq, scrambled_seq)
-        self.assertEqual(seq, descrambled_seq)
-
-
 class Loopback(Module):
     def __init__(self, nwords):
         ks = [Signal() for k in range(nwords)]
@@ -45,12 +36,9 @@ class TestLinkLayer(unittest.TestCase):
     def test_packets(self):
         dut = Loopback(4)
 
-        def link_init():
-            yield dut.tx.link_init.eq(1)
-            yield
-            yield
-            yield dut.tx.link_init.eq(0)
-            yield
+        def scrambler_sync():
+            for i in range(8):
+                yield
 
         rt_packets = [
             [0x12459970, 0x9938cdef, 0x12340000],
@@ -59,10 +47,7 @@ class TestLinkLayer(unittest.TestCase):
             [0x88277475, 0x19883332, 0x19837662, 0x81726668, 0x81876261]
         ]
         def transmit_rt_packets():
-            while not (yield dut.tx.link_init):
-                yield
-            while (yield dut.tx.link_init):
-                yield
+            yield from scrambler_sync()
 
             for packet in rt_packets:
                 yield dut.tx.rt_frame.eq(1)
@@ -78,10 +63,7 @@ class TestLinkLayer(unittest.TestCase):
         rx_rt_packets = []
         @passive
         def receive_rt_packets():
-            while not (yield dut.rx.link_init):
-                yield
-            while (yield dut.rx.link_init):
-                yield
+            yield from scrambler_sync()
 
             previous_frame = 0
             while True:
@@ -100,10 +82,7 @@ class TestLinkLayer(unittest.TestCase):
             [0xbb, 0xaa, 0xdd, 0xcc, 0x00, 0xff, 0xee]
         ]
         def transmit_aux_packets():
-            while not (yield dut.tx.link_init):
-                yield
-            while (yield dut.tx.link_init):
-                yield
+            yield from scrambler_sync()
 
             for packet in aux_packets:
                 yield dut.tx.aux_frame.eq(1)
@@ -123,10 +102,7 @@ class TestLinkLayer(unittest.TestCase):
         rx_aux_packets = []
         @passive
         def receive_aux_packets():
-            while not (yield dut.rx.link_init):
-                yield
-            while (yield dut.rx.link_init):
-                yield
+            yield from scrambler_sync()
 
             previous_frame = 0
             while True:
@@ -140,8 +116,7 @@ class TestLinkLayer(unittest.TestCase):
                         packet.append((yield dut.rx.aux_data))
                 yield
 
-        run_simulation(dut, [link_init(),
-                             transmit_rt_packets(), receive_rt_packets(),
+        run_simulation(dut, [transmit_rt_packets(), receive_rt_packets(),
                              transmit_aux_packets(), receive_aux_packets()])
 
         # print("RT:")

From f040e270415ee38c76b8df9e014925b8439f50d4 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Fri, 18 Nov 2016 17:44:48 +0800
Subject: [PATCH 089/134] drtio: add timeout on FIFO get space request

---
 artiq/gateware/drtio/rt_controller.py | 25 ++++++++++++++++++++-----
 1 file changed, 20 insertions(+), 5 deletions(-)

diff --git a/artiq/gateware/drtio/rt_controller.py b/artiq/gateware/drtio/rt_controller.py
index e301443df..a7c51c782 100644
--- a/artiq/gateware/drtio/rt_controller.py
+++ b/artiq/gateware/drtio/rt_controller.py
@@ -1,5 +1,6 @@
 from migen import *
 from migen.genlib.cdc import MultiReg
+from migen.genlib.misc import WaitTimer
 
 from misoc.interconnect.csr import *
 
@@ -21,6 +22,7 @@ class _CSRs(AutoCSR):
         self.o_dbg_last_timestamp = CSRStatus(64)
         self.o_reset_channel_status = CSR()
         self.o_wait = CSRStatus()
+        self.o_fifo_space_timeout = CSR()
 
 
 class RTController(Module):
@@ -96,6 +98,14 @@ class RTController(Module):
             If(sequence_error_set, status_sequence_error.eq(1)),
         ]
 
+        signal_fifo_space_timeout = Signal()
+        self.sync += [
+            If(self.csrs.o_fifo_space_timeout.re, self.csrs.o_fifo_space_timeout.w.eq(0)),
+            If(signal_fifo_space_timeout, self.csrs.o_fifo_space_timeout.w.eq(1))
+        ]
+        timeout_counter = WaitTimer(8191)
+        self.submodules += timeout_counter
+
         # TODO: collision, replace, busy
         cond_sequence_error = self.kcsrs.o_timestamp.storage < last_timestamps.dat_r
         cond_underflow = ((self.kcsrs.o_timestamp.storage[fine_ts_width:]
@@ -154,6 +164,11 @@ class RTController(Module):
                 ).Else(
                     NextState("GET_FIFO_SPACE")
                 )
+            ),
+            timeout_counter.wait.eq(1),
+            If(timeout_counter.done,
+                signal_fifo_space_timeout.eq(1),
+                NextState("IDLE")
             )
         )
 
@@ -180,8 +195,8 @@ class RTManager(Module, AutoCSR):
     def __init__(self, rt_packets):
         self.request_echo = CSR()
 
-        self.err_present = CSR()
-        self.err_code = CSRStatus(8)
+        self.packet_err_present = CSR()
+        self.packet_err_code = CSRStatus(8)
 
         self.update_packet_cnt = CSR()
         self.packet_cnt_tx = CSRStatus(32)
@@ -196,9 +211,9 @@ class RTManager(Module, AutoCSR):
         ]
 
         self.comb += [
-            self.err_present.w.eq(rt_packets.error_not),
-            rt_packets.error_not_ack.eq(self.err_present.re),
-            self.err_code.status.eq(rt_packets.error_code)
+            self.packet_err_present.w.eq(rt_packets.error_not),
+            rt_packets.error_not_ack.eq(self.packet_err_present.re),
+            self.packet_err_code.status.eq(rt_packets.error_code)
         ]
 
         self.sync += \

From 4d07974a3409467e5c0aa83cccc3a85892ad0bf7 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Fri, 18 Nov 2016 17:45:33 +0800
Subject: [PATCH 090/134] drtio: reset link from CPU

---
 artiq/gateware/drtio/link_layer.py | 41 +++++++++++++++++-------------
 artiq/runtime.rs/src/drtio.rs      | 23 +++++++++++------
 2 files changed, 39 insertions(+), 25 deletions(-)

diff --git a/artiq/gateware/drtio/link_layer.py b/artiq/gateware/drtio/link_layer.py
index 24d325f9c..52af15b4a 100644
--- a/artiq/gateware/drtio/link_layer.py
+++ b/artiq/gateware/drtio/link_layer.py
@@ -3,7 +3,7 @@ from operator import xor, or_
 
 from migen import *
 from migen.genlib.fsm import *
-from migen.genlib.cdc import MultiReg
+from migen.genlib.cdc import MultiReg, PulseSynchronizer
 from migen.genlib.misc import WaitTimer
 
 from misoc.interconnect.csr import *
@@ -223,10 +223,8 @@ class LinkLayerRX(Module):
 class LinkLayer(Module, AutoCSR):
     def __init__(self, encoder, decoders):
         self.link_status = CSRStatus()
+        self.link_reset = CSR()
 
-        # control signals, in rtio clock domain
-        self.reset = Signal()
-        self.ready = Signal()
         # pulsed to reset receiver, rx_ready must immediately go low
         self.rx_reset = Signal()
         # receiver locked including comma alignment
@@ -252,37 +250,46 @@ class LinkLayer(Module, AutoCSR):
 
         # # #
 
-        ready_r = Signal()
+        ready = Signal()
+        reset_ps = PulseSynchronizer("sys", "rtio")
+        done_ps = PulseSynchronizer("rtio", "sys")
+        self.submodules += reset_ps, done_ps
+        self.comb += reset_ps.i.eq(self.link_reset.re)
+        self.sync += [
+            If(done_ps.o, ready.eq(1)),
+            If(reset_ps.i, ready.eq(0)),
+        ]
+        self.comb += self.link_status.status.eq(ready)
+
         ready_rx = Signal()
-        self.sync.rtio += ready_r.eq(self.ready)
-        ready_r.attr.add("no_retiming")
-        self.specials += MultiReg(ready_r, ready_rx, "rtio_rx")
+        ready.attr.add("no_retiming")
+        self.specials += MultiReg(ready, ready_rx, "rtio_rx")
         self.comb += [
             self.rx_aux_frame.eq(rx.aux_frame & ready_rx),
             self.rx_rt_frame.eq(rx.rt_frame & ready_rx),
         ]
-        self.specials += MultiReg(ready_r, self.link_status.status)
 
-        wait_scrambler = WaitTimer(15)
+        wait_scrambler = ClockDomainsRenamer("rtio")(WaitTimer(15))
         self.submodules += wait_scrambler
 
-        fsm = ClockDomainsRenamer("rtio")(
-            ResetInserter()(FSM(reset_state="RESET_RX")))
+        fsm = ClockDomainsRenamer("rtio")(FSM(reset_state="RESET_RX"))
         self.submodules += fsm
 
-        self.comb += fsm.reset.eq(self.reset)
-
         fsm.act("RESET_RX",
             self.rx_reset.eq(1),
             NextState("WAIT_RX_READY")
         )
         fsm.act("WAIT_RX_READY",
-            If(self.rx_ready, NextState("WAIT_SCRAMBLER_SYNC"))
+            If(self.rx_ready, NextState("WAIT_SCRAMBLER_SYNC")),
+            If(reset_ps.o, NextState("RESET_RX"))
         )
         fsm.act("WAIT_SCRAMBLER_SYNC",
             wait_scrambler.wait.eq(1),
-            If(wait_scrambler.done, NextState("READY")),
+            If(wait_scrambler.done,
+                done_ps.i.eq(1),
+                NextState("READY")
+            ),
         )
         fsm.act("READY",
-            self.ready.eq(1)
+            If(reset_ps.o, NextState("RESET_RX"))
         )
diff --git a/artiq/runtime.rs/src/drtio.rs b/artiq/runtime.rs/src/drtio.rs
index 089dd55b4..618a9f7dd 100644
--- a/artiq/runtime.rs/src/drtio.rs
+++ b/artiq/runtime.rs/src/drtio.rs
@@ -7,6 +7,12 @@ fn drtio_link_is_up() -> bool {
     }
 }
 
+fn drtio_reset_link() {
+    unsafe {
+        csr::drtio::link_reset_write(1)
+    }
+}
+
 fn drtio_sync_tsc() {
     unsafe {
         csr::drtio::set_time_write(1);
@@ -33,7 +39,7 @@ pub fn link_thread(waiter: Waiter, _spawner: Spawner) {
         waiter.until(drtio_link_is_up).unwrap();
         info!("link RX is up");
 
-        waiter.sleep(300);
+        waiter.sleep(300).unwrap();
         info!("wait for remote side done");
 
         drtio_sync_tsc();
@@ -48,23 +54,24 @@ pub fn link_thread(waiter: Waiter, _spawner: Spawner) {
     }
 }
 
-fn drtio_error_present() -> bool {
+fn drtio_packet_error_present() -> bool {
     unsafe {
-        csr::drtio::err_present_read() != 0
+        csr::drtio::packet_err_present_read() != 0
     }
 }
 
-fn drtio_get_error() -> u8 {
+fn drtio_get_packet_error() -> u8 {
     unsafe {
-        let err = csr::drtio::err_code_read();
-        csr::drtio::err_present_write(1);
+        let err = csr::drtio::packet_err_code_read();
+        csr::drtio::packet_err_present_write(1);
         err
     }
 }
 
 pub fn error_thread(waiter: Waiter, _spawner: Spawner) {
     loop {
-        waiter.until(drtio_error_present).unwrap();
-        error!("DRTIO error {}", drtio_get_error());   
+        waiter.until(drtio_packet_error_present).unwrap();
+        error!("DRTIO packet error {}", drtio_get_packet_error());
+        drtio_reset_link();
     }
 }

From ba94ed8f4bc8ddcd7d4106721059265cc5e260fa Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Sat, 19 Nov 2016 00:05:59 +0800
Subject: [PATCH 091/134] drtio: check for absence of disparity errors before
 claiming RX ready

---
 artiq/gateware/drtio/link_layer.py            |  2 +-
 .../drtio/transceiver/gtx_7series_init.py     | 81 +++++++++++++------
 2 files changed, 59 insertions(+), 24 deletions(-)

diff --git a/artiq/gateware/drtio/link_layer.py b/artiq/gateware/drtio/link_layer.py
index 52af15b4a..bd8164dd8 100644
--- a/artiq/gateware/drtio/link_layer.py
+++ b/artiq/gateware/drtio/link_layer.py
@@ -288,7 +288,7 @@ class LinkLayer(Module, AutoCSR):
             If(wait_scrambler.done,
                 done_ps.i.eq(1),
                 NextState("READY")
-            ),
+            )
         )
         fsm.act("READY",
             If(reset_ps.o, NextState("RESET_RX"))
diff --git a/artiq/gateware/drtio/transceiver/gtx_7series_init.py b/artiq/gateware/drtio/transceiver/gtx_7series_init.py
index cf502ec24..01dfd7b2f 100644
--- a/artiq/gateware/drtio/transceiver/gtx_7series_init.py
+++ b/artiq/gateware/drtio/transceiver/gtx_7series_init.py
@@ -1,4 +1,6 @@
 from math import ceil
+from functools import reduce
+from operator import add
 
 from migen import *
 from migen.genlib.cdc import MultiReg, PulseSynchronizer
@@ -134,7 +136,7 @@ class GTXInit(Module):
 # Warning: Xilinx transceivers are LSB first, and comma needs to be flipped
 # compared to the usual 8b10b binary representation.
 class BruteforceClockAligner(Module):
-    def __init__(self, comma, rtio_clk_freq, check_period=6e-3, ready_time=50e-3):
+    def __init__(self, comma, rtio_clk_freq, check_period=6e-3):
         self.rxdata = Signal(20)
         self.restart = Signal()
 
@@ -144,9 +146,12 @@ class BruteforceClockAligner(Module):
         check_max_val = ceil(check_period*rtio_clk_freq)
         check_counter = Signal(max=check_max_val+1)
         check = Signal()
+        reset_check_counter = Signal()
         self.sync.rtio += [
             check.eq(0),
-            If(~self.ready,
+            If(reset_check_counter,
+                check_counter.eq(check_max_val)
+            ).Else(
                 If(check_counter == 0,
                     check.eq(1),
                     check_counter.eq(check_max_val)
@@ -156,37 +161,67 @@ class BruteforceClockAligner(Module):
             )
         ]
 
+        checks_reset = PulseSynchronizer("rtio", "rtio_rx")
+        self.submodules += checks_reset
+
         comma_n = ~comma & 0b1111111111
         comma_seen_rxclk = Signal()
         comma_seen = Signal()
         comma_seen_rxclk.attr.add("no_retiming")
         self.specials += MultiReg(comma_seen_rxclk, comma_seen)
-        comma_seen_reset = PulseSynchronizer("rtio", "rtio_rx")
-        self.submodules += comma_seen_reset
         self.sync.rtio_rx += \
-            If(comma_seen_reset.o,
+            If(checks_reset.o,
                 comma_seen_rxclk.eq(0)
             ).Elif((self.rxdata[:10] == comma) | (self.rxdata[:10] == comma_n),
                 comma_seen_rxclk.eq(1)
             )
 
-        self.comb += \
-            If(check,
-                If(~comma_seen, self.restart.eq(1)),
-                comma_seen_reset.i.eq(1)
+        error_seen_rxclk = Signal()
+        error_seen = Signal()
+        error_seen_rxclk.attr.add("no_retiming")
+        self.specials += MultiReg(error_seen_rxclk, error_seen)
+        rx1cnt = Signal(max=11)
+        self.sync.rtio_rx += [
+            rx1cnt.eq(reduce(add, [self.rxdata[i] for i in range(10)])),
+            If(checks_reset.o,
+                error_seen_rxclk.eq(0)
+            ).Elif((rx1cnt != 4) & (rx1cnt != 5) & (rx1cnt != 6),
+                error_seen_rxclk.eq(1)
             )
-
-        ready_counts = ceil(ready_time/check_period)
-        assert ready_counts > 1
-        ready_counter = Signal(max=ready_counts+1, reset=ready_counts)
-        self.sync.rtio += [
-            If(check,
-                If(comma_seen,
-                    If(ready_counter != 0, ready_counter.eq(ready_counter-1))
-                ).Else(
-                    ready_counter.eq(ready_counts)
-                )
-            ),
-            If(self.reset, ready_counter.eq(ready_counts))
         ]
-        self.comb += self.ready.eq(ready_counter == 0)
+
+        fsm = ClockDomainsRenamer("rtio")(FSM(reset_state="WAIT_COMMA"))
+        self.submodules += fsm
+
+        fsm.act("WAIT_COMMA",
+            If(check,
+                # Errors are still OK at this stage, as the transceiver
+                # has just been reset and may output garbage data.
+                If(comma_seen,
+                    NextState("WAIT_NOERROR")
+                ).Else(
+                    self.restart.eq(1)
+                ),
+                checks_reset.i.eq(1)
+            )
+        )
+        fsm.act("WAIT_NOERROR",
+            If(check,
+                If(comma_seen & ~error_seen,
+                    NextState("READY")
+                ).Else(
+                    self.restart.eq(1),
+                    NextState("WAIT_COMMA")
+                ),
+                checks_reset.i.eq(1)
+            )
+        )
+        fsm.act("READY",
+            reset_check_counter.eq(1),
+            self.ready.eq(1),
+            If(self.reset,
+                checks_reset.i.eq(1),
+                self.restart.eq(1),
+                NextState("WAIT_COMMA")
+            )
+        )

From 381e58434f2870ebaa0d7a043541307b67fa1708 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Sat, 19 Nov 2016 10:46:56 +0800
Subject: [PATCH 092/134] drtio: handle link restarts at transceiver level

---
 artiq/gateware/drtio/core.py                  | 11 ++---
 artiq/gateware/drtio/link_layer.py            | 40 +++++--------------
 .../gateware/drtio/transceiver/gtx_7series.py |  5 +--
 .../drtio/transceiver/gtx_7series_init.py     |  3 +-
 artiq/runtime.rs/src/drtio.rs                 |  7 ----
 5 files changed, 16 insertions(+), 50 deletions(-)

diff --git a/artiq/gateware/drtio/core.py b/artiq/gateware/drtio/core.py
index a51d22629..58baeb635 100644
--- a/artiq/gateware/drtio/core.py
+++ b/artiq/gateware/drtio/core.py
@@ -9,10 +9,7 @@ class DRTIOSatellite(Module):
     def __init__(self, transceiver, rx_synchronizer, channels, fine_ts_width=3, full_ts_width=63):
         self.submodules.link_layer = link_layer.LinkLayer(
             transceiver.encoder, transceiver.decoders)
-        self.comb += [
-            transceiver.rx_reset.eq(self.link_layer.rx_reset),
-            self.link_layer.rx_ready.eq(transceiver.rx_ready)
-        ]
+        self.comb += self.link_layer.rx_ready.eq(transceiver.rx_ready)
 
         link_layer_sync = SimpleNamespace(
             tx_aux_frame=self.link_layer.tx_aux_frame,
@@ -54,10 +51,8 @@ class DRTIOMaster(Module):
     def __init__(self, transceiver, channel_count=1024, fine_ts_width=3):
         self.submodules.link_layer = link_layer.LinkLayer(
             transceiver.encoder, transceiver.decoders)
-        self.comb += [
-            transceiver.rx_reset.eq(self.link_layer.rx_reset),
-            self.link_layer.rx_ready.eq(transceiver.rx_ready)
-        ]
+        self.comb += self.link_layer.rx_ready.eq(transceiver.rx_ready)
+
         self.submodules.rt_packets = rt_packets.RTPacketMaster(self.link_layer)
         self.submodules.rt_controller = rt_controller.RTController(
             self.rt_packets, channel_count, fine_ts_width)
diff --git a/artiq/gateware/drtio/link_layer.py b/artiq/gateware/drtio/link_layer.py
index bd8164dd8..bebc9dc7c 100644
--- a/artiq/gateware/drtio/link_layer.py
+++ b/artiq/gateware/drtio/link_layer.py
@@ -3,7 +3,7 @@ from operator import xor, or_
 
 from migen import *
 from migen.genlib.fsm import *
-from migen.genlib.cdc import MultiReg, PulseSynchronizer
+from migen.genlib.cdc import MultiReg
 from migen.genlib.misc import WaitTimer
 
 from misoc.interconnect.csr import *
@@ -223,11 +223,8 @@ class LinkLayerRX(Module):
 class LinkLayer(Module, AutoCSR):
     def __init__(self, encoder, decoders):
         self.link_status = CSRStatus()
-        self.link_reset = CSR()
 
-        # pulsed to reset receiver, rx_ready must immediately go low
-        self.rx_reset = Signal()
-        # receiver locked including comma alignment
+        # receiver locked, comma aligned, receiving valid 8b10b symbols
         self.rx_ready = Signal()
 
         tx = ClockDomainsRenamer("rtio")(LinkLayerTX(encoder))
@@ -251,45 +248,30 @@ class LinkLayer(Module, AutoCSR):
         # # #
 
         ready = Signal()
-        reset_ps = PulseSynchronizer("sys", "rtio")
-        done_ps = PulseSynchronizer("rtio", "sys")
-        self.submodules += reset_ps, done_ps
-        self.comb += reset_ps.i.eq(self.link_reset.re)
-        self.sync += [
-            If(done_ps.o, ready.eq(1)),
-            If(reset_ps.i, ready.eq(0)),
-        ]
-        self.comb += self.link_status.status.eq(ready)
-
+        ready_r = Signal()
+        self.sync.rtio += ready_r.eq(ready)
         ready_rx = Signal()
-        ready.attr.add("no_retiming")
-        self.specials += MultiReg(ready, ready_rx, "rtio_rx")
+        ready_r.attr.add("no_retiming")
+        self.specials += MultiReg(ready_r, ready_rx, "rtio_rx")
         self.comb += [
             self.rx_aux_frame.eq(rx.aux_frame & ready_rx),
             self.rx_rt_frame.eq(rx.rt_frame & ready_rx),
         ]
+        self.specials += MultiReg(ready_r, self.link_status.status)
 
         wait_scrambler = ClockDomainsRenamer("rtio")(WaitTimer(15))
         self.submodules += wait_scrambler
 
-        fsm = ClockDomainsRenamer("rtio")(FSM(reset_state="RESET_RX"))
+        fsm = ClockDomainsRenamer("rtio")(FSM(reset_state="WAIT_RX_READY"))
         self.submodules += fsm
 
-        fsm.act("RESET_RX",
-            self.rx_reset.eq(1),
-            NextState("WAIT_RX_READY")
-        )
         fsm.act("WAIT_RX_READY",
-            If(self.rx_ready, NextState("WAIT_SCRAMBLER_SYNC")),
-            If(reset_ps.o, NextState("RESET_RX"))
+            If(self.rx_ready, NextState("WAIT_SCRAMBLER_SYNC"))
         )
         fsm.act("WAIT_SCRAMBLER_SYNC",
             wait_scrambler.wait.eq(1),
-            If(wait_scrambler.done,
-                done_ps.i.eq(1),
-                NextState("READY")
-            )
+            If(wait_scrambler.done, NextState("READY"))
         )
         fsm.act("READY",
-            If(reset_ps.o, NextState("RESET_RX"))
+            ready.eq(1)
         )
diff --git a/artiq/gateware/drtio/transceiver/gtx_7series.py b/artiq/gateware/drtio/transceiver/gtx_7series.py
index 68885bf08..74ee4f1af 100644
--- a/artiq/gateware/drtio/transceiver/gtx_7series.py
+++ b/artiq/gateware/drtio/transceiver/gtx_7series.py
@@ -20,7 +20,6 @@ class GTX_1000BASE_BX10(Module):
             Decoder(True)) for _ in range(2)]
         self.submodules += self.decoders
 
-        self.rx_reset = Signal()
         self.rx_ready = Signal()
 
         # # #
@@ -49,8 +48,7 @@ class GTX_1000BASE_BX10(Module):
             GTXInit(self.rtio_clk_freq, True))
         self.submodules += tx_init, rx_init
         self.comb += tx_init.cplllock.eq(cplllock), \
-                     rx_init.cplllock.eq(cplllock), \
-                     rx_init.restart.eq(self.rx_reset)
+                     rx_init.cplllock.eq(cplllock)
 
         txoutclk = Signal()
         txdata = Signal(20)
@@ -190,7 +188,6 @@ class GTX_1000BASE_BX10(Module):
         self.comb += [
             clock_aligner.rxdata.eq(rxdata),
             rx_init.restart.eq(clock_aligner.restart),
-            clock_aligner.reset.eq(self.rx_reset),
             self.rx_ready.eq(clock_aligner.ready)
         ]
 
diff --git a/artiq/gateware/drtio/transceiver/gtx_7series_init.py b/artiq/gateware/drtio/transceiver/gtx_7series_init.py
index 01dfd7b2f..2b7ae39bc 100644
--- a/artiq/gateware/drtio/transceiver/gtx_7series_init.py
+++ b/artiq/gateware/drtio/transceiver/gtx_7series_init.py
@@ -140,7 +140,6 @@ class BruteforceClockAligner(Module):
         self.rxdata = Signal(20)
         self.restart = Signal()
 
-        self.reset = Signal()
         self.ready = Signal()
 
         check_max_val = ceil(check_period*rtio_clk_freq)
@@ -219,7 +218,7 @@ class BruteforceClockAligner(Module):
         fsm.act("READY",
             reset_check_counter.eq(1),
             self.ready.eq(1),
-            If(self.reset,
+            If(error_seen,
                 checks_reset.i.eq(1),
                 self.restart.eq(1),
                 NextState("WAIT_COMMA")
diff --git a/artiq/runtime.rs/src/drtio.rs b/artiq/runtime.rs/src/drtio.rs
index 618a9f7dd..9804bc211 100644
--- a/artiq/runtime.rs/src/drtio.rs
+++ b/artiq/runtime.rs/src/drtio.rs
@@ -7,12 +7,6 @@ fn drtio_link_is_up() -> bool {
     }
 }
 
-fn drtio_reset_link() {
-    unsafe {
-        csr::drtio::link_reset_write(1)
-    }
-}
-
 fn drtio_sync_tsc() {
     unsafe {
         csr::drtio::set_time_write(1);
@@ -72,6 +66,5 @@ pub fn error_thread(waiter: Waiter, _spawner: Spawner) {
     loop {
         waiter.until(drtio_packet_error_present).unwrap();
         error!("DRTIO packet error {}", drtio_get_packet_error());
-        drtio_reset_link();
     }
 }

From abd1b2a94e7c9e0adfe8f39d5e740209707fcbcb Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Sat, 19 Nov 2016 11:01:09 +0800
Subject: [PATCH 093/134] drtio: wait longer for remote (bruteforce clock
 aligner can be slow)

---
 artiq/runtime.rs/src/drtio.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/artiq/runtime.rs/src/drtio.rs b/artiq/runtime.rs/src/drtio.rs
index 9804bc211..6050a8c1b 100644
--- a/artiq/runtime.rs/src/drtio.rs
+++ b/artiq/runtime.rs/src/drtio.rs
@@ -33,7 +33,7 @@ pub fn link_thread(waiter: Waiter, _spawner: Spawner) {
         waiter.until(drtio_link_is_up).unwrap();
         info!("link RX is up");
 
-        waiter.sleep(300).unwrap();
+        waiter.sleep(500).unwrap();
         info!("wait for remote side done");
 
         drtio_sync_tsc();

From 02adae7397f24e2048279f3d6830746a90b48b79 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Sat, 19 Nov 2016 11:01:33 +0800
Subject: [PATCH 094/134] drtio: fix link shutdown

---
 artiq/gateware/drtio/link_layer.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/artiq/gateware/drtio/link_layer.py b/artiq/gateware/drtio/link_layer.py
index bebc9dc7c..2b4b7b45b 100644
--- a/artiq/gateware/drtio/link_layer.py
+++ b/artiq/gateware/drtio/link_layer.py
@@ -273,5 +273,6 @@ class LinkLayer(Module, AutoCSR):
             If(wait_scrambler.done, NextState("READY"))
         )
         fsm.act("READY",
-            ready.eq(1)
+            ready.eq(1),
+            If(~self.rx_ready, NextState("WAIT_RX_READY"))
         )

From 0aaf120ca7658d515481661defcea7c0cbcd17dd Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Tue, 22 Nov 2016 22:46:19 +0800
Subject: [PATCH 095/134] kc705: remove stale DDS definition

---
 artiq/gateware/targets/kc705.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/artiq/gateware/targets/kc705.py b/artiq/gateware/targets/kc705.py
index d7672b435..502ca5902 100755
--- a/artiq/gateware/targets/kc705.py
+++ b/artiq/gateware/targets/kc705.py
@@ -229,8 +229,6 @@ class NIST_CLOCK(_NIST_Ions):
         rtio_channels.append(rtio.LogChannel())
 
         self.add_rtio(rtio_channels)
-        assert self.rtio.fine_ts_width <= 3
-        self.config["DDS_RTIO_CLK_RATIO"] = 24 >> self.rtio.fine_ts_width
 
 
 class NIST_QC2(_NIST_Ions):
@@ -309,8 +307,6 @@ class NIST_QC2(_NIST_Ions):
         rtio_channels.append(rtio.LogChannel())
 
         self.add_rtio(rtio_channels)
-        assert self.rtio.fine_ts_width <= 3
-        self.config["DDS_RTIO_CLK_RATIO"] = 24 >> self.rtio.fine_ts_width
 
 
 def main():

From 9acc7d135ea80cd05aa6236618c7963b5aa0deb1 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Tue, 22 Nov 2016 22:46:50 +0800
Subject: [PATCH 096/134] gateware: common RTIO interface

---
 artiq/gateware/drtio/core.py                 |   4 +-
 artiq/gateware/drtio/rt_controller.py        |  33 +++--
 artiq/gateware/rtio/__init__.py              |   3 +-
 artiq/gateware/rtio/analyzer.py              |  51 ++------
 artiq/gateware/rtio/core.py                  |  77 ++++++------
 artiq/gateware/rtio/cri.py                   | 124 +++++++++++++++++++
 artiq/gateware/rtio/kernel_csrs.py           |  27 ----
 artiq/gateware/targets/kc705.py              |   7 +-
 artiq/gateware/targets/kc705_drtio_master.py |   4 +-
 artiq/gateware/targets/pipistrello.py        |   5 +-
 artiq/protocols/analyzer.py                  |   8 +-
 artiq/runtime.rs/libksupport/rtio.rs         |   2 -
 12 files changed, 203 insertions(+), 142 deletions(-)
 create mode 100644 artiq/gateware/rtio/cri.py
 delete mode 100644 artiq/gateware/rtio/kernel_csrs.py

diff --git a/artiq/gateware/drtio/core.py b/artiq/gateware/drtio/core.py
index 58baeb635..93e4b0e82 100644
--- a/artiq/gateware/drtio/core.py
+++ b/artiq/gateware/drtio/core.py
@@ -57,13 +57,11 @@ class DRTIOMaster(Module):
         self.submodules.rt_controller = rt_controller.RTController(
             self.rt_packets, channel_count, fine_ts_width)
         self.submodules.rt_manager = rt_controller.RTManager(self.rt_packets)
+        self.cri = self.rt_controller.cri
 
         self.submodules.aux_controller = aux_controller.AuxController(
             self.link_layer)
 
-    def get_kernel_csrs(self):
-        return self.rt_controller.get_kernel_csrs()
-
     def get_csrs(self):
         return (self.link_layer.get_csrs() +
                 self.rt_controller.get_csrs() +
diff --git a/artiq/gateware/drtio/rt_controller.py b/artiq/gateware/drtio/rt_controller.py
index a7c51c782..b169ee3c8 100644
--- a/artiq/gateware/drtio/rt_controller.py
+++ b/artiq/gateware/drtio/rt_controller.py
@@ -5,7 +5,7 @@ from migen.genlib.misc import WaitTimer
 from misoc.interconnect.csr import *
 
 from artiq.gateware.rtio.cdc import RTIOCounter
-from artiq.gateware.rtio.kernel_csrs import KernelCSRs
+from artiq.gateware.rtio import cri
 
 
 class _CSRs(AutoCSR):
@@ -27,20 +27,20 @@ class _CSRs(AutoCSR):
 
 class RTController(Module):
     def __init__(self, rt_packets, channel_count, fine_ts_width):
-        self.kcsrs = KernelCSRs()
         self.csrs = _CSRs()
+        self.cri = cri.Interface()
+        self.comb += self.cri.arb_gnt.eq(1)
 
         # channel selection
         chan_sel = Signal(16)
         self.comb += chan_sel.eq(
             Mux(self.csrs.chan_sel_override_en.storage,
                 self.csrs.chan_sel_override.storage,
-                self.kcsrs.chan_sel.storage))
+                self.cri.chan_sel[:16]))
 
         # master RTIO counter and counter synchronization
         self.submodules.counter = RTIOCounter(64-fine_ts_width)
-        self.sync += If(self.kcsrs.counter_update.re, 
-            self.kcsrs.counter.status.eq(self.counter.value_sys << fine_ts_width))
+        self.comb += self.cri.counter.eq(self.counter.value_sys << fine_ts_width)
         tsc_correction = Signal(64)
         self.csrs.tsc_correction.storage.attr.add("no_retiming")
         self.specials += MultiReg(self.csrs.tsc_correction.storage, tsc_correction)
@@ -67,14 +67,14 @@ class RTController(Module):
         self.comb += [
             fifo_spaces.adr.eq(chan_sel),
             last_timestamps.adr.eq(chan_sel),
-            last_timestamps.dat_w.eq(self.kcsrs.o_timestamp.storage),
+            last_timestamps.dat_w.eq(self.cri.o_timestamp),
             rt_packets.write_channel.eq(chan_sel),
-            rt_packets.write_address.eq(self.kcsrs.o_address.storage),
-            rt_packets.write_data.eq(self.kcsrs.o_data.storage),
+            rt_packets.write_address.eq(self.cri.o_address),
+            rt_packets.write_data.eq(self.cri.o_data),
             If(rt_packets_fifo_request,
                 rt_packets.write_timestamp.eq(0xffff000000000000)
             ).Else(
-                rt_packets.write_timestamp.eq(self.kcsrs.o_timestamp.storage)
+                rt_packets.write_timestamp.eq(self.cri.o_timestamp)
             )
         ]
 
@@ -85,15 +85,15 @@ class RTController(Module):
         status_underflow = Signal()
         status_sequence_error = Signal()
         self.comb += [
-            self.kcsrs.o_status.status.eq(Cat(
+            self.cri.o_status.eq(Cat(
                 status_wait, status_underflow, status_sequence_error)),
             self.csrs.o_wait.status.eq(status_wait)
         ]
         sequence_error_set = Signal()
         underflow_set = Signal()
         self.sync += [
-            If(self.kcsrs.o_underflow_reset.re, status_underflow.eq(0)),
-            If(self.kcsrs.o_sequence_error_reset.re, status_sequence_error.eq(0)),
+            If(self.cri.cmd == cri.commands["o_underflow_reset"], status_underflow.eq(0)),
+            If(self.cri.cmd == cri.commands["o_sequence_error_reset"], status_sequence_error.eq(0)),
             If(underflow_set, status_underflow.eq(1)),
             If(sequence_error_set, status_sequence_error.eq(1)),
         ]
@@ -107,14 +107,14 @@ class RTController(Module):
         self.submodules += timeout_counter
 
         # TODO: collision, replace, busy
-        cond_sequence_error = self.kcsrs.o_timestamp.storage < last_timestamps.dat_r
-        cond_underflow = ((self.kcsrs.o_timestamp.storage[fine_ts_width:]
+        cond_sequence_error = self.cri.o_timestamp < last_timestamps.dat_r
+        cond_underflow = ((self.cri.o_timestamp[fine_ts_width:]
                            - self.csrs.underflow_margin.storage[fine_ts_width:]) < self.counter.value_sys)
         cond_fifo_emptied = ((last_timestamps.dat_r[fine_ts_width:] < self.counter.value_sys)
                              & (last_timestamps.dat_r != 0))
 
         fsm.act("IDLE",
-            If(self.kcsrs.o_we.re,
+            If(self.cri.cmd == cri.commands["write"],
                 If(cond_sequence_error,
                     sequence_error_set.eq(1)
                 ).Elif(cond_underflow,
@@ -184,9 +184,6 @@ class RTController(Module):
             )
         ]
 
-    def get_kernel_csrs(self):
-        return self.kcsrs.get_csrs()
-
     def get_csrs(self):
         return self.csrs.get_csrs()
 
diff --git a/artiq/gateware/rtio/__init__.py b/artiq/gateware/rtio/__init__.py
index 88ef7fb57..57fcbbc38 100644
--- a/artiq/gateware/rtio/__init__.py
+++ b/artiq/gateware/rtio/__init__.py
@@ -1,3 +1,4 @@
-from artiq.gateware.rtio.core import Channel, LogChannel, RTIO
+from artiq.gateware.rtio.cri import KernelInitiator
+from artiq.gateware.rtio.core import Channel, LogChannel, Core
 from artiq.gateware.rtio.analyzer import Analyzer
 from artiq.gateware.rtio.moninj import MonInj
diff --git a/artiq/gateware/rtio/analyzer.py b/artiq/gateware/rtio/analyzer.py
index cb927b740..b702d538c 100644
--- a/artiq/gateware/rtio/analyzer.py
+++ b/artiq/gateware/rtio/analyzer.py
@@ -42,7 +42,7 @@ assert layout_len(stopped_layout) == message_len
 
 
 class MessageEncoder(Module, AutoCSR):
-    def __init__(self, rtio_core, enable):
+    def __init__(self, kcsrs, rtio_counter, enable):
         self.source = stream.Endpoint([("data", message_len)])
 
         self.overflow = CSRStatus()
@@ -50,27 +50,15 @@ class MessageEncoder(Module, AutoCSR):
 
         # # #
 
-        kcsrs = rtio_core.kcsrs
-
         input_output_stb = Signal()
         input_output = Record(input_output_layout)
-        if hasattr(kcsrs, "o_data"):
-            o_data = kcsrs.o_data.storage
-        else:
-            o_data = 0
-        if hasattr(kcsrs, "o_address"):
-            o_address = kcsrs.o_address.storage
-        else:
-            o_address = 0
-        if hasattr(kcsrs, "i_data"):
-            i_data = kcsrs.i_data.status
-        else:
-            i_data = 0
+        o_data = kcsrs.o_data.storage
+        o_address = kcsrs.o_address.storage
+        i_data = kcsrs.i_data.status
         self.comb += [
             input_output.channel.eq(kcsrs.chan_sel.storage),
             input_output.address_padding.eq(o_address),
-            input_output.rtio_counter.eq(
-                rtio_core.counter.value_sys << rtio_core.fine_ts_width),
+            input_output.rtio_counter.eq(rtio_counter),
             If(kcsrs.o_we.re,
                 input_output.message_type.eq(MessageType.output.value),
                 input_output.timestamp.eq(kcsrs.o_timestamp.storage),
@@ -88,10 +76,10 @@ class MessageEncoder(Module, AutoCSR):
         self.comb += [
             exception.message_type.eq(MessageType.exception.value),
             exception.channel.eq(kcsrs.chan_sel.storage),
-            exception.rtio_counter.eq(
-                rtio_core.counter.value_sys << rtio_core.fine_ts_width),
+            exception.rtio_counter.eq(rtio_counter),
         ]
-        for ename in ("o_underflow_reset", "o_sequence_error_reset",
+        for ename in ("reset", "reset_phy",
+                      "o_underflow_reset", "o_sequence_error_reset",
                       "o_collision_reset", "i_overflow_reset"):
             self.comb += \
                 If(getattr(kcsrs, ename).re,
@@ -99,28 +87,11 @@ class MessageEncoder(Module, AutoCSR):
                     exception.exception_type.eq(
                         getattr(ExceptionType, ename).value)
                 )
-        for rname in "reset", "reset_phy":
-            r_d = Signal(reset=1)
-            r = getattr(kcsrs, rname).storage
-            self.sync += r_d.eq(r)
-            self.comb += [
-                If(r & ~r_d,
-                    exception_stb.eq(1),
-                    exception.exception_type.eq(
-                        getattr(ExceptionType, rname+"_rising").value)
-                ),
-                If(~r & r_d,
-                    exception_stb.eq(1),
-                    exception.exception_type.eq(
-                        getattr(ExceptionType, rname+"_falling").value)
-                )
-            ]
 
         stopped = Record(stopped_layout)
         self.comb += [
             stopped.message_type.eq(MessageType.stopped.value),
-            stopped.rtio_counter.eq(
-                rtio_core.counter.value_sys << rtio_core.fine_ts_width),
+            stopped.rtio_counter.eq(rtio_counter),
         ]
 
         enable_r = Signal()
@@ -210,13 +181,13 @@ class DMAWriter(Module, AutoCSR):
 
 
 class Analyzer(Module, AutoCSR):
-    def __init__(self, rtio_core, membus, fifo_depth=128):
+    def __init__(self, kcsrs, rtio_counter, membus, fifo_depth=128):
         # shutdown procedure: set enable to 0, wait until busy=0
         self.enable = CSRStorage()
         self.busy = CSRStatus()
 
         self.submodules.message_encoder = MessageEncoder(
-            rtio_core, self.enable.storage)
+            kcsrs, rtio_counter, self.enable.storage)
         self.submodules.fifo = stream.SyncFIFO(
             [("data", message_len)], fifo_depth, True)
         self.submodules.converter = stream.Converter(
diff --git a/artiq/gateware/rtio/core.py b/artiq/gateware/rtio/core.py
index 43e9dea5d..3fd350206 100644
--- a/artiq/gateware/rtio/core.py
+++ b/artiq/gateware/rtio/core.py
@@ -6,8 +6,7 @@ from migen.genlib.record import Record
 from migen.genlib.fifo import AsyncFIFO
 from migen.genlib.resetsync import AsyncResetSynchronizer
 
-from artiq.gateware.rtio import rtlink
-from artiq.gateware.rtio.kernel_csrs import KernelCSRs
+from artiq.gateware.rtio import cri, rtlink
 from artiq.gateware.rtio.cdc import *
 
 
@@ -265,7 +264,7 @@ class LogChannel:
         self.overrides = []
 
 
-class RTIO(Module):
+class Core(Module):
     def __init__(self, channels, full_ts_width=63, guard_io_cycles=20):
         data_width = max(rtlink.get_data_width(c.interface)
                          for c in channels)
@@ -278,35 +277,43 @@ class RTIO(Module):
         self.address_width = address_width
         self.fine_ts_width = fine_ts_width
 
-        self.kcsrs = KernelCSRs()
+        self.cri = cri.Interface()
+        self.comb += self.cri.arb_gnt.eq(1)
 
         # Clocking/Reset
         # Create rsys, rio and rio_phy domains based on sys and rtio
-        # with reset controlled by CSR.
+        # with reset controlled by CRI.
+        cmd_reset = Signal(reset=1)
+        cmd_reset_phy = Signal(reset=1)
+        self.sync += [
+            cmd_reset.eq(self.cri.cmd == cri.commands["reset"]),
+            cmd_reset_phy.eq(self.cri.cmd == cri.commands["reset_phy"])
+        ]
+        cmd_reset.attr.add("no_retiming")
+        cmd_reset_phy.attr.add("no_retiming")
+
         self.clock_domains.cd_rsys = ClockDomain()
         self.clock_domains.cd_rio = ClockDomain()
         self.clock_domains.cd_rio_phy = ClockDomain()
         self.comb += [
             self.cd_rsys.clk.eq(ClockSignal()),
-            self.cd_rsys.rst.eq(self.kcsrs.reset.storage)
+            self.cd_rsys.rst.eq(cmd_reset)
         ]
         self.comb += self.cd_rio.clk.eq(ClockSignal("rtio"))
         self.specials += AsyncResetSynchronizer(
             self.cd_rio,
-            self.kcsrs.reset.storage | ResetSignal("rtio",
-                                                   allow_reset_less=True))
+            cmd_reset | ResetSignal("rtio", allow_reset_less=True))
         self.comb += self.cd_rio_phy.clk.eq(ClockSignal("rtio"))
         self.specials += AsyncResetSynchronizer(
             self.cd_rio_phy,
-            self.kcsrs.reset_phy.storage | ResetSignal("rtio",
-                                                       allow_reset_less=True))
+            cmd_reset_phy | ResetSignal("rtio", allow_reset_less=True))
 
         # Managers
         self.submodules.counter = RTIOCounter(full_ts_width - fine_ts_width)
 
         i_datas, i_timestamps = [], []
         o_statuses, i_statuses = [], []
-        sel = self.kcsrs.chan_sel.storage
+        sel = self.cri.chan_sel[:16]
         for n, channel in enumerate(channels):
             if isinstance(channel, LogChannel):
                 i_datas.append(0)
@@ -322,30 +329,27 @@ class RTIO(Module):
             self.submodules += o_manager
 
             if hasattr(o_manager.ev, "data"):
-                self.comb += o_manager.ev.data.eq(
-                    self.kcsrs.o_data.storage)
+                self.comb += o_manager.ev.data.eq(self.cri.o_data)
             if hasattr(o_manager.ev, "address"):
-                self.comb += o_manager.ev.address.eq(
-                    self.kcsrs.o_address.storage)
-            ts_shift = (len(self.kcsrs.o_timestamp.storage)
-                        - len(o_manager.ev.timestamp))
-            self.comb += o_manager.ev.timestamp.eq(
-                self.kcsrs.o_timestamp.storage[ts_shift:])
+                self.comb += o_manager.ev.address.eq(self.cri.o_address)
+            ts_shift = len(self.cri.o_timestamp) - len(o_manager.ev.timestamp)
+            self.comb += o_manager.ev.timestamp.eq(self.cri.o_timestamp[ts_shift:])
 
-            self.comb += o_manager.we.eq(selected & self.kcsrs.o_we.re)
+            self.comb += o_manager.we.eq(selected &
+                (self.cri.cmd == cri.commands["write"]))
 
             underflow = Signal()
             sequence_error = Signal()
             collision = Signal()
             busy = Signal()
             self.sync.rsys += [
-                If(selected & self.kcsrs.o_underflow_reset.re,
+                If(selected & (self.cri.cmd == cri.commands["o_underflow_reset"]),
                    underflow.eq(0)),
-                If(selected & self.kcsrs.o_sequence_error_reset.re,
+                If(selected & (self.cri.cmd == cri.commands["o_sequence_error_reset"]),
                    sequence_error.eq(0)),
-                If(selected & self.kcsrs.o_collision_reset.re,
+                If(selected & (self.cri.cmd == cri.commands["o_collision_reset"]),
                    collision.eq(0)),
-                If(selected & self.kcsrs.o_busy_reset.re,
+                If(selected & (self.cri.cmd == cri.commands["o_busy_reset"]),
                    busy.eq(0)),
                 If(o_manager.underflow, underflow.eq(1)),
                 If(o_manager.sequence_error, sequence_error.eq(1)),
@@ -368,17 +372,17 @@ class RTIO(Module):
                 else:
                     i_datas.append(0)
                 if channel.interface.i.timestamped:
-                    ts_shift = (len(self.kcsrs.i_timestamp.status)
+                    ts_shift = (len(self.cri.i_timestamp)
                                 - len(i_manager.ev.timestamp))
                     i_timestamps.append(i_manager.ev.timestamp << ts_shift)
                 else:
                     i_timestamps.append(0)
 
-                self.comb += i_manager.re.eq(selected & self.kcsrs.i_re.re)
+                self.comb += i_manager.re.eq(selected & (self.cri.cmd == cri.commands["read"]))
 
                 overflow = Signal()
                 self.sync.rsys += [
-                    If(selected & self.kcsrs.i_overflow_reset.re,
+                    If(selected & (self.cri.cmd == cri.commands["i_overflow_reset"]),
                        overflow.eq(0)),
                     If(i_manager.overflow,
                        overflow.eq(1))
@@ -389,20 +393,11 @@ class RTIO(Module):
                 i_datas.append(0)
                 i_timestamps.append(0)
                 i_statuses.append(0)
-        if data_width:
-            self.comb += self.kcsrs.i_data.status.eq(Array(i_datas)[sel])
         self.comb += [
-            self.kcsrs.i_timestamp.status.eq(Array(i_timestamps)[sel]),
-            self.kcsrs.o_status.status.eq(Array(o_statuses)[sel]),
-            self.kcsrs.i_status.status.eq(Array(i_statuses)[sel])
+            self.cri.i_data.eq(Array(i_datas)[sel]),
+            self.cri.i_timestamp.eq(Array(i_timestamps)[sel]),
+            self.cri.o_status.eq(Array(o_statuses)[sel]),
+            self.cri.i_status.eq(Array(i_statuses)[sel])
         ]
 
-        # Counter access
-        self.sync += \
-           If(self.kcsrs.counter_update.re,
-               self.kcsrs.counter.status.eq(self.counter.value_sys
-                                                << fine_ts_width)
-           )
-
-    def get_csrs(self):
-        return self.kcsrs.get_csrs()
+        self.cri.counter.eq(self.counter.value_sys << fine_ts_width)
diff --git a/artiq/gateware/rtio/cri.py b/artiq/gateware/rtio/cri.py
new file mode 100644
index 000000000..b838b34d4
--- /dev/null
+++ b/artiq/gateware/rtio/cri.py
@@ -0,0 +1,124 @@
+"""Common RTIO Interface"""
+
+from migen import *
+from migen.genlib.record import *
+
+from misoc.interconnect.csr import *
+
+
+commands = {
+    "nop": 0,
+    "reset": 1,
+    "reset_phy": 2,
+
+    "write": 3,
+    "read": 4,
+
+    "o_underflow_reset": 5,
+    "o_sequence_error_reset": 6,
+    "o_collision_reset": 7,
+    "o_busy_reset": 8,
+    "i_overflow_reset": 9
+}
+
+
+layout = [
+    ("arb_req", 1, DIR_M_TO_S),
+    ("arb_gnt", 1, DIR_S_TO_M),
+
+    ("cmd", 4, DIR_M_TO_S),
+    # 8 MSBs of chan_sel are used to select core
+    ("chan_sel", 24, DIR_M_TO_S),
+
+    ("o_data", 512, DIR_M_TO_S),
+    ("o_address", 16, DIR_M_TO_S),
+    ("o_timestamp", 64, DIR_M_TO_S),
+    # o_status bits:
+    # <0:wait> <1:underflow> <2:sequence_error> <3:collision> <4:busy>
+    ("o_status", 5, DIR_S_TO_M),
+
+    ("i_data", 32, DIR_S_TO_M),
+    ("i_timestamp", 64, DIR_S_TO_M),
+    # i_status bits:
+    # <0:wait> <1:overflow>
+    ("i_status", 2, DIR_S_TO_M),
+
+    ("counter", 64, DIR_S_TO_M)
+]
+
+
+class Interface(Record):
+    def __init__(self):
+        Record.__init__(self, layout)
+
+
+class KernelInitiator(Module, AutoCSR):
+    def __init__(self, cri=None):
+        self.arb_req = CSRStorage()
+        self.arb_gnt = CSRStatus()
+
+        self.reset = CSR()
+        self.reset_phy = CSR()
+        self.chan_sel = CSRStorage(24)
+
+        self.o_data = CSRStorage(32, write_from_dev=True)  # XXX -> 512
+        self.o_address = CSRStorage(16)
+        self.o_timestamp = CSRStorage(64)
+        self.o_we = CSR()
+        self.o_status = CSRStatus(5)
+        self.o_underflow_reset = CSR()
+        self.o_sequence_error_reset = CSR()
+        self.o_collision_reset = CSR()
+        self.o_busy_reset = CSR()
+
+        self.i_data = CSRStatus(32)
+        self.i_timestamp = CSRStatus(64)
+        self.i_re = CSR()
+        self.i_status = CSRStatus(2)
+        self.i_overflow_reset = CSR()
+
+        self.counter = CSRStatus(64)
+        self.counter_update = CSR()
+
+        if cri is None:
+            cri = Interface()
+        self.cri = cri
+
+        # # #
+
+        self.comb += [
+            self.cri.arb_req.eq(self.arb_req.storage),
+            self.arb_gnt.status.eq(self.cri.arb_gnt),
+
+            self.cri.cmd.eq(commands["nop"]),
+            If(self.reset.re, self.cri.cmd.eq(commands["reset"])),
+            If(self.reset_phy.re, self.cri.cmd.eq(commands["reset_phy"])),
+            If(self.o_we.re, self.cri.cmd.eq(commands["write"])),
+            If(self.i_re.re, self.cri.cmd.eq(commands["read"])),
+            If(self.o_underflow_reset.re, self.cri.cmd.eq(commands["o_underflow_reset"])),
+            If(self.o_sequence_error_reset.re, self.cri.cmd.eq(commands["o_sequence_error_reset"])),
+            If(self.o_collision_reset.re, self.cri.cmd.eq(commands["o_collision_reset"])),
+            If(self.o_busy_reset.re, self.cri.cmd.eq(commands["o_busy_reset"])),
+            If(self.i_overflow_reset.re, self.cri.cmd.eq(commands["i_overflow_reset"])),
+
+            self.cri.chan_sel.eq(self.chan_sel.storage),
+
+            self.cri.o_data.eq(self.o_data.storage),
+            self.cri.o_address.eq(self.o_address.storage),
+            self.cri.o_timestamp.eq(self.o_timestamp.storage),
+            self.o_status.status.eq(self.cri.o_status),
+
+            self.i_data.status.eq(self.cri.i_data),
+            self.i_timestamp.status.eq(self.cri.i_timestamp),
+            self.i_status.status.eq(self.cri.i_status),
+
+            self.o_data.dat_w.eq(0),
+            self.o_data.we.eq(self.o_timestamp.re),
+        ]
+        self.sync += If(self.counter_update.re, self.counter.status.eq(self.cri.counter))
+
+    def get_csrs(self):
+        return []
+
+    def get_kernel_csrs(self):
+        return AutoCSR.get_csrs(self)
diff --git a/artiq/gateware/rtio/kernel_csrs.py b/artiq/gateware/rtio/kernel_csrs.py
deleted file mode 100644
index d897a7566..000000000
--- a/artiq/gateware/rtio/kernel_csrs.py
+++ /dev/null
@@ -1,27 +0,0 @@
-from misoc.interconnect.csr import *
-
-
-class KernelCSRs(AutoCSR):
-    def __init__(self):
-        self.reset = CSRStorage(reset=1)
-        self.reset_phy = CSRStorage(reset=1)
-        self.chan_sel = CSRStorage(16)
-
-        self.o_data = CSRStorage(32)
-        self.o_address = CSRStorage(16)
-        self.o_timestamp = CSRStorage(64)
-        self.o_we = CSR()
-        self.o_status = CSRStatus(5)
-        self.o_underflow_reset = CSR()
-        self.o_sequence_error_reset = CSR()
-        self.o_collision_reset = CSR()
-        self.o_busy_reset = CSR()
-
-        self.i_data = CSRStatus(32)
-        self.i_timestamp = CSRStatus(64)
-        self.i_re = CSR()
-        self.i_status = CSRStatus(2)
-        self.i_overflow_reset = CSR()
-
-        self.counter = CSRStatus(64)
-        self.counter_update = CSR()
diff --git a/artiq/gateware/targets/kc705.py b/artiq/gateware/targets/kc705.py
index 502ca5902..0596efcd7 100755
--- a/artiq/gateware/targets/kc705.py
+++ b/artiq/gateware/targets/kc705.py
@@ -142,7 +142,8 @@ class _NIST_Ions(MiniSoC, AMPSoC):
     def add_rtio(self, rtio_channels):
         self.submodules.rtio_crg = _RTIOCRG(self.platform, self.crg.cd_sys.clk)
         self.csr_devices.append("rtio_crg")
-        self.submodules.rtio = rtio.RTIO(rtio_channels)
+        self.submodules.rtio_core = rtio.Core(rtio_channels)
+        self.submodules.rtio = rtio.KernelInitiator(self.rtio_core.cri)
         self.register_kernel_cpu_csrdevice("rtio")
         self.submodules.rtio_moninj = rtio.MonInj(rtio_channels)
         self.csr_devices.append("rtio_moninj")
@@ -153,8 +154,8 @@ class _NIST_Ions(MiniSoC, AMPSoC):
             self.crg.cd_sys.clk,
             self.rtio_crg.cd_rtio.clk)
 
-        self.submodules.rtio_analyzer = rtio.Analyzer(self.rtio,
-            self.get_native_sdram_if())
+        self.submodules.rtio_analyzer = rtio.Analyzer(
+            self.rtio, self.rtio_core.cri.counter, self.get_native_sdram_if())
         self.csr_devices.append("rtio_analyzer")
 
 
diff --git a/artiq/gateware/targets/kc705_drtio_master.py b/artiq/gateware/targets/kc705_drtio_master.py
index fe48909de..fb6bfadc8 100755
--- a/artiq/gateware/targets/kc705_drtio_master.py
+++ b/artiq/gateware/targets/kc705_drtio_master.py
@@ -8,6 +8,7 @@ from misoc.targets.kc705 import MiniSoC, soc_kc705_args, soc_kc705_argdict
 from misoc.integration.builder import builder_args, builder_argdict
 
 from artiq.gateware.soc import AMPSoC, build_artiq_soc
+from artiq.gateware import rtio
 from artiq.gateware.drtio.transceiver import gtx_7series
 from artiq.gateware.drtio import DRTIOMaster
 from artiq import __version__ as artiq_version
@@ -41,7 +42,8 @@ class Master(MiniSoC, AMPSoC):
             sys_clk_freq=self.clk_freq,
             clock_div2=True)
         self.submodules.drtio = DRTIOMaster(self.transceiver)
-        self.register_kernel_cpu_csrdevice("rtio", self.drtio.get_kernel_csrs())
+        self.submodules.rtio = rtio.KernelInitiator(self.drtio.cri)
+        self.register_kernel_cpu_csrdevice("rtio")
         self.csr_devices.append("drtio")
 
 
diff --git a/artiq/gateware/targets/pipistrello.py b/artiq/gateware/targets/pipistrello.py
index 42dc87dac..b124b8f63 100755
--- a/artiq/gateware/targets/pipistrello.py
+++ b/artiq/gateware/targets/pipistrello.py
@@ -216,12 +216,13 @@ trce -v 12 -fastpaths -tsi {build_name}.tsi -o {build_name}.twr {build_name}.ncd
         rtio_channels.append(rtio.LogChannel())
 
         # RTIO logic
-        self.submodules.rtio = rtio.RTIO(rtio_channels)
+        self.submodules.rtio_core = rtio.Core(rtio_channels)
+        self.submodules.rtio = rtio.KernelInitiator(self.rtio_core.cri)
         self.register_kernel_cpu_csrdevice("rtio")
         self.submodules.rtio_moninj = rtio.MonInj(rtio_channels)
         self.csr_devices.append("rtio_moninj")
         self.submodules.rtio_analyzer = rtio.Analyzer(
-            self.rtio, self.get_native_sdram_if())
+            self.rtio, self.rtio_core.cri.counter, self.get_native_sdram_if())
         self.csr_devices.append("rtio_analyzer")
 
 
diff --git a/artiq/protocols/analyzer.py b/artiq/protocols/analyzer.py
index e3ec7da7b..c6f8c78cc 100644
--- a/artiq/protocols/analyzer.py
+++ b/artiq/protocols/analyzer.py
@@ -9,10 +9,10 @@ class MessageType(Enum):
 
 
 class ExceptionType(Enum):
-    reset_rising = 0b000000
-    reset_falling = 0b000001
-    reset_phy_rising = 0b000010
-    reset_phy_falling = 0b000011
+    reset = 0b000000
+    legacy_reset_falling = 0b000001
+    reset_phy = 0b000010
+    legacy_reset_phy_falling = 0b000011
 
     o_underflow_reset = 0b010000
     o_sequence_error_reset = 0b010001
diff --git a/artiq/runtime.rs/libksupport/rtio.rs b/artiq/runtime.rs/libksupport/rtio.rs
index 107583da4..3f3a2f6ce 100644
--- a/artiq/runtime.rs/libksupport/rtio.rs
+++ b/artiq/runtime.rs/libksupport/rtio.rs
@@ -11,8 +11,6 @@ const RTIO_I_STATUS_OVERFLOW:       u32 = 2;
 pub extern fn init() {
     unsafe {
         csr::rtio::reset_write(1);
-        csr::rtio::reset_write(0);
-        csr::rtio::reset_phy_write(0);
     }
 }
 

From aa00627c0e36542baf7d11d5a799806d09784df3 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Tue, 22 Nov 2016 22:57:04 +0800
Subject: [PATCH 097/134] rtio: fix CRI CSRs

---
 artiq/gateware/rtio/cri.py | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/artiq/gateware/rtio/cri.py b/artiq/gateware/rtio/cri.py
index b838b34d4..1e9dd40d2 100644
--- a/artiq/gateware/rtio/cri.py
+++ b/artiq/gateware/rtio/cri.py
@@ -116,9 +116,3 @@ class KernelInitiator(Module, AutoCSR):
             self.o_data.we.eq(self.o_timestamp.re),
         ]
         self.sync += If(self.counter_update.re, self.counter.status.eq(self.cri.counter))
-
-    def get_csrs(self):
-        return []
-
-    def get_kernel_csrs(self):
-        return AutoCSR.get_csrs(self)

From ffefdb92690aab25fce3add658499526e98e9585 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Wed, 23 Nov 2016 00:54:47 +0800
Subject: [PATCH 098/134] rtio: fix counter readback

---
 artiq/gateware/rtio/core.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/artiq/gateware/rtio/core.py b/artiq/gateware/rtio/core.py
index 3fd350206..4c8e16929 100644
--- a/artiq/gateware/rtio/core.py
+++ b/artiq/gateware/rtio/core.py
@@ -400,4 +400,4 @@ class Core(Module):
             self.cri.i_status.eq(Array(i_statuses)[sel])
         ]
 
-        self.cri.counter.eq(self.counter.value_sys << fine_ts_width)
+        self.comb += self.cri.counter.eq(self.counter.value_sys << fine_ts_width)

From 3c5a62243dfef00a54292098eba5ae47357acd08 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Wed, 23 Nov 2016 10:43:33 +0800
Subject: [PATCH 099/134] runtime: support for targets without I2C

---
 artiq/runtime.rs/libksupport/api.rs | 5 +++++
 artiq/runtime.rs/libksupport/lib.rs | 1 +
 2 files changed, 6 insertions(+)

diff --git a/artiq/runtime.rs/libksupport/api.rs b/artiq/runtime.rs/libksupport/api.rs
index 5a12ea691..dbfbd759b 100644
--- a/artiq/runtime.rs/libksupport/api.rs
+++ b/artiq/runtime.rs/libksupport/api.rs
@@ -105,9 +105,14 @@ static mut API: &'static [(&'static str, *const ())] = &[
     api!(rtio_input_timestamp = ::rtio::input_timestamp),
     api!(rtio_input_data = ::rtio::input_data),
 
+    #[cfg(has_i2c)]
     api!(i2c_init = ::i2c::init),
+    #[cfg(has_i2c)]
     api!(i2c_start = ::i2c::start),
+    #[cfg(has_i2c)]
     api!(i2c_stop = ::i2c::stop),
+    #[cfg(has_i2c)]
     api!(i2c_write = ::i2c::write),
+    #[cfg(has_i2c)]
     api!(i2c_read = ::i2c::read),
 ];
diff --git a/artiq/runtime.rs/libksupport/lib.rs b/artiq/runtime.rs/libksupport/lib.rs
index 6b285f889..027c0e3b3 100644
--- a/artiq/runtime.rs/libksupport/lib.rs
+++ b/artiq/runtime.rs/libksupport/lib.rs
@@ -50,6 +50,7 @@ macro_rules! artiq_raise {
 }
 
 mod rtio;
+#[cfg(has_i2c)]
 mod i2c;
 
 use core::{mem, ptr, slice, str};

From 0c4967998410b39ff06f96681661f82661f93df0 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Wed, 23 Nov 2016 10:48:26 +0800
Subject: [PATCH 100/134] runtime: support for targets without RTIO log channel

---
 artiq/gateware/targets/kc705.py       | 2 ++
 artiq/gateware/targets/pipistrello.py | 1 +
 artiq/runtime.rs/libksupport/rtio.rs  | 4 ++++
 3 files changed, 7 insertions(+)

diff --git a/artiq/gateware/targets/kc705.py b/artiq/gateware/targets/kc705.py
index 0596efcd7..7a9bde597 100755
--- a/artiq/gateware/targets/kc705.py
+++ b/artiq/gateware/targets/kc705.py
@@ -226,6 +226,7 @@ class NIST_CLOCK(_NIST_Ions):
                                                    ofifo_depth=512,
                                                    ififo_depth=4))
 
+        self.config["HAS_RTIO_LOG"] = None
         self.config["RTIO_LOG_CHANNEL"] = len(rtio_channels)
         rtio_channels.append(rtio.LogChannel())
 
@@ -304,6 +305,7 @@ class NIST_QC2(_NIST_Ions):
                                                        ofifo_depth=512,
                                                        ififo_depth=4))
 
+        self.config["HAS_RTIO_LOG"] = None
         self.config["RTIO_LOG_CHANNEL"] = len(rtio_channels)
         rtio_channels.append(rtio.LogChannel())
 
diff --git a/artiq/gateware/targets/pipistrello.py b/artiq/gateware/targets/pipistrello.py
index b124b8f63..3a4d5beed 100755
--- a/artiq/gateware/targets/pipistrello.py
+++ b/artiq/gateware/targets/pipistrello.py
@@ -212,6 +212,7 @@ trce -v 12 -fastpaths -tsi {build_name}.tsi -o {build_name}.twr {build_name}.ncd
         rtio_channels.append(rtio.Channel.from_phy(
             phy, ofifo_depth=64, ififo_depth=64))
 
+        self.config["HAS_RTIO_LOG"] = None
         self.config["RTIO_LOG_CHANNEL"] = len(rtio_channels)
         rtio_channels.append(rtio.LogChannel())
 
diff --git a/artiq/runtime.rs/libksupport/rtio.rs b/artiq/runtime.rs/libksupport/rtio.rs
index 3f3a2f6ce..b98b9eaa6 100644
--- a/artiq/runtime.rs/libksupport/rtio.rs
+++ b/artiq/runtime.rs/libksupport/rtio.rs
@@ -123,6 +123,7 @@ pub extern fn input_data(channel: u32) -> u32 {
     }
 }
 
+#[cfg(has_rtio_log)]
 pub fn log(timestamp: i64, data: &[u8]) {
     unsafe {
         csr::rtio::chan_sel_write(csr::CONFIG_RTIO_LOG_CHANNEL);
@@ -144,3 +145,6 @@ pub fn log(timestamp: i64, data: &[u8]) {
         csr::rtio::o_we_write(1);
     }
 }
+
+#[cfg(not(has_rtio_log))]
+pub fn log(timestamp: i64, data: &[u8]) {}

From e532261a9bb2e404807cdd1fd68e9640265921a9 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Wed, 23 Nov 2016 12:25:43 +0800
Subject: [PATCH 101/134] drtio: fix FullMemoryWE usage

---
 artiq/gateware/drtio/aux_controller.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/artiq/gateware/drtio/aux_controller.py b/artiq/gateware/drtio/aux_controller.py
index 8755fedf2..e6b0879c9 100644
--- a/artiq/gateware/drtio/aux_controller.py
+++ b/artiq/gateware/drtio/aux_controller.py
@@ -205,14 +205,15 @@ class Receiver(Module, AutoCSR):
         )
 
 
+# TODO: FullMemoryWE should be applied by migen.build
+@FullMemoryWE()
 class AuxController(Module):
     def __init__(self, link_layer):
         self.bus = wishbone.Interface()
         self.submodules.transmitter = Transmitter(link_layer, len(self.bus.dat_w))
         self.submodules.receiver = Receiver(link_layer, len(self.bus.dat_w))
 
-        # TODO: FullMemoryWE should be applied by migen.build
-        tx_sdram_if = FullMemoryWE()(wishbone.SRAM(self.transmitter.mem, read_only=False))
+        tx_sdram_if = wishbone.SRAM(self.transmitter.mem, read_only=False)
         rx_sdram_if = wishbone.SRAM(self.receiver.mem, read_only=True)
         wsb = log2_int(len(self.bus.dat_w)//8)
         decoder = wishbone.Decoder(self.bus,

From 5a2edef4223abecf8911eccd4a1010fd9cbe5211 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Wed, 23 Nov 2016 12:43:17 +0800
Subject: [PATCH 102/134] drtio: adapt example to new mu/second API

---
 artiq/examples/drtio/repository/pulse_rate.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/artiq/examples/drtio/repository/pulse_rate.py b/artiq/examples/drtio/repository/pulse_rate.py
index 091a30786..0d1a13468 100644
--- a/artiq/examples/drtio/repository/pulse_rate.py
+++ b/artiq/examples/drtio/repository/pulse_rate.py
@@ -11,7 +11,7 @@ class PulseRate(EnvExperiment):
         #self.core.reset()
         self.core.break_realtime()
 
-        dt = seconds_to_mu(300*ns)
+        dt = self.core.seconds_to_mu(300*ns)
         while True:
             for i in range(10000):
                 try:
@@ -22,5 +22,5 @@ class PulseRate(EnvExperiment):
                     self.core.break_realtime()
                     break
             else:
-                print(mu_to_seconds(dt))
+                print(self.core.mu_to_seconds(dt))
                 return

From 4e931c7dd2ab3831dffd9583a27f5129a7031bca Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Wed, 23 Nov 2016 13:36:30 +0800
Subject: [PATCH 103/134] rtio: fix timestamp shift

---
 artiq/gateware/rtio/core.py | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/artiq/gateware/rtio/core.py b/artiq/gateware/rtio/core.py
index 4c8e16929..da7e2e02b 100644
--- a/artiq/gateware/rtio/core.py
+++ b/artiq/gateware/rtio/core.py
@@ -265,7 +265,7 @@ class LogChannel:
 
 
 class Core(Module):
-    def __init__(self, channels, full_ts_width=63, guard_io_cycles=20):
+    def __init__(self, channels, guard_io_cycles=20):
         data_width = max(rtlink.get_data_width(c.interface)
                          for c in channels)
         address_width = max(rtlink.get_address_width(c.interface)
@@ -309,7 +309,7 @@ class Core(Module):
             cmd_reset_phy | ResetSignal("rtio", allow_reset_less=True))
 
         # Managers
-        self.submodules.counter = RTIOCounter(full_ts_width - fine_ts_width)
+        self.submodules.counter = RTIOCounter(len(self.cri.o_timestamp) - fine_ts_width)
 
         i_datas, i_timestamps = [], []
         o_statuses, i_statuses = [], []
@@ -333,10 +333,10 @@ class Core(Module):
             if hasattr(o_manager.ev, "address"):
                 self.comb += o_manager.ev.address.eq(self.cri.o_address)
             ts_shift = len(self.cri.o_timestamp) - len(o_manager.ev.timestamp)
+            print(n, ts_shift, channel)
             self.comb += o_manager.ev.timestamp.eq(self.cri.o_timestamp[ts_shift:])
 
-            self.comb += o_manager.we.eq(selected &
-                (self.cri.cmd == cri.commands["write"]))
+            self.comb += o_manager.we.eq(selected & (self.cri.cmd == cri.commands["write"]))
 
             underflow = Signal()
             sequence_error = Signal()
@@ -372,8 +372,7 @@ class Core(Module):
                 else:
                     i_datas.append(0)
                 if channel.interface.i.timestamped:
-                    ts_shift = (len(self.cri.i_timestamp)
-                                - len(i_manager.ev.timestamp))
+                    ts_shift = (len(self.cri.i_timestamp) - len(i_manager.ev.timestamp))
                     i_timestamps.append(i_manager.ev.timestamp << ts_shift)
                 else:
                     i_timestamps.append(0)

From d400c81cb23fc12abbf394fe35596dfbd8e6397d Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Wed, 23 Nov 2016 13:37:14 +0800
Subject: [PATCH 104/134] rtio: remove debug print

---
 artiq/gateware/rtio/core.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/artiq/gateware/rtio/core.py b/artiq/gateware/rtio/core.py
index da7e2e02b..4f1c36e91 100644
--- a/artiq/gateware/rtio/core.py
+++ b/artiq/gateware/rtio/core.py
@@ -333,7 +333,6 @@ class Core(Module):
             if hasattr(o_manager.ev, "address"):
                 self.comb += o_manager.ev.address.eq(self.cri.o_address)
             ts_shift = len(self.cri.o_timestamp) - len(o_manager.ev.timestamp)
-            print(n, ts_shift, channel)
             self.comb += o_manager.ev.timestamp.eq(self.cri.o_timestamp[ts_shift:])
 
             self.comb += o_manager.we.eq(selected & (self.cri.cmd == cri.commands["write"]))

From 9941f3557d13c50f916dbd09005b8d22fbe1da79 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Wed, 23 Nov 2016 23:19:14 +0800
Subject: [PATCH 105/134] rtio: use only CRI commands for rio/rio_phy resets

---
 artiq/gateware/rtio/core.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/artiq/gateware/rtio/core.py b/artiq/gateware/rtio/core.py
index 4f1c36e91..23fff3039 100644
--- a/artiq/gateware/rtio/core.py
+++ b/artiq/gateware/rtio/core.py
@@ -301,12 +301,10 @@ class Core(Module):
         ]
         self.comb += self.cd_rio.clk.eq(ClockSignal("rtio"))
         self.specials += AsyncResetSynchronizer(
-            self.cd_rio,
-            cmd_reset | ResetSignal("rtio", allow_reset_less=True))
+            self.cd_rio, cmd_reset)
         self.comb += self.cd_rio_phy.clk.eq(ClockSignal("rtio"))
         self.specials += AsyncResetSynchronizer(
-            self.cd_rio_phy,
-            cmd_reset_phy | ResetSignal("rtio", allow_reset_less=True))
+            self.cd_rio_phy, cmd_reset_phy)
 
         # Managers
         self.submodules.counter = RTIOCounter(len(self.cri.o_timestamp) - fine_ts_width)

From 07f2d842756f7d4f65ac97474ec667c1afb3e2be Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Wed, 23 Nov 2016 23:19:31 +0800
Subject: [PATCH 106/134] drtio: remote resets

---
 .../drtio/repository/blink_forever.py         |  3 +-
 artiq/examples/drtio/repository/pulse_rate.py |  3 +-
 artiq/gateware/drtio/core.py                  |  7 ++-
 artiq/gateware/drtio/rt_controller.py         | 13 ++++++
 artiq/gateware/drtio/rt_packets.py            | 44 ++++++++++++++++++-
 5 files changed, 61 insertions(+), 9 deletions(-)

diff --git a/artiq/examples/drtio/repository/blink_forever.py b/artiq/examples/drtio/repository/blink_forever.py
index 589d389cb..fb074661c 100644
--- a/artiq/examples/drtio/repository/blink_forever.py
+++ b/artiq/examples/drtio/repository/blink_forever.py
@@ -8,8 +8,7 @@ class BlinkForever(EnvExperiment):
 
     @kernel
     def run(self):
-        #self.core.reset()
-        self.core.break_realtime()
+        self.core.reset()
 
         while True:
             for led in self.leds:
diff --git a/artiq/examples/drtio/repository/pulse_rate.py b/artiq/examples/drtio/repository/pulse_rate.py
index 0d1a13468..055d7450b 100644
--- a/artiq/examples/drtio/repository/pulse_rate.py
+++ b/artiq/examples/drtio/repository/pulse_rate.py
@@ -8,8 +8,7 @@ class PulseRate(EnvExperiment):
 
     @kernel
     def run(self):
-        #self.core.reset()
-        self.core.break_realtime()
+        self.core.reset()
 
         dt = self.core.seconds_to_mu(300*ns)
         while True:
diff --git a/artiq/gateware/drtio/core.py b/artiq/gateware/drtio/core.py
index 93e4b0e82..995b0b137 100644
--- a/artiq/gateware/drtio/core.py
+++ b/artiq/gateware/drtio/core.py
@@ -27,17 +27,16 @@ class DRTIOSatellite(Module):
         self.submodules.rt_packets = ClockDomainsRenamer("rtio")(
             rt_packets.RTPacketSatellite(link_layer_sync))
 
-        self.submodules.iot = ClockDomainsRenamer("rtio")(
+        self.submodules.iot = ClockDomainsRenamer("rio")(
             iot.IOT(self.rt_packets, channels, fine_ts_width, full_ts_width))
 
-        # TODO: remote resets
         self.clock_domains.cd_rio = ClockDomain()
         self.clock_domains.cd_rio_phy = ClockDomain()
         self.comb += [
             self.cd_rio.clk.eq(ClockSignal("rtio")),
-            self.cd_rio.rst.eq(ResetSignal("rtio", allow_reset_less=True)),
+            self.cd_rio.rst.eq(self.rt_packets.reset),
             self.cd_rio_phy.clk.eq(ClockSignal("rtio")),
-            self.cd_rio_phy.rst.eq(ResetSignal("rtio", allow_reset_less=True)),
+            self.cd_rio_phy.rst.eq(self.rt_packets.reset_phy),
         ]
 
         self.submodules.aux_controller = aux_controller.AuxController(
diff --git a/artiq/gateware/drtio/rt_controller.py b/artiq/gateware/drtio/rt_controller.py
index b169ee3c8..3891c30dd 100644
--- a/artiq/gateware/drtio/rt_controller.py
+++ b/artiq/gateware/drtio/rt_controller.py
@@ -54,6 +54,19 @@ class RTController(Module):
             If(self.csrs.set_time.re, rt_packets.set_time_stb.eq(1))
         ]
 
+        # reset
+        self.sync += [
+            If(rt_packets.reset_ack, rt_packets.reset_stb.eq(0)),
+            If(self.cri.cmd == cri.commands["reset"],
+                rt_packets.reset_stb.eq(1),
+                rt_packets.reset_phy.eq(0)
+            ),
+            If(self.cri.cmd == cri.commands["reset_phy"],
+                rt_packets.reset_stb.eq(1),
+                rt_packets.reset_phy.eq(1)
+            ),
+        ]
+
         # remote channel status cache
         fifo_spaces_mem = Memory(16, channel_count)
         fifo_spaces = fifo_spaces_mem.get_port(write_capable=True)
diff --git a/artiq/gateware/drtio/rt_packets.py b/artiq/gateware/drtio/rt_packets.py
index 648c9919f..1e33e3bd4 100644
--- a/artiq/gateware/drtio/rt_packets.py
+++ b/artiq/gateware/drtio/rt_packets.py
@@ -34,6 +34,7 @@ def get_m2s_layouts(alignment):
     plm = PacketLayoutManager(alignment)
     plm.add_type("echo_request")
     plm.add_type("set_time", ("timestamp", 64))
+    plm.add_type("reset", ("phy", 1))
     plm.add_type("write", ("timestamp", 64),
                           ("channel", 16),
                           ("address", 16),
@@ -178,7 +179,10 @@ class RTPacketSatellite(Module):
     def __init__(self, link_layer):
         self.tsc_load = Signal()
         self.tsc_value = Signal(64)
-        
+
+        self.reset = Signal(reset=1)
+        self.reset_phy = Signal(reset=1)
+
         self.fifo_space_channel = Signal(16)
         self.fifo_space_update = Signal()
         self.fifo_space = Signal(16)
@@ -240,6 +244,13 @@ class RTPacketSatellite(Module):
                 rx_dp.packet_as["write"].short_data)
         ]
 
+        reset = Signal()
+        reset_phy = Signal()
+        self.sync += [
+            self.reset.eq(reset),
+            self.reset_phy.eq(reset_phy)
+        ]
+
         rx_fsm = FSM(reset_state="INPUT")
         self.submodules += rx_fsm
 
@@ -252,6 +263,7 @@ class RTPacketSatellite(Module):
                         # mechanism
                         rx_plm.types["echo_request"]: echo_req.eq(1),
                         rx_plm.types["set_time"]: NextState("SET_TIME"),
+                        rx_plm.types["reset"]: NextState("RESET"),
                         rx_plm.types["write"]: NextState("WRITE"),
                         rx_plm.types["fifo_space_request"]:
                             NextState("FIFO_SPACE"),
@@ -266,6 +278,14 @@ class RTPacketSatellite(Module):
             self.tsc_load.eq(1),
             NextState("INPUT")
         )
+        rx_fsm.act("RESET",
+            If(rx_dp.packet_as["reset"].phy,
+                reset_phy.eq(1)
+            ).Else(
+                reset.eq(1)
+            ),
+            NextState("INPUT")
+        )
         rx_fsm.act("WRITE",
             self.write_stb.eq(1),
             NextState("INPUT")
@@ -401,6 +421,11 @@ class RTPacketMaster(Module):
         # a set_time request pending
         self.tsc_value = Signal(64)
 
+        # reset interface
+        self.reset_stb = Signal()
+        self.reset_ack = Signal()
+        self.reset_phy = Signal()
+
         # errors
         self.error_not = Signal()
         self.error_not_ack = Signal()
@@ -441,6 +466,13 @@ class RTPacketMaster(Module):
             self.set_time_stb, self.set_time_ack, None,
             set_time_stb, set_time_ack, None)
 
+        reset_stb = Signal()
+        reset_ack = Signal()
+        reset_phy = Signal()
+        self.submodules += _CrossDomainRequest("rtio",
+            self.reset_stb, self.reset_ack, self.reset_phy,
+            reset_stb, reset_ack, reset_phy)
+
         echo_stb = Signal()
         echo_ack = Signal()
         self.submodules += _CrossDomainRequest("rtio",
@@ -496,6 +528,8 @@ class RTPacketMaster(Module):
                 ).Elif(set_time_stb,
                     tsc_value_load.eq(1),
                     NextState("SET_TIME")
+                ).Elif(reset_stb,
+                    NextState("RESET")
                 )
             )
         )
@@ -523,6 +557,14 @@ class RTPacketMaster(Module):
                 NextState("IDLE_WRITE")
             )
         )
+        tx_fsm.act("RESET",
+            tx_dp.send("reset", phy=reset_phy),
+            tx_dp.stb.eq(1),
+            If(tx_dp.done,
+                reset_ack.eq(1),
+                NextState("IDLE_WRITE")
+            )
+        )
 
         # RX FSM
         rx_fsm = ClockDomainsRenamer("rtio_rx")(FSM(reset_state="INPUT"))

From 2d62a89143ab07f71f2d395a4318525e21e53ef7 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Wed, 23 Nov 2016 23:23:27 +0800
Subject: [PATCH 107/134] rtio: use large data register

---
 artiq/gateware/rtio/cri.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/artiq/gateware/rtio/cri.py b/artiq/gateware/rtio/cri.py
index 1e9dd40d2..62e354a39 100644
--- a/artiq/gateware/rtio/cri.py
+++ b/artiq/gateware/rtio/cri.py
@@ -61,7 +61,7 @@ class KernelInitiator(Module, AutoCSR):
         self.reset_phy = CSR()
         self.chan_sel = CSRStorage(24)
 
-        self.o_data = CSRStorage(32, write_from_dev=True)  # XXX -> 512
+        self.o_data = CSRStorage(512, write_from_dev=True)
         self.o_address = CSRStorage(16)
         self.o_timestamp = CSRStorage(64)
         self.o_we = CSR()

From eab18d8e34a9ba4f0f4b77a7c268d74f497e5249 Mon Sep 17 00:00:00 2001
From: Robert Jordens <rj@m-labs.hk>
Date: Mon, 21 Nov 2016 23:12:16 +0100
Subject: [PATCH 108/134] runtime.rs: wide rtio data

---
 artiq/runtime.rs/libksupport/api.rs  |  1 +
 artiq/runtime.rs/libksupport/rtio.rs | 45 +++++++++++++++++++++++++---
 2 files changed, 42 insertions(+), 4 deletions(-)

diff --git a/artiq/runtime.rs/libksupport/api.rs b/artiq/runtime.rs/libksupport/api.rs
index dbfbd759b..57b6302d8 100644
--- a/artiq/runtime.rs/libksupport/api.rs
+++ b/artiq/runtime.rs/libksupport/api.rs
@@ -102,6 +102,7 @@ static mut API: &'static [(&'static str, *const ())] = &[
     api!(rtio_get_counter = ::rtio::get_counter),
     api!(rtio_log),
     api!(rtio_output = ::rtio::output),
+    api!(rtio_output_list = ::rtio::output_list),
     api!(rtio_input_timestamp = ::rtio::input_timestamp),
     api!(rtio_input_data = ::rtio::input_data),
 
diff --git a/artiq/runtime.rs/libksupport/rtio.rs b/artiq/runtime.rs/libksupport/rtio.rs
index b98b9eaa6..76fda2149 100644
--- a/artiq/runtime.rs/libksupport/rtio.rs
+++ b/artiq/runtime.rs/libksupport/rtio.rs
@@ -1,4 +1,6 @@
 use board::csr;
+use core::ptr::{read_volatile, write_volatile};
+use core::slice;
 
 const RTIO_O_STATUS_FULL:           u32 = 1;
 const RTIO_O_STATUS_UNDERFLOW:      u32 = 2;
@@ -21,6 +23,21 @@ pub extern fn get_counter() -> i64 {
     }
 }
 
+#[inline(always)]
+pub unsafe fn rtio_o_data_write(w: u32) {
+    write_volatile(
+        csr::rtio::O_DATA_ADDR.offset((csr::rtio::O_DATA_SIZE - 1) as isize),
+        w);
+}
+
+#[inline(always)]
+pub unsafe fn rtio_i_data_read() -> u32 {
+    read_volatile(
+        csr::rtio::I_DATA_ADDR.offset((csr::rtio::I_DATA_SIZE - 1) as isize)
+        )
+}
+
+
 #[inline(never)]
 unsafe fn process_exceptional_status(timestamp: i64, channel: u32, status: u32) {
     if status & RTIO_O_STATUS_FULL != 0 {
@@ -57,7 +74,27 @@ pub extern fn output(timestamp: i64, channel: u32, addr: u32, data: u32) {
         csr::rtio::chan_sel_write(channel);
         csr::rtio::o_timestamp_write(timestamp as u64);
         csr::rtio::o_address_write(addr);
-        csr::rtio::o_data_write(data);
+        rtio_o_data_write(data);
+        csr::rtio::o_we_write(1);
+        let status = csr::rtio::o_status_read();
+        if status != 0 {
+            process_exceptional_status(timestamp, channel, status);
+        }
+    }
+}
+
+pub extern fn output_list(timestamp: i64, channel: u32, addr: u32,
+                          &(len, ptr): &(usize, *const u32)) {
+    unsafe {
+        csr::rtio::chan_sel_write(channel);
+        csr::rtio::o_timestamp_write(timestamp as u64);
+        csr::rtio::o_address_write(addr);
+        let data = slice::from_raw_parts(ptr, len);
+        for i in 0..data.len() {
+            write_volatile(
+                csr::rtio::O_DATA_ADDR.offset((csr::rtio::O_DATA_SIZE - 1 - i) as isize),
+                data[i]);
+        }
         csr::rtio::o_we_write(1);
         let status = csr::rtio::o_status_read();
         if status != 0 {
@@ -117,7 +154,7 @@ pub extern fn input_data(channel: u32) -> u32 {
             }
         }
 
-        let data = csr::rtio::i_data_read();
+        let data = rtio_i_data_read();
         csr::rtio::i_re_write(1);
         data
     }
@@ -134,14 +171,14 @@ pub fn log(timestamp: i64, data: &[u8]) {
             word <<= 8;
             word |= data[i] as u32;
             if i % 4 == 0 {
-                csr::rtio::o_data_write(word);
+                rtio_o_data_write(word);
                 csr::rtio::o_we_write(1);
                 word = 0;
             }
         }
 
         word <<= 8;
-        csr::rtio::o_data_write(word);
+        rtio_o_data_write(word);
         csr::rtio::o_we_write(1);
     }
 }

From 0d5f962d0cc72fc09a8c27655fd4a268af502d36 Mon Sep 17 00:00:00 2001
From: Robert Jordens <rj@m-labs.hk>
Date: Wed, 23 Nov 2016 14:56:20 +0100
Subject: [PATCH 109/134] runtime.rs/rtio.rs: style

---
 artiq/runtime.rs/libksupport/rtio.rs | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/artiq/runtime.rs/libksupport/rtio.rs b/artiq/runtime.rs/libksupport/rtio.rs
index 76fda2149..11279fb86 100644
--- a/artiq/runtime.rs/libksupport/rtio.rs
+++ b/artiq/runtime.rs/libksupport/rtio.rs
@@ -33,11 +33,9 @@ pub unsafe fn rtio_o_data_write(w: u32) {
 #[inline(always)]
 pub unsafe fn rtio_i_data_read() -> u32 {
     read_volatile(
-        csr::rtio::I_DATA_ADDR.offset((csr::rtio::I_DATA_SIZE - 1) as isize)
-        )
+        csr::rtio::I_DATA_ADDR.offset((csr::rtio::I_DATA_SIZE - 1) as isize))
 }
 
-
 #[inline(never)]
 unsafe fn process_exceptional_status(timestamp: i64, channel: u32, status: u32) {
     if status & RTIO_O_STATUS_FULL != 0 {

From 7cd27abaa61aa03714ea4122aa2d84642e051273 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Thu, 24 Nov 2016 00:09:53 +0800
Subject: [PATCH 110/134] drtio: do not reset remote TSC on reset command

---
 artiq/gateware/drtio/core.py |  4 ++--
 artiq/gateware/drtio/iot.py  | 17 +++++++++--------
 2 files changed, 11 insertions(+), 10 deletions(-)

diff --git a/artiq/gateware/drtio/core.py b/artiq/gateware/drtio/core.py
index 995b0b137..b7d9698b8 100644
--- a/artiq/gateware/drtio/core.py
+++ b/artiq/gateware/drtio/core.py
@@ -27,8 +27,8 @@ class DRTIOSatellite(Module):
         self.submodules.rt_packets = ClockDomainsRenamer("rtio")(
             rt_packets.RTPacketSatellite(link_layer_sync))
 
-        self.submodules.iot = ClockDomainsRenamer("rio")(
-            iot.IOT(self.rt_packets, channels, fine_ts_width, full_ts_width))
+        self.submodules.iot = iot.IOT(
+            self.rt_packets, channels, fine_ts_width, full_ts_width)
 
         self.clock_domains.cd_rio = ClockDomain()
         self.clock_domains.cd_rio_phy = ClockDomain()
diff --git a/artiq/gateware/drtio/iot.py b/artiq/gateware/drtio/iot.py
index 21e362367..9b6ec2a83 100644
--- a/artiq/gateware/drtio/iot.py
+++ b/artiq/gateware/drtio/iot.py
@@ -8,7 +8,7 @@ from artiq.gateware.rtio import rtlink
 class IOT(Module):
     def __init__(self, rt_packets, channels, max_fine_ts_width, full_ts_width):
         tsc = Signal(full_ts_width - max_fine_ts_width)
-        self.sync += \
+        self.sync.rtio += \
             If(rt_packets.tsc_load,
                 tsc.eq(rt_packets.tsc_value)
             ).Else(
@@ -30,7 +30,8 @@ class IOT(Module):
                 ev_layout.append(("address", address_width))
             ev_layout.append(("timestamp", len(tsc) + fine_ts_width))
 
-            fifo = SyncFIFOBuffered(layout_len(ev_layout), channel.ofifo_depth)
+            fifo = ClockDomainsRenamer("rio")(
+                SyncFIFOBuffered(layout_len(ev_layout), channel.ofifo_depth))
             self.submodules += fifo
             fifo_in = Record(ev_layout)
             fifo_out = Record(ev_layout)
@@ -40,7 +41,7 @@ class IOT(Module):
             ]
 
             # FIFO level
-            self.sync += \
+            self.sync.rio += \
                 If(rt_packets.fifo_space_update &
                    (rt_packets.fifo_space_channel == n),
                     rt_packets.fifo_space.eq(channel.ofifo_depth - fifo.level))
@@ -48,7 +49,7 @@ class IOT(Module):
             # FIFO write
             self.comb += fifo.we.eq(rt_packets.write_stb
                                     & (rt_packets.write_channel == n))
-            self.sync += [
+            self.sync.rio += [
                 If(rt_packets.write_overflow_ack,
                     rt_packets.write_overflow.eq(0)),
                 If(rt_packets.write_underflow_ack,
@@ -68,7 +69,7 @@ class IOT(Module):
                 rt_packets.write_timestamp[max_fine_ts_width-fine_ts_width:])
 
             # FIFO read
-            self.sync += [
+            self.sync.rio += [
                 fifo.re.eq(0),
                 interface.stb.eq(0),
                 If(fifo.readable &
@@ -78,8 +79,8 @@ class IOT(Module):
                 )
             ]
             if data_width:
-                self.sync += interface.data.eq(fifo_out.data)
+                self.sync.rio += interface.data.eq(fifo_out.data)
             if address_width:
-                self.sync += interface.address.eq(fifo_out.address)
+                self.sync.rio += interface.address.eq(fifo_out.address)
             if fine_ts_width:
-                self.sync += interface.fine_ts.eq(fifo_out.timestamp[:fine_ts_width])
+                self.sync.rio += interface.fine_ts.eq(fifo_out.timestamp[:fine_ts_width])

From 8b736ddbc90182965d4ee891f6b2a85bfe3ac23c Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Thu, 24 Nov 2016 00:37:53 +0800
Subject: [PATCH 111/134] drtio: update test

---
 artiq/test/gateware/drtio/test_full_stack.py | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/artiq/test/gateware/drtio/test_full_stack.py b/artiq/test/gateware/drtio/test_full_stack.py
index ae0ce7290..3c2c13975 100644
--- a/artiq/test/gateware/drtio/test_full_stack.py
+++ b/artiq/test/gateware/drtio/test_full_stack.py
@@ -40,8 +40,9 @@ class DUT(Module):
         self.ttl0 = Signal()
         self.ttl1 = Signal()
         self.transceivers = DummyTransceiverPair(nwords)
-        
+
         self.submodules.master = DRTIOMaster(self.transceivers.alice)
+        self.submodules.master_ki = rtio.KernelInitiator(self.master.cri)
 
         rx_synchronizer = DummyRXSynchronizer()
         self.submodules.phy0 = ttl_simple.Output(self.ttl0)
@@ -59,7 +60,7 @@ class TestFullStack(unittest.TestCase):
 
     def test_controller(self):
         dut = DUT(2)
-        kcsrs = dut.master.rt_controller.kcsrs
+        kcsrs = dut.master_ki
         csrs = dut.master.rt_controller.csrs
         mgr = dut.master.rt_manager
 
@@ -160,7 +161,7 @@ class TestFullStack(unittest.TestCase):
             self.assertEqual(wlen, 2)
 
         def test_tsc_error():
-            err_present = yield from mgr.err_present.read()
+            err_present = yield from mgr.packet_err_present.read()
             self.assertEqual(err_present, 0)
             yield from csrs.tsc_correction.write(10000000)
             yield from csrs.set_time.write(1)
@@ -170,17 +171,17 @@ class TestFullStack(unittest.TestCase):
             yield from write(0, 1)
             for i in range(10):
                yield
-            err_present = yield from mgr.err_present.read()
-            err_code = yield from mgr.err_code.read()
+            err_present = yield from mgr.packet_err_present.read()
+            err_code = yield from mgr.packet_err_code.read()
             self.assertEqual(err_present, 1)
             self.assertEqual(err_code, 3)
-            yield from mgr.err_present.write(1)
+            yield from mgr.packet_err_present.write(1)
             yield
-            err_present = yield from mgr.err_present.read()
+            err_present = yield from mgr.packet_err_present.read()
             self.assertEqual(err_present, 0)
 
         def test():
-            while not (yield dut.master.link_layer.ready):
+            while not (yield from dut.master.link_layer.link_status.read()):
                 yield
 
             yield from test_init()
@@ -214,7 +215,7 @@ class TestFullStack(unittest.TestCase):
         mgr = dut.master.rt_manager
 
         def test():
-            while not (yield dut.master.link_layer.ready):
+            while not (yield from dut.master.link_layer.link_status.read()):
                 yield
 
             yield from mgr.update_packet_cnt.write(1)

From 8090abef5d4fbc51406b3a71d3f29651414f5a20 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Fri, 25 Nov 2016 17:04:09 +0800
Subject: [PATCH 112/134] drtio: large data support

---
 artiq/gateware/drtio/rt_packets.py | 245 +++++++++++++++++++----------
 1 file changed, 165 insertions(+), 80 deletions(-)

diff --git a/artiq/gateware/drtio/rt_packets.py b/artiq/gateware/drtio/rt_packets.py
index 1e33e3bd4..83053865b 100644
--- a/artiq/gateware/drtio/rt_packets.py
+++ b/artiq/gateware/drtio/rt_packets.py
@@ -29,8 +29,20 @@ class PacketLayoutManager:
             layout.append(("packet_pad", self.alignment - misalignment))
         self.layouts[name] = layout
 
+    def field_length(self, type_name, field_name):
+        layout = self.layouts[type_name]
+        for name, length in layout:
+            if name == field_name:
+                return length
+        raise KeyError
+
 
 def get_m2s_layouts(alignment):
+    if alignment > 128:
+        short_data_len = alignment - 128 + 16
+    else:
+        short_data_len = 16
+
     plm = PacketLayoutManager(alignment)
     plm.add_type("echo_request")
     plm.add_type("set_time", ("timestamp", 64))
@@ -38,8 +50,8 @@ def get_m2s_layouts(alignment):
     plm.add_type("write", ("timestamp", 64),
                           ("channel", 16),
                           ("address", 16),
-                          ("data_len", 8),
-                          ("short_data", 8))
+                          ("extra_data_cnt", 8),
+                          ("short_data", short_data_len))
     plm.add_type("fifo_space_request", ("channel", 16))
     return plm
 
@@ -125,34 +137,33 @@ class TransmitDatapath(Module):
         self.ws = ws
         self.plm = plm
 
-        # inputs
         self.packet_buffer = Signal(max(layout_len(l)
                                         for l in plm.layouts.values()))
         w_in_packet = len(self.packet_buffer)//ws
-        self.packet_len = Signal(max=w_in_packet+1)
+        self.packet_last_n = Signal(max=w_in_packet)
+        self.packet_stb = Signal()
+        self.packet_last = Signal()
 
-        # control
-        self.stb = Signal()
-        self.done = Signal()
+        self.raw_stb = Signal()
+        self.raw_data = Signal(ws)
 
         # # #
 
-        packet_buffer_count = Signal(max=w_in_packet+1)
+        packet_buffer_count = Signal(max=w_in_packet)
+        self.comb += self.packet_last.eq(packet_buffer_count == self.packet_last_n)
         self.sync += [
-            self.done.eq(0),
             frame.eq(0),
             packet_buffer_count.eq(0),
-
-            If(self.stb & ~self.done,
-                If(packet_buffer_count == self.packet_len,
-                    self.done.eq(1)
-                ).Else(
-                    frame.eq(1),
-                    Case(packet_buffer_count, 
-                         {i: data.eq(self.packet_buffer[i*ws:(i+1)*ws])
-                          for i in range(w_in_packet)}),
-                    packet_buffer_count.eq(packet_buffer_count + 1)
-                )
+            If(self.packet_stb,
+                frame.eq(1),
+                Case(packet_buffer_count,
+                     {i: data.eq(self.packet_buffer[i*ws:(i+1)*ws])
+                      for i in range(w_in_packet)}),
+                packet_buffer_count.eq(packet_buffer_count + 1)
+            ),
+            If(self.raw_stb,
+                frame.eq(1),
+                data.eq(self.raw_data)
             )
         ]
 
@@ -170,8 +181,9 @@ class TransmitDatapath(Module):
         if kwargs:
             raise ValueError
         return [
+            self.packet_stb.eq(1),
             self.packet_buffer.eq(value),
-            self.packet_len.eq(idx//self.ws)
+            self.packet_last_n.eq(idx//self.ws-1)
         ]
 
 
@@ -191,7 +203,7 @@ class RTPacketSatellite(Module):
         self.write_timestamp = Signal(64)
         self.write_channel = Signal(16)
         self.write_address = Signal(16)
-        self.write_data = Signal(256)
+        self.write_data = Signal(512)
         self.write_overflow = Signal()
         self.write_overflow_ack = Signal()
         self.write_underflow = Signal()
@@ -212,6 +224,20 @@ class RTPacketSatellite(Module):
             link_layer.tx_rt_frame, link_layer.tx_rt_data, tx_plm)
         self.submodules += tx_dp
 
+        # RX write data buffer
+        write_data_buffer_load = Signal()
+        write_data_buffer_cnt = Signal(max=512//ws+1)
+        write_data_buffer = Signal(512)
+        self.sync += \
+            If(write_data_buffer_load,
+                Case(write_data_buffer_cnt,
+                     {i: write_data_buffer[i*ws:(i+1)*ws].eq(link_layer.rx_rt_data)
+                      for i in range(512//ws)}),
+                write_data_buffer_cnt.eq(write_data_buffer_cnt + 1)
+            ).Else(
+                write_data_buffer_cnt.eq(0)
+            )
+
         # RX->TX
         echo_req = Signal()
         err_set = Signal()
@@ -241,7 +267,7 @@ class RTPacketSatellite(Module):
             self.write_address.eq(
                 rx_dp.packet_as["write"].address),
             self.write_data.eq(
-                rx_dp.packet_as["write"].short_data)
+                Cat(rx_dp.packet_as["write"].short_data, write_data_buffer))
         ]
 
         reset = Signal()
@@ -287,8 +313,11 @@ class RTPacketSatellite(Module):
             NextState("INPUT")
         )
         rx_fsm.act("WRITE",
-            self.write_stb.eq(1),
-            NextState("INPUT")
+            write_data_buffer_load.eq(1),
+            If(write_data_buffer_cnt == rx_dp.packet_as["write"].extra_data_cnt,
+                self.write_stb.eq(1),
+                NextState("INPUT")
+            )
         )
         rx_fsm.act("FIFO_SPACE",
             fifo_space_set.eq(1),
@@ -309,32 +338,27 @@ class RTPacketSatellite(Module):
         )
         tx_fsm.act("ECHO",
             tx_dp.send("echo_reply"),
-            tx_dp.stb.eq(1),
-            If(tx_dp.done, NextState("IDLE"))
+            If(tx_dp.packet_last, NextState("IDLE"))
         )
         tx_fsm.act("FIFO_SPACE",
             fifo_space_ack.eq(1),
             tx_dp.send("fifo_space_reply", space=self.fifo_space),
-            tx_dp.stb.eq(1),
-            If(tx_dp.done, NextState("IDLE"))
+            If(tx_dp.packet_last, NextState("IDLE"))
         )
         tx_fsm.act("ERROR_WRITE_OVERFLOW",
             self.write_overflow_ack.eq(1),
             tx_dp.send("error", code=error_codes["write_overflow"]),
-            tx_dp.stb.eq(1),
-            If(tx_dp.done, NextState("IDLE"))
+            If(tx_dp.packet_last, NextState("IDLE"))
         )
         tx_fsm.act("ERROR_WRITE_UNDERFLOW",
             self.write_underflow_ack.eq(1),
             tx_dp.send("error", code=error_codes["write_underflow"]),
-            tx_dp.stb.eq(1),
-            If(tx_dp.done, NextState("IDLE"))
+            If(tx_dp.packet_last, NextState("IDLE"))
         )
         tx_fsm.act("ERROR",
             err_ack.eq(1),
             tx_dp.send("error", code=err_code),
-            tx_dp.stb.eq(1),
-            If(tx_dp.done, NextState("IDLE"))
+            If(tx_dp.packet_last, NextState("IDLE"))
         )
 
 
@@ -399,7 +423,7 @@ class RTPacketMaster(Module):
         self.write_timestamp = Signal(64)
         self.write_channel = Signal(16)
         self.write_address = Signal(16)
-        self.write_data = Signal(256)
+        self.write_data = Signal(512)
 
         # fifo space interface
         # write with timestamp[48:] == 0xffff to make a fifo space request
@@ -437,23 +461,83 @@ class RTPacketMaster(Module):
 
         # # #
 
-        # CDC
+        # RX/TX datapath
+        assert len(link_layer.tx_rt_data) == len(link_layer.rx_rt_data)
+        assert len(link_layer.tx_rt_data) % 8 == 0
+        ws = len(link_layer.tx_rt_data)
+        tx_plm = get_m2s_layouts(ws)
+        tx_dp = ClockDomainsRenamer("rtio")(TransmitDatapath(
+            link_layer.tx_rt_frame, link_layer.tx_rt_data, tx_plm))
+        self.submodules += tx_dp
+        rx_plm = get_s2m_layouts(ws)
+        rx_dp = ClockDomainsRenamer("rtio_rx")(ReceiveDatapath(
+            link_layer.rx_rt_frame, link_layer.rx_rt_data, rx_plm))
+        self.submodules += rx_dp
+
+        # Write FIFO and extra data count
         wfifo = ClockDomainsRenamer({"write": "sys", "read": "rtio"})(
-            AsyncFIFO(64+16+16+256, write_fifo_depth))
+            AsyncFIFO(64+16+16+512, write_fifo_depth))
         self.submodules += wfifo
-        write_timestamp = Signal(64)
-        write_channel = Signal(16)
-        write_address = Signal(16)
-        write_data = Signal(256)
+        write_timestamp_d = Signal(64)
+        write_channel_d = Signal(16)
+        write_address_d = Signal(16)
+        write_data_d = Signal(512)
         self.comb += [
             wfifo.we.eq(self.write_stb),
             self.write_ack.eq(wfifo.writable),
             wfifo.din.eq(Cat(self.write_timestamp, self.write_channel,
                              self.write_address, self.write_data)),
-            Cat(write_timestamp, write_channel,
-                write_address, write_data).eq(wfifo.dout)
+            Cat(write_timestamp_d, write_channel_d,
+                write_address_d, write_data_d).eq(wfifo.dout)
         ]
 
+        wfb_readable = Signal()
+        wfb_re = Signal()
+
+        self.comb += wfifo.re.eq(wfifo.readable & (~wfb_readable | wfb_re))
+        self.sync.rtio += \
+            If(wfifo.re,
+                wfb_readable.eq(1),
+            ).Elif(wfb_re,
+                wfb_readable.eq(0),
+            )
+
+        write_timestamp = Signal(64)
+        write_channel = Signal(16)
+        write_address = Signal(16)
+        write_extra_data_cnt = Signal(8)
+        write_data = Signal(512)
+
+        self.sync.rtio += If(wfifo.re,
+            write_timestamp.eq(write_timestamp_d),
+            write_channel.eq(write_channel_d),
+            write_address.eq(write_address_d),
+            write_data.eq(write_data_d))
+
+        short_data_len = tx_plm.field_length("write", "short_data")
+        write_extra_data = Signal(512)
+        self.comb += write_extra_data.eq(write_data[short_data_len:])
+        for i in range(512//ws):
+            self.sync.rtio += If(wfifo.re,
+                If(write_extra_data[ws*i:ws*(i+1)] != 0, write_extra_data_cnt.eq(i+1)))
+
+        extra_data_ce = Signal()
+        extra_data_last = Signal()
+        extra_data_counter = Signal(max=512//ws+1)
+        self.comb += [
+            Case(extra_data_counter, 
+                {i+1: tx_dp.raw_data.eq(write_extra_data[i*ws:(i+1)*ws])
+                 for i in range(512//ws)}),
+            extra_data_last.eq(extra_data_counter == write_extra_data_cnt)
+        ]
+        self.sync.rtio += \
+            If(extra_data_ce,
+                extra_data_counter.eq(extra_data_counter + 1),
+            ).Else(
+                extra_data_counter.eq(1)
+            )
+
+        # CDC
         fifo_space_not = Signal()
         fifo_space = Signal(16)
         self.submodules += _CrossDomainNotification("rtio_rx",
@@ -485,21 +569,8 @@ class RTPacketMaster(Module):
             error_not, error_code,
             self.error_not, self.error_not_ack, self.error_code)
 
-        # RX/TX datapath
-        assert len(link_layer.tx_rt_data) == len(link_layer.rx_rt_data)
-        assert len(link_layer.tx_rt_data) % 8 == 0
-        ws = len(link_layer.tx_rt_data)
-        tx_plm = get_m2s_layouts(ws)
-        tx_dp = ClockDomainsRenamer("rtio")(TransmitDatapath(
-            link_layer.tx_rt_frame, link_layer.tx_rt_data, tx_plm))
-        self.submodules += tx_dp
-        rx_plm = get_s2m_layouts(ws)
-        rx_dp = ClockDomainsRenamer("rtio_rx")(ReceiveDatapath(
-            link_layer.rx_rt_frame, link_layer.rx_rt_data, rx_plm))
-        self.submodules += rx_dp
-
         # TX FSM
-        tx_fsm = ClockDomainsRenamer("rtio")(FSM(reset_state="IDLE_WRITE"))
+        tx_fsm = ClockDomainsRenamer("rtio")(FSM(reset_state="IDLE"))
         self.submodules += tx_fsm
 
         echo_sent_now = Signal()
@@ -508,18 +579,12 @@ class RTPacketMaster(Module):
         tsc_value_load = Signal()
         self.sync.rtio += If(tsc_value_load, tsc_value.eq(self.tsc_value))
 
-        tx_fsm.act("IDLE_WRITE",
-            tx_dp.send("write",
-                timestamp=write_timestamp,
-                channel=write_channel,
-                address=write_address,
-                short_data=write_data),
-            If(wfifo.readable,
+        tx_fsm.act("IDLE",
+            If(wfb_readable,
                 If(write_timestamp[48:] == 0xffff,
                     NextState("FIFO_SPACE")
                 ).Else(
-                    tx_dp.stb.eq(1),
-                    wfifo.re.eq(tx_dp.done)
+                    NextState("WRITE")
                 )
             ).Else(
                 If(echo_stb,
@@ -533,36 +598,56 @@ class RTPacketMaster(Module):
                 )
             )
         )
+        tx_fsm.act("WRITE",
+            tx_dp.send("write",
+                timestamp=write_timestamp,
+                channel=write_channel,
+                address=write_address,
+                extra_data_cnt=write_extra_data_cnt,
+                short_data=write_data[:short_data_len]),
+            If(tx_dp.packet_last,
+                If(write_extra_data_cnt == 0,
+                    wfb_re.eq(1),
+                    NextState("IDLE")
+                ).Else(
+                    NextState("WRITE_EXTRA")
+                )
+            )
+        )
+        tx_fsm.act("WRITE_EXTRA",
+            tx_dp.raw_stb.eq(1),
+            extra_data_ce.eq(1),
+            If(extra_data_last,
+                wfb_re.eq(1),
+                NextState("IDLE")
+            )
+        )
         tx_fsm.act("FIFO_SPACE",
             tx_dp.send("fifo_space_request", channel=write_channel),
-            tx_dp.stb.eq(1),
-            If(tx_dp.done,
-                wfifo.re.eq(1),
-                NextState("IDLE_WRITE")
+            If(tx_dp.packet_last,
+                wfb_re.eq(1),
+                NextState("IDLE")
             )
         )
         tx_fsm.act("ECHO",
             tx_dp.send("echo_request"),
-            tx_dp.stb.eq(1),
-            If(tx_dp.done,
+            If(tx_dp.packet_last,
                 echo_ack.eq(1),
-                NextState("IDLE_WRITE")
+                NextState("IDLE")
             )
         )
         tx_fsm.act("SET_TIME",
             tx_dp.send("set_time", timestamp=tsc_value),
-            tx_dp.stb.eq(1),
-            If(tx_dp.done,
+            If(tx_dp.packet_last,
                 set_time_ack.eq(1),
-                NextState("IDLE_WRITE")
+                NextState("IDLE")
             )
         )
         tx_fsm.act("RESET",
             tx_dp.send("reset", phy=reset_phy),
-            tx_dp.stb.eq(1),
-            If(tx_dp.done,
+            If(tx_dp.packet_last,
                 reset_ack.eq(1),
-                NextState("IDLE_WRITE")
+                NextState("IDLE")
             )
         )
 

From d381dd53843893dcb6599962924fde8f5543b152 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Fri, 25 Nov 2016 18:33:55 +0800
Subject: [PATCH 113/134] drtio: remove stale signal from test

---
 artiq/test/gateware/drtio/test_full_stack.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/artiq/test/gateware/drtio/test_full_stack.py b/artiq/test/gateware/drtio/test_full_stack.py
index 3c2c13975..bf22d8e8d 100644
--- a/artiq/test/gateware/drtio/test_full_stack.py
+++ b/artiq/test/gateware/drtio/test_full_stack.py
@@ -19,13 +19,11 @@ class DummyTransceiverPair:
         self.alice = SimpleNamespace(
             encoder=SimpleNamespace(k=a2b_k, d=a2b_d),
             decoders=[SimpleNamespace(k=k, d=d) for k, d in zip(b2a_k, b2a_d)],
-            rx_reset=Signal(),
             rx_ready=1
         )
         self.bob = SimpleNamespace(
             encoder=SimpleNamespace(k=b2a_k, d=b2a_d),
             decoders=[SimpleNamespace(k=k, d=d) for k, d in zip(a2b_k, a2b_d)],
-            rx_reset=Signal(),
             rx_ready=1
         )
 

From 090396448863b00066e48e7a9c3f959319a691d8 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Sun, 27 Nov 2016 02:12:50 +0800
Subject: [PATCH 114/134] drtio: large data fixes

---
 artiq/gateware/drtio/rt_packets.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/artiq/gateware/drtio/rt_packets.py b/artiq/gateware/drtio/rt_packets.py
index 83053865b..fc6289ba9 100644
--- a/artiq/gateware/drtio/rt_packets.py
+++ b/artiq/gateware/drtio/rt_packets.py
@@ -231,7 +231,7 @@ class RTPacketSatellite(Module):
         self.sync += \
             If(write_data_buffer_load,
                 Case(write_data_buffer_cnt,
-                     {i: write_data_buffer[i*ws:(i+1)*ws].eq(link_layer.rx_rt_data)
+                     {i: write_data_buffer[i*ws:(i+1)*ws].eq(rx_dp.data_r)
                       for i in range(512//ws)}),
                 write_data_buffer_cnt.eq(write_data_buffer_cnt + 1)
             ).Else(
@@ -515,11 +515,14 @@ class RTPacketMaster(Module):
             write_data.eq(write_data_d))
 
         short_data_len = tx_plm.field_length("write", "short_data")
-        write_extra_data = Signal(512)
-        self.comb += write_extra_data.eq(write_data[short_data_len:])
+        write_extra_data_d = Signal(512)
+        self.comb += write_extra_data_d.eq(write_data_d[short_data_len:])
         for i in range(512//ws):
             self.sync.rtio += If(wfifo.re,
-                If(write_extra_data[ws*i:ws*(i+1)] != 0, write_extra_data_cnt.eq(i+1)))
+                If(write_extra_data_d[ws*i:ws*(i+1)] != 0, write_extra_data_cnt.eq(i+1)))
+
+        write_extra_data = Signal(512)
+        self.sync.rtio += If(wfifo.re, write_extra_data.eq(write_extra_data_d))
 
         extra_data_ce = Signal()
         extra_data_last = Signal()

From b2450c7c56aaf5dbf7009490209fe59e53c96f53 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Sun, 27 Nov 2016 12:57:12 +0800
Subject: [PATCH 115/134] drtio: test large data

---
 artiq/test/gateware/drtio/test_full_stack.py | 34 +++++++++++++++++---
 1 file changed, 29 insertions(+), 5 deletions(-)

diff --git a/artiq/test/gateware/drtio/test_full_stack.py b/artiq/test/gateware/drtio/test_full_stack.py
index bf22d8e8d..79833256e 100644
--- a/artiq/test/gateware/drtio/test_full_stack.py
+++ b/artiq/test/gateware/drtio/test_full_stack.py
@@ -1,10 +1,12 @@
 import unittest
 from types import SimpleNamespace
+import random
 
 from migen import *
 
 from artiq.gateware.drtio import *
 from artiq.gateware import rtio
+from artiq.gateware.rtio import rtlink
 from artiq.gateware.rtio.phy import ttl_simple
 from artiq.coredevice.exceptions import *
 
@@ -33,6 +35,14 @@ class DummyRXSynchronizer:
         return signal
 
 
+class LargeDataReceiver(Module):
+    def __init__(self, width):
+        self.rtlink = rtlink.Interface(rtlink.OInterface(width))
+        self.received_data = Signal(width)
+        self.sync.rio_phy += If(self.rtlink.o.stb,
+            self.received_data.eq(self.rtlink.o.data))
+
+
 class DUT(Module):
     def __init__(self, nwords):
         self.ttl0 = Signal()
@@ -45,9 +55,11 @@ class DUT(Module):
         rx_synchronizer = DummyRXSynchronizer()
         self.submodules.phy0 = ttl_simple.Output(self.ttl0)
         self.submodules.phy1 = ttl_simple.Output(self.ttl1)
+        self.submodules.phy2 = LargeDataReceiver(512)
         rtio_channels = [
             rtio.Channel.from_phy(self.phy0, ofifo_depth=4),
-            rtio.Channel.from_phy(self.phy1, ofifo_depth=4)
+            rtio.Channel.from_phy(self.phy1, ofifo_depth=4),
+            rtio.Channel.from_phy(self.phy2, ofifo_depth=4),
         ]
         self.submodules.satellite = DRTIOSatellite(
             self.transceivers.bob, rx_synchronizer, rtio_channels)
@@ -64,11 +76,13 @@ class TestFullStack(unittest.TestCase):
 
         ttl_changes = []
         correct_ttl_changes = [
+            # from test_pulses
             (203, 0),
             (208, 0),
             (208, 1),
             (214, 1),
 
+            # from test_fifo_space
             (414, 0),
             (454, 0),
             (494, 0),
@@ -136,6 +150,15 @@ class TestFullStack(unittest.TestCase):
                 yield from write(0, 1)
             delay(200*8)
 
+        def test_large_data():
+            correct_large_data = random.Random(0).randrange(2**512-1)
+            self.assertNotEqual((yield dut.phy2.received_data), correct_large_data)
+            delay(10*8)
+            yield from write(2, correct_large_data)
+            for i in range(40):
+                yield
+            self.assertEqual((yield dut.phy2.received_data), correct_large_data)
+
         def test_fifo_space():
             delay(200*8)
             max_wlen = 0
@@ -161,11 +184,11 @@ class TestFullStack(unittest.TestCase):
         def test_tsc_error():
             err_present = yield from mgr.packet_err_present.read()
             self.assertEqual(err_present, 0)
-            yield from csrs.tsc_correction.write(10000000)
+            yield from csrs.tsc_correction.write(100000000)
             yield from csrs.set_time.write(1)
-            for i in range(5):
+            for i in range(15):
                yield
-            delay(10000)
+            delay(10000*8)
             yield from write(0, 1)
             for i in range(10):
                yield
@@ -187,6 +210,7 @@ class TestFullStack(unittest.TestCase):
             yield from test_pulses()
             yield from test_sequence_error()
             yield from test_fifo_space()
+            yield from test_large_data()
             yield from test_fifo_emptied()
             yield from test_tsc_error()
 
@@ -203,7 +227,7 @@ class TestFullStack(unittest.TestCase):
                 yield
                 cycle += 1
 
-        run_simulation(dut, 
+        run_simulation(dut,
             {"sys": test(), "rtio": check_ttls()}, self.clocks)
         self.assertEqual(ttl_changes, correct_ttl_changes)
 

From 046b8bfd33d43ce54b0d20f40faeb32a1eb5d1dc Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Sun, 27 Nov 2016 13:19:12 +0800
Subject: [PATCH 116/134] drtio: fix transmit datapath with transceiver width >
 max packet width

---
 artiq/gateware/drtio/rt_packets.py | 36 ++++++++++++++++++++----------
 1 file changed, 24 insertions(+), 12 deletions(-)

diff --git a/artiq/gateware/drtio/rt_packets.py b/artiq/gateware/drtio/rt_packets.py
index fc6289ba9..ccdd0124c 100644
--- a/artiq/gateware/drtio/rt_packets.py
+++ b/artiq/gateware/drtio/rt_packets.py
@@ -140,7 +140,7 @@ class TransmitDatapath(Module):
         self.packet_buffer = Signal(max(layout_len(l)
                                         for l in plm.layouts.values()))
         w_in_packet = len(self.packet_buffer)//ws
-        self.packet_last_n = Signal(max=w_in_packet)
+        self.packet_last_n = Signal(max=max(w_in_packet, 2))
         self.packet_stb = Signal()
         self.packet_last = Signal()
 
@@ -149,18 +149,30 @@ class TransmitDatapath(Module):
 
         # # #
 
-        packet_buffer_count = Signal(max=w_in_packet)
-        self.comb += self.packet_last.eq(packet_buffer_count == self.packet_last_n)
+        self.sync += frame.eq(0)
+
+        if w_in_packet > 1:
+            packet_buffer_count = Signal(max=w_in_packet)
+            self.comb += self.packet_last.eq(packet_buffer_count == self.packet_last_n)
+            self.sync += [
+                packet_buffer_count.eq(0),
+                If(self.packet_stb,
+                    frame.eq(1),
+                    Case(packet_buffer_count,
+                         {i: data.eq(self.packet_buffer[i*ws:(i+1)*ws])
+                          for i in range(w_in_packet)}),
+                    packet_buffer_count.eq(packet_buffer_count + 1)
+                )
+            ]
+        else:
+            self.comb += self.packet_last.eq(1)
+            self.sync += \
+                If(self.packet_stb,
+                    frame.eq(1),
+                    data.eq(self.packet_buffer)
+                )
+
         self.sync += [
-            frame.eq(0),
-            packet_buffer_count.eq(0),
-            If(self.packet_stb,
-                frame.eq(1),
-                Case(packet_buffer_count,
-                     {i: data.eq(self.packet_buffer[i*ws:(i+1)*ws])
-                      for i in range(w_in_packet)}),
-                packet_buffer_count.eq(packet_buffer_count + 1)
-            ),
             If(self.raw_stb,
                 frame.eq(1),
                 data.eq(self.raw_data)

From d37b73fd312e32fecba76c727b077338f9e3f9ac Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Mon, 28 Nov 2016 14:33:06 +0800
Subject: [PATCH 117/134] drtio: FIFO timeout is handled in gateware + give
 remote side more time

---
 artiq/runtime.rs/src/drtio.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/artiq/runtime.rs/src/drtio.rs b/artiq/runtime.rs/src/drtio.rs
index 6050a8c1b..1f05803f3 100644
--- a/artiq/runtime.rs/src/drtio.rs
+++ b/artiq/runtime.rs/src/drtio.rs
@@ -21,7 +21,7 @@ fn drtio_init_channel(channel: u16) {
         
         csr::drtio::o_reset_channel_status_write(1);
         csr::drtio::o_get_fifo_space_write(1);
-        while csr::drtio::o_wait_read() == 1 {}  // TODO: timeout
+        while csr::drtio::o_wait_read() == 1 {}
         info!("FIFO space on channel {} is {}", channel, csr::drtio::o_dbg_fifo_space_read());
 
         csr::drtio::chan_sel_override_en_write(0);
@@ -33,7 +33,7 @@ pub fn link_thread(waiter: Waiter, _spawner: Spawner) {
         waiter.until(drtio_link_is_up).unwrap();
         info!("link RX is up");
 
-        waiter.sleep(500).unwrap();
+        waiter.sleep(600).unwrap();
         info!("wait for remote side done");
 
         drtio_sync_tsc();

From c419c422fa9e7034001dbc05bcf4c43e9b5d4635 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Mon, 28 Nov 2016 14:33:26 +0800
Subject: [PATCH 118/134] drtio: support for local RTIO core

---
 artiq/examples/drtio/device_db.pyon           | 61 +++++++++++++++++++
 .../drtio/repository/blink_forever.py         | 13 +++-
 artiq/gateware/rtio/__init__.py               |  2 +-
 artiq/gateware/rtio/cri.py                    | 31 ++++++++++
 artiq/gateware/targets/kc705_drtio_master.py  | 18 +++++-
 5 files changed, 119 insertions(+), 6 deletions(-)

diff --git a/artiq/examples/drtio/device_db.pyon b/artiq/examples/drtio/device_db.pyon
index 7696aacce..f8d9e2fb4 100644
--- a/artiq/examples/drtio/device_db.pyon
+++ b/artiq/examples/drtio/device_db.pyon
@@ -83,5 +83,66 @@
         "arguments": {"channel": 9}
     },
 
+    "led0": {
+        "type": "local",
+        "module": "artiq.coredevice.ttl",
+        "class": "TTLOut",
+        "arguments": {"channel": 0x010000},
+    },
+    "led1": {
+        "type": "local",
+        "module": "artiq.coredevice.ttl",
+        "class": "TTLOut",
+        "arguments": {"channel": 0x010001},
+    },
+    "led2": {
+        "type": "local",
+        "module": "artiq.coredevice.ttl",
+        "class": "TTLOut",
+        "arguments": {"channel": 0x010002},
+    },
+    "led3": {
+        "type": "local",
+        "module": "artiq.coredevice.ttl",
+        "class": "TTLOut",
+        "arguments": {"channel": 0x010003},
+    },
+    "led4": {
+        "type": "local",
+        "module": "artiq.coredevice.ttl",
+        "class": "TTLOut",
+        "arguments": {"channel": 0x010004},
+    },
+    "led5": {
+        "type": "local",
+        "module": "artiq.coredevice.ttl",
+        "class": "TTLOut",
+        "arguments": {"channel": 0x010005},
+    },
+    "led6": {
+        "type": "local",
+        "module": "artiq.coredevice.ttl",
+        "class": "TTLOut",
+        "arguments": {"channel": 0x010006},
+    },
+    "led7": {
+        "type": "local",
+        "module": "artiq.coredevice.ttl",
+        "class": "TTLOut",
+        "arguments": {"channel": 0x010007},
+    },
+
+    "smap": {
+        "type": "local",
+        "module": "artiq.coredevice.ttl",
+        "class": "TTLOut",
+        "arguments": {"channel": 0x010008}
+    },
+    "sman": {
+        "type": "local",
+        "module": "artiq.coredevice.ttl",
+        "class": "TTLOut",
+        "arguments": {"channel": 0x010009}
+    },
 
 }
diff --git a/artiq/examples/drtio/repository/blink_forever.py b/artiq/examples/drtio/repository/blink_forever.py
index fb074661c..a7b1294b5 100644
--- a/artiq/examples/drtio/repository/blink_forever.py
+++ b/artiq/examples/drtio/repository/blink_forever.py
@@ -4,17 +4,24 @@ from artiq.experiment import *
 class BlinkForever(EnvExperiment):
     def build(self):
         self.setattr_device("core")
-        self.leds = [self.get_device("rled" + str(i)) for i in range(8)]
+        self.rleds = [self.get_device("rled" + str(i)) for i in range(8)]
+        self.leds = [self.get_device("led" + str(i)) for i in range(8)]
 
     @kernel
     def run(self):
         self.core.reset()
 
         while True:
-            for led in self.leds:
-                led.pulse(250*ms)
+            with parallel:
+                for led in self.leds:
+                    led.pulse(250*ms)
+                for led in self.rleds:
+                    led.pulse(250*ms)
             t = now_mu()
             for led in self.leds:
                 at_mu(t)
                 led.pulse(500*ms)
+            for led in self.rleds:
+                at_mu(t)
+                led.pulse(500*ms)
             delay(250*ms)
diff --git a/artiq/gateware/rtio/__init__.py b/artiq/gateware/rtio/__init__.py
index 57fcbbc38..fd78f6430 100644
--- a/artiq/gateware/rtio/__init__.py
+++ b/artiq/gateware/rtio/__init__.py
@@ -1,4 +1,4 @@
-from artiq.gateware.rtio.cri import KernelInitiator
+from artiq.gateware.rtio.cri import KernelInitiator, CRIDecoder
 from artiq.gateware.rtio.core import Channel, LogChannel, Core
 from artiq.gateware.rtio.analyzer import Analyzer
 from artiq.gateware.rtio.moninj import MonInj
diff --git a/artiq/gateware/rtio/cri.py b/artiq/gateware/rtio/cri.py
index 62e354a39..ed19612d2 100644
--- a/artiq/gateware/rtio/cri.py
+++ b/artiq/gateware/rtio/cri.py
@@ -116,3 +116,34 @@ class KernelInitiator(Module, AutoCSR):
             self.o_data.we.eq(self.o_timestamp.re),
         ]
         self.sync += If(self.counter_update.re, self.counter.status.eq(self.cri.counter))
+
+
+class CRIDecoder(Module):
+    def __init__(self, slaves=2, master=None):
+        if isinstance(slaves, int):
+            slaves = [Interface() for _ in range(slaves)]
+        if master is None:
+            master = Interface()
+        self.slaves = slaves
+        self.master = master
+
+        # # #
+
+        selected = Signal(8)
+        self.sync += selected.eq(self.master.chan_sel[16:])
+
+        # master -> slave
+        for n, slave in enumerate(slaves):
+            for name, size, direction in _layout:
+                if direction == DIR_M_TO_S and name != "cmd":
+                    self.comb += getattr(slave, name).eq(getattr(master, name))
+            self.comb += If(selected == n, slave.cmd.eq(master.cmd))
+
+        # slave -> master
+        cases = dict()
+        for n, slave in enumerate(slaves):
+            cases[n] = []
+            for name, size, direction in _layout:
+                if direction == DIR_S_TO_M:
+                    cases[n].append(getattr(master, name).eq(getattr(slave, name)))
+        self.comb += Case(selected, cases)
diff --git a/artiq/gateware/targets/kc705_drtio_master.py b/artiq/gateware/targets/kc705_drtio_master.py
index fb6bfadc8..f49373bbc 100755
--- a/artiq/gateware/targets/kc705_drtio_master.py
+++ b/artiq/gateware/targets/kc705_drtio_master.py
@@ -9,6 +9,7 @@ from misoc.integration.builder import builder_args, builder_argdict
 
 from artiq.gateware.soc import AMPSoC, build_artiq_soc
 from artiq.gateware import rtio
+from artiq.gateware.rtio.phy import ttl_simple
 from artiq.gateware.drtio.transceiver import gtx_7series
 from artiq.gateware.drtio import DRTIOMaster
 from artiq import __version__ as artiq_version
@@ -42,10 +43,23 @@ class Master(MiniSoC, AMPSoC):
             sys_clk_freq=self.clk_freq,
             clock_div2=True)
         self.submodules.drtio = DRTIOMaster(self.transceiver)
-        self.submodules.rtio = rtio.KernelInitiator(self.drtio.cri)
-        self.register_kernel_cpu_csrdevice("rtio")
         self.csr_devices.append("drtio")
 
+        rtio_channels = []
+        for i in range(8):
+            phy = ttl_simple.Output(platform.request("user_led", i))
+            self.submodules += phy
+            rtio_channels.append(rtio.Channel.from_phy(phy))
+        for sma in "user_sma_gpio_p", "user_sma_gpio_n":
+            phy = ttl_simple.Inout(platform.request(sma))
+            self.submodules += phy
+            rtio_channels.append(rtio.Channel.from_phy(phy))
+        self.submodules.rtio_core = rtio.Core(rtio_channels)
+
+        self.submodules.cridec = rtio.CRIDecoder([self.drtio.cri, self.rtio_core.cri])
+        self.submodules.rtio = rtio.KernelInitiator(self.cridec.cri)
+        self.register_kernel_cpu_csrdevice("rtio")
+
 
 def main():
     parser = argparse.ArgumentParser(

From 4e1b4977420a3804d07fa78c20ea660ca326350b Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Mon, 28 Nov 2016 14:34:58 +0800
Subject: [PATCH 119/134] drtio: typo

---
 artiq/gateware/rtio/cri.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/artiq/gateware/rtio/cri.py b/artiq/gateware/rtio/cri.py
index ed19612d2..487ce2996 100644
--- a/artiq/gateware/rtio/cri.py
+++ b/artiq/gateware/rtio/cri.py
@@ -134,7 +134,7 @@ class CRIDecoder(Module):
 
         # master -> slave
         for n, slave in enumerate(slaves):
-            for name, size, direction in _layout:
+            for name, size, direction in layout:
                 if direction == DIR_M_TO_S and name != "cmd":
                     self.comb += getattr(slave, name).eq(getattr(master, name))
             self.comb += If(selected == n, slave.cmd.eq(master.cmd))

From 5460202220f9c272b5609090da907134b3fac4b9 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Mon, 28 Nov 2016 14:35:21 +0800
Subject: [PATCH 120/134] drtio: typo

---
 artiq/gateware/rtio/cri.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/artiq/gateware/rtio/cri.py b/artiq/gateware/rtio/cri.py
index 487ce2996..ca71e224c 100644
--- a/artiq/gateware/rtio/cri.py
+++ b/artiq/gateware/rtio/cri.py
@@ -143,7 +143,7 @@ class CRIDecoder(Module):
         cases = dict()
         for n, slave in enumerate(slaves):
             cases[n] = []
-            for name, size, direction in _layout:
+            for name, size, direction in layout:
                 if direction == DIR_S_TO_M:
                     cases[n].append(getattr(master, name).eq(getattr(slave, name)))
         self.comb += Case(selected, cases)

From 9fdd29ddae84940a43ff5554e68037eacc22347f Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Mon, 28 Nov 2016 14:36:18 +0800
Subject: [PATCH 121/134] drtio: connect KernelInitiator correctly

---
 artiq/gateware/targets/kc705_drtio_master.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/artiq/gateware/targets/kc705_drtio_master.py b/artiq/gateware/targets/kc705_drtio_master.py
index f49373bbc..d74b8614f 100755
--- a/artiq/gateware/targets/kc705_drtio_master.py
+++ b/artiq/gateware/targets/kc705_drtio_master.py
@@ -57,7 +57,7 @@ class Master(MiniSoC, AMPSoC):
         self.submodules.rtio_core = rtio.Core(rtio_channels)
 
         self.submodules.cridec = rtio.CRIDecoder([self.drtio.cri, self.rtio_core.cri])
-        self.submodules.rtio = rtio.KernelInitiator(self.cridec.cri)
+        self.submodules.rtio = rtio.KernelInitiator(self.cridec.master)
         self.register_kernel_cpu_csrdevice("rtio")
 
 

From 85f2467e2c0d4771a14c69ba9ff3f2591048a5b6 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Mon, 28 Nov 2016 15:01:46 +0800
Subject: [PATCH 122/134] rtio: fix RTIO/DRTIO timestamp resolution discrepancy

---
 artiq/gateware/rtio/core.py                  | 15 ++++-----------
 artiq/gateware/targets/kc705_drtio_master.py |  2 +-
 2 files changed, 5 insertions(+), 12 deletions(-)

diff --git a/artiq/gateware/rtio/core.py b/artiq/gateware/rtio/core.py
index 23fff3039..0f7ef5bed 100644
--- a/artiq/gateware/rtio/core.py
+++ b/artiq/gateware/rtio/core.py
@@ -265,17 +265,10 @@ class LogChannel:
 
 
 class Core(Module):
-    def __init__(self, channels, guard_io_cycles=20):
-        data_width = max(rtlink.get_data_width(c.interface)
-                         for c in channels)
-        address_width = max(rtlink.get_address_width(c.interface)
-                            for c in channels)
-        fine_ts_width = max(rtlink.get_fine_ts_width(c.interface)
-                            for c in channels)
-
-        self.data_width = data_width
-        self.address_width = address_width
-        self.fine_ts_width = fine_ts_width
+    def __init__(self, channels, fine_ts_width=None, guard_io_cycles=20):
+        if fine_ts_width is None:
+            fine_ts_width = max(rtlink.get_fine_ts_width(c.interface)
+                                for c in channels)
 
         self.cri = cri.Interface()
         self.comb += self.cri.arb_gnt.eq(1)
diff --git a/artiq/gateware/targets/kc705_drtio_master.py b/artiq/gateware/targets/kc705_drtio_master.py
index d74b8614f..f1de131cb 100755
--- a/artiq/gateware/targets/kc705_drtio_master.py
+++ b/artiq/gateware/targets/kc705_drtio_master.py
@@ -54,7 +54,7 @@ class Master(MiniSoC, AMPSoC):
             phy = ttl_simple.Inout(platform.request(sma))
             self.submodules += phy
             rtio_channels.append(rtio.Channel.from_phy(phy))
-        self.submodules.rtio_core = rtio.Core(rtio_channels)
+        self.submodules.rtio_core = rtio.Core(rtio_channels, 4)
 
         self.submodules.cridec = rtio.CRIDecoder([self.drtio.cri, self.rtio_core.cri])
         self.submodules.rtio = rtio.KernelInitiator(self.cridec.master)

From f4c6d6eb69a6969fd687106cf6dfe752c5e86285 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Mon, 28 Nov 2016 15:18:54 +0800
Subject: [PATCH 123/134] kc705_drtio_master: fix number of fine RTIO timestamp
 bits

---
 artiq/gateware/targets/kc705_drtio_master.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/artiq/gateware/targets/kc705_drtio_master.py b/artiq/gateware/targets/kc705_drtio_master.py
index f1de131cb..2d32bd0e3 100755
--- a/artiq/gateware/targets/kc705_drtio_master.py
+++ b/artiq/gateware/targets/kc705_drtio_master.py
@@ -54,7 +54,7 @@ class Master(MiniSoC, AMPSoC):
             phy = ttl_simple.Inout(platform.request(sma))
             self.submodules += phy
             rtio_channels.append(rtio.Channel.from_phy(phy))
-        self.submodules.rtio_core = rtio.Core(rtio_channels, 4)
+        self.submodules.rtio_core = rtio.Core(rtio_channels, 2)
 
         self.submodules.cridec = rtio.CRIDecoder([self.drtio.cri, self.rtio_core.cri])
         self.submodules.rtio = rtio.KernelInitiator(self.cridec.master)

From cf342eca6ee574cffb2894fdc95a66179af6b859 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Tue, 29 Nov 2016 10:44:27 +0800
Subject: [PATCH 124/134] kc705_drtio_master: fix number of fine RTIO timestamp
 bits

---
 artiq/gateware/targets/kc705_drtio_master.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/artiq/gateware/targets/kc705_drtio_master.py b/artiq/gateware/targets/kc705_drtio_master.py
index 2d32bd0e3..5dd5e192a 100755
--- a/artiq/gateware/targets/kc705_drtio_master.py
+++ b/artiq/gateware/targets/kc705_drtio_master.py
@@ -54,7 +54,7 @@ class Master(MiniSoC, AMPSoC):
             phy = ttl_simple.Inout(platform.request(sma))
             self.submodules += phy
             rtio_channels.append(rtio.Channel.from_phy(phy))
-        self.submodules.rtio_core = rtio.Core(rtio_channels, 2)
+        self.submodules.rtio_core = rtio.Core(rtio_channels, 3)
 
         self.submodules.cridec = rtio.CRIDecoder([self.drtio.cri, self.rtio_core.cri])
         self.submodules.rtio = rtio.KernelInitiator(self.cridec.master)

From cd3f68ba7668f00bc91b4560dc149820a635b926 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Wed, 30 Nov 2016 18:43:19 +0800
Subject: [PATCH 125/134] rtio: DMA core (untested)

---
 artiq/gateware/rtio/dma.py | 331 +++++++++++++++++++++++++++++++++++++
 1 file changed, 331 insertions(+)
 create mode 100644 artiq/gateware/rtio/dma.py

diff --git a/artiq/gateware/rtio/dma.py b/artiq/gateware/rtio/dma.py
new file mode 100644
index 000000000..31af3d430
--- /dev/null
+++ b/artiq/gateware/rtio/dma.py
@@ -0,0 +1,331 @@
+from migen import *
+from migen.genlib.record import Record, layout_len
+from migen.genlib.fsm import FSM
+from misoc.interconnect.csr import *
+from misoc.interconnect import stream, wishbone
+
+from artiq.gateware.rtio import cri
+
+
+class WishboneReader(Module):
+    def __init__(self, bus=None):
+        if bus is None:
+            bus = wishbone.Interface
+        self.bus = bus
+
+        aw = len(bus.adr)
+        dw = len(bus.dat_w)        
+        self.sink = stream.Endpoint(["address", aw])
+        self.source = stream.Endpoint(["data", dw])
+
+        # # #
+
+        bus_stb = Signal()
+        data_reg_loaded = Signal()
+
+        self.comb += [
+            bus_stb.eq(self.sink.stb & (~data_reg_loaded | self.source.ack)),
+            bus.cyc.eq(bus_stb),
+            bus.stb.eq(bus_stb),
+            bus.adr.eq(self.sink.address),
+            self.sink.ack.eq(bus.ack),
+            self.source.stb.eq(data_reg_loaded),
+        ]
+        self.sync += [
+            If(self.source.ack, data_reg_loaded.eq(0)),
+            If(bus.ack,
+                data_reg_loaded.eq(1),
+                self.source.data.eq(bus.dat_r),
+                self.source.eop.eq(self.sink.eop)
+            )
+        ]
+
+
+class DMAReader(Module, AutoCSR):
+    def __init__(self, membus, enable):
+        aw = len(membus.adr)
+        data_alignment = log2_int(len(membus.dat_w)//8)
+
+        self.submodules.wb_reader = WishboneReader(membus)
+        self.source = self.wb_reader.source
+
+        # All numbers in bytes
+        self.base_address = CSRStorage(aw + data_alignment,
+                                       alignment_bits=data_alignment)
+        self.last_address = CSRStorage(aw + data_alignment,
+                                       alignment_bits=data_alignment)
+
+        # # #
+
+        enable_r = Signal()
+        address = self.wb_reader.sink
+        self.sync += [
+            enable_r.eq(enable),
+            If(enable & ~enable_r,
+                address.address.eq(self.base_address.storage),
+                address.eop.eq(0),
+                address.stb.eq(1)
+            ),
+            If(address.stb & address.ack,
+                If(address.eop,
+                    address.stb.eq(0)
+                ).Else(
+                    address.address.eq(address.address + 1),
+                    If(~enable | (address.address == self.last_address.storage),
+                        address.eop.eq(1)
+                    )
+                )
+            )
+        ]
+
+
+class RawSlicer(Module):
+    def __init__(self, in_size, out_size, granularity):
+        g = granularity
+
+        self.sink = stream.Endpoint([("data", in_size*g)])
+        self.source = Signal(out_size*g)
+        self.source_stb = Signal()
+        self.source_consume = Signal(max=out_size+1)
+
+        # # #
+
+        # worst-case buffer space required (when loading):
+        #          <data being shifted out>   <new incoming word>   <EOP marker>   
+        buf_size =       out_size - 1       +       in_size       +      1
+        buf = Signal(buf_size*g)
+        self.comb += self.source.eq(buf[:out_size])
+
+        level = Signal(max=buf_size+1)
+        next_level = Signal(max=buf_size+1)
+        self.sync += level.eq(next_level)
+        self.comb += next_level.eq(level)
+
+        load_buf = Signal()
+        shift_buf = Signal()
+
+        self.sync += [
+            If(load_buf, Case(level,
+                # note how the MSBs of the buffer are set to 0
+                # (including the EOP marker position) 
+                {i: buf[i*g:].eq(self.sink.data)
+                 for i in range(out_size)})),
+            If(shift_buf, buf.eq(buf >> self.source_consume*g))
+        ]
+
+        fsm = FSM(reset_state="FETCH")
+        self.submodules += fsm
+
+        fsm.act("FETCH",
+            self.sink.ack.eq(1),
+            load_buf.eq(1),
+            If(self.sink.stb,
+                If(self.sink.eop,
+                    # insert <granularity> bits of 0 to mark EOP
+                    next_level.eq(level + in_size + 1)
+                ).Else(
+                    next_level.eq(level + in_size)
+                )
+            ),
+            If(next_level >= out_size, NextState("OUTPUT"))
+        )
+        fsm.act("OUTPUT",
+            self.source_stb.eq(1),
+            shift_buf.eq(1),
+            next_level.eq(level - self.source_consume),
+            If(next_level < out_size, NextState("FETCH"))
+        )
+
+
+record_layout = [
+    ("length", 8),  # of whole record (header+data)
+    ("channel", 24),
+    ("timestamp", 64),
+    ("address", 16),
+    ("data", 512)  # variable length
+]
+
+
+class RecordConverter(Module):
+    def __init__(self, stream_slicer):
+        self.source = stream.Endpoint(record_layout)
+
+        hdrlen = layout_len(header_layout) - 512
+        record_raw = Record(record_layout)
+        self.comb += [
+            record_raw.raw_bits().eq(stream_slicer.source),
+
+            record.channel.eq(record_raw.channel),
+            record.timestamp.eq(record_raw.timestamp),
+            record.address.eq(record_raw.address),
+            Case(record_raw.length,
+                 {hdrlen+i*8: self.cri.o_data.eq(header.data[:])
+                  for i in range(512//8)}),
+
+            self.source.stb.eq(stream_slicer.source_stb),
+            self.source.eop.eq(record_raw.length == 0),
+            If(self.source.ack,
+                If(record_raw.length == 0,
+                    stream_slicer.source_consume.eq(1)
+                ).Else(
+                    stream_slicer.source_consume.eq(record_raw.length)
+                )
+            )
+        ]
+
+
+class RecordSlicer(Module):
+    def __init__(self, in_size):
+        self.submodules.raw_slicer = RawSlicer(
+            in_size, layout_len(record_layout)//8, 8)
+        self.submodules.record_converter = RecordConverter(self.raw_slicer)
+        self.sink = self.raw_slicer.sink
+        self.source = self.record_converter.source
+
+
+class TimeOffset(Module, AutoCSR):
+    def __init__(self):
+        self.time_offset = CSRStorage(64)
+        self.source = stream.Endpoint(record_layout)
+        self.sink = stream.Endpoint(record_layout)
+
+        # # #
+
+        pipe_ce = Signal()
+        self.sync += \
+            If(pipe_ce,
+                self.source.payload.connect(self.sink.payload,
+                                            exclude={"timestamp"}),
+                self.source.payload.timestamp.eq(self.sink.payload.timestamp
+                                                 + self.time_offset.storage),
+                self.source.stb.eq(self.sink.stb)
+            )
+        self.comb += [
+            self.pipe_ce.eq(self.source.ack | ~self.source.stb),
+            self.sink.ack.eq(self.pipe_ce)
+        ]
+
+
+class CRIMaster(Module, AutoCSR):
+    def __init__(self):
+        self.arb_req = CSRStorage()
+        self.arb_gnt = CSRStatus()
+
+        self.error_status = CSRStatus(5)  # same encoding as RTIO status
+        self.error_underflow_reset = CSR()
+        self.error_sequence_error_reset = CSR()
+        self.error_collision_reset = CSR()
+        self.error_busy_reset = CSR()
+
+        self.error_channel = CSRStatus(24)
+        self.error_timestamp = CSRStatus(64)
+        self.error_address = CSRStatus(16)
+
+        self.sink = stream.Endpoint(record_layout)
+        self.cri = cri.Interface()
+        self.busy = Signal()
+
+        # # #
+
+        self.comb += [
+            self.cri.arb_req.eq(self.arb_req.storage),
+            self.arb_gnt.status.eq(self.cri.arb_gnt)
+        ]
+
+        error_set = Signal(4)
+        for i, rcsr in enumerate([self.error_underflow_reset, self.error_sequence_error_reset,
+                                  self.error_collision_reset, self.error_busy_reset]):
+            # bit 0 is RTIO wait and always 0 here
+            bit = i + 1
+            self.sync += [
+                If(error_set[i],
+                    self.error_status[bit].eq(1),
+                    self.error_channel.status.eq(self.sink.channel),
+                    self.error_timestamp.status.eq(self.sink.timestamp),
+                    self.error_address.status.eq(self.sink.address)
+                ),
+                If(rcsr.re, self.error_status[bit].eq(0))
+            ]
+
+        self.comb += [
+            self.cri.chan_sel.eq(self.sink.channel),
+            self.cri.o_timestamp.eq(self.sink.timestamp),
+            self.cri.o_address.eq(self.sink.address),
+        ]
+
+        fsm = FSM(reset_state="IDLE")
+        self.submodules += fsm
+
+        fsm.act("IDLE",
+            If(self.error_status.status == 0,
+                If(self.sink.stb, NextState("WRITE"))
+            ).Else(
+                # discard all data until errors are acked
+                self.sink.ack.eq(1)
+            )
+        )
+        fsm.act("WRITE",
+            self.busy.eq(1),
+            self.cri.cmd.eq(cri.commands["write"]),
+            NextState("CHECK_STATE")
+        )
+        fsm.act("CHECK_STATE",
+            self.busy.eq(1),
+            If(~self.cri.o_status,
+                self.sink.ack.eq(1),
+                NextState("IDLE")
+            ),
+            If(self.cri.o_status[1], NextState("UNDERFLOW")),
+            If(self.cri.o_status[2], NextState("SEQUENCE_ERROR")),
+            If(self.cri.o_status[3], NextState("COLLISION")),
+            If(self.cri.o_status[4], NextState("BUSY"))
+        )
+        for n, name in enumerate(["UNDERFLOW", "SEQUENCE_ERROR",
+                                  "COLLISION", "BUSY"]):
+            fsm.act(name,
+                self.busy.eq(1),
+                error_set.eq(1 << n),
+                self.cri.cmd.eq(cri.commands["o_" + name.lower() + "_reset"]),
+                self.sink.ack.eq(1),
+                NextState("IDLE")
+            )
+
+
+class DMA(Module):
+    def __init__(self, membus):
+        # shutdown procedure: set enable to 0, wait until busy=0
+        self.enable = CSRStorage()
+        self.busy = CSRStatus()
+
+        self.submodules.dma = DMAReader(membus, self.enable.storage)
+        self.submodules.slicer = RecordSlicer(len(membus.dat_w))
+        self.submodules.time_offset = TimeOffset()
+        self.submodules.cri_master = CRIMaster()
+
+        self.comb += [
+            self.dma.source.connect(self.slicer.sink),
+            self.slicer.source.connect(self.time_offset.sink),
+            self.time_offset.source.connect(self.cri_master.sink)
+        ]
+
+        fsm = FSM(reset_state="IDLE")
+        self.submodules += fsm
+
+        fsm.act("IDLE",
+            If(self.enable.storage, NextState("FLOWING"))
+        )
+        fsm.act("FLOWING",
+            self.busy.status.eq(1),
+            If(self.cri_master.sink.stb & self.cri_master.sink.ack & self.cri_master.sink.eop,
+                NextState("WAIT_CRI_MASTER")
+            )
+        )
+        fsm.act("WAIT_CRI_MASTER",
+            self.busy.status.eq(1),
+            If(~self.cri_master.busy, NextState("IDLE"))
+        )
+
+    def get_csrs(self):
+        return ([self.enable, self.busy] +
+                self.dma.get_csrs() + self.time_offset.get_csrs() +
+                self.cri_master.get_csrs())

From a3182430835785365ff6515a489dd22bd80e91b5 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Thu, 1 Dec 2016 15:41:43 +0800
Subject: [PATCH 126/134] rtio: CRI arbiter (untested)

---
 artiq/gateware/rtio/cri.py | 44 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 44 insertions(+)

diff --git a/artiq/gateware/rtio/cri.py b/artiq/gateware/rtio/cri.py
index ca71e224c..c76501584 100644
--- a/artiq/gateware/rtio/cri.py
+++ b/artiq/gateware/rtio/cri.py
@@ -147,3 +147,47 @@ class CRIDecoder(Module):
                 if direction == DIR_S_TO_M:
                     cases[n].append(getattr(master, name).eq(getattr(slave, name)))
         self.comb += Case(selected, cases)
+
+
+class CRIArbiter(Module):
+    def __init__(self, masters=2, slave=None):
+        if isinstance(masters, int):
+            masters = [Interface() for _ in range(masters)]
+        if slave is None:
+            slave = Interface()
+        self.masters = masters
+        self.slave = slave
+
+        # # #
+
+        selected = Signal(max=len(masters))
+
+        # mux master->slave signals
+        for name, size, direction in layout:
+            if direction == DIR_M_TO_S:
+                choices = Array(getattr(m, name) for m in masters)
+                self.comb += getattr(slave, name).eq(choices[selected])
+
+        # connect slave->master signals
+        for name, size, direction in layout:
+            if direction == DIR_S_TO_M:
+                source = getattr(slave, name)
+                for i, m in enumerate(masters):
+                    dest = getattr(m, name)
+                    if name == "arb_gnt":
+                        self.comb += dest.eq(source & (selected == i))
+                    else:
+                        self.comb += dest.eq(source)
+
+        # select master
+        self.sync += \
+            If(~slave.arb_req,
+                [If(m.arb_req, selected.eq(i)) for i, m in enumerate(masters)]
+            )
+
+
+class CRIInterconnectShared(Module):
+    def __init__(self, masters=2, slaves=2):
+        shared = Interface()
+        self.submodules.arbiter = CRIArbiter(masters, shared)
+        self.submodules.decoder = CRIDecoder(slaves, shared)

From 6c97a97d8c8c19758fea8f6693daee149d462b8b Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Thu, 1 Dec 2016 16:30:11 +0800
Subject: [PATCH 127/134] rtio: support single-master CRI arbiter

---
 artiq/gateware/rtio/cri.py | 45 ++++++++++++++++++++------------------
 1 file changed, 24 insertions(+), 21 deletions(-)

diff --git a/artiq/gateware/rtio/cri.py b/artiq/gateware/rtio/cri.py
index c76501584..e753ad390 100644
--- a/artiq/gateware/rtio/cri.py
+++ b/artiq/gateware/rtio/cri.py
@@ -160,30 +160,33 @@ class CRIArbiter(Module):
 
         # # #
 
-        selected = Signal(max=len(masters))
+        if len(masters) == 1:
+            self.comb += masters[0].connect(slave)
+        else:
+            selected = Signal(max=len(masters))
 
-        # mux master->slave signals
-        for name, size, direction in layout:
-            if direction == DIR_M_TO_S:
-                choices = Array(getattr(m, name) for m in masters)
-                self.comb += getattr(slave, name).eq(choices[selected])
+            # mux master->slave signals
+            for name, size, direction in layout:
+                if direction == DIR_M_TO_S:
+                    choices = Array(getattr(m, name) for m in masters)
+                    self.comb += getattr(slave, name).eq(choices[selected])
 
-        # connect slave->master signals
-        for name, size, direction in layout:
-            if direction == DIR_S_TO_M:
-                source = getattr(slave, name)
-                for i, m in enumerate(masters):
-                    dest = getattr(m, name)
-                    if name == "arb_gnt":
-                        self.comb += dest.eq(source & (selected == i))
-                    else:
-                        self.comb += dest.eq(source)
+            # connect slave->master signals
+            for name, size, direction in layout:
+                if direction == DIR_S_TO_M:
+                    source = getattr(slave, name)
+                    for i, m in enumerate(masters):
+                        dest = getattr(m, name)
+                        if name == "arb_gnt":
+                            self.comb += dest.eq(source & (selected == i))
+                        else:
+                            self.comb += dest.eq(source)
 
-        # select master
-        self.sync += \
-            If(~slave.arb_req,
-                [If(m.arb_req, selected.eq(i)) for i, m in enumerate(masters)]
-            )
+            # select master
+            self.sync += \
+                If(~slave.arb_req,
+                    [If(m.arb_req, selected.eq(i)) for i, m in enumerate(masters)]
+                )
 
 
 class CRIInterconnectShared(Module):

From 46dbc44c8f90d19410b86e293dbbfe06495f64ec Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Thu, 1 Dec 2016 16:30:29 +0800
Subject: [PATCH 128/134] rtio: export DMA and CRIInterconnectShared

---
 artiq/gateware/rtio/__init__.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/artiq/gateware/rtio/__init__.py b/artiq/gateware/rtio/__init__.py
index fd78f6430..18feec299 100644
--- a/artiq/gateware/rtio/__init__.py
+++ b/artiq/gateware/rtio/__init__.py
@@ -1,4 +1,5 @@
-from artiq.gateware.rtio.cri import KernelInitiator, CRIDecoder
+from artiq.gateware.rtio.cri import KernelInitiator, CRIInterconnectShared
 from artiq.gateware.rtio.core import Channel, LogChannel, Core
 from artiq.gateware.rtio.analyzer import Analyzer
 from artiq.gateware.rtio.moninj import MonInj
+from artiq.gateware.rtio.dma import DMA

From 7c59688a120178322728c1657d771c088221f330 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Thu, 1 Dec 2016 16:30:48 +0800
Subject: [PATCH 129/134] rtio: simple DMA fixes

---
 artiq/gateware/rtio/dma.py | 25 +++++++++++++------------
 1 file changed, 13 insertions(+), 12 deletions(-)

diff --git a/artiq/gateware/rtio/dma.py b/artiq/gateware/rtio/dma.py
index 31af3d430..210a352d5 100644
--- a/artiq/gateware/rtio/dma.py
+++ b/artiq/gateware/rtio/dma.py
@@ -15,8 +15,8 @@ class WishboneReader(Module):
 
         aw = len(bus.adr)
         dw = len(bus.dat_w)        
-        self.sink = stream.Endpoint(["address", aw])
-        self.source = stream.Endpoint(["data", dw])
+        self.sink = stream.Endpoint([("address", aw)])
+        self.source = stream.Endpoint([("data", dw)])
 
         # # #
 
@@ -150,16 +150,16 @@ class RecordConverter(Module):
     def __init__(self, stream_slicer):
         self.source = stream.Endpoint(record_layout)
 
-        hdrlen = layout_len(header_layout) - 512
+        hdrlen = layout_len(record_layout) - 512
         record_raw = Record(record_layout)
         self.comb += [
             record_raw.raw_bits().eq(stream_slicer.source),
 
-            record.channel.eq(record_raw.channel),
-            record.timestamp.eq(record_raw.timestamp),
-            record.address.eq(record_raw.address),
+            self.source.channel.eq(record_raw.channel),
+            self.source.timestamp.eq(record_raw.timestamp),
+            self.source.address.eq(record_raw.address),
             Case(record_raw.length,
-                 {hdrlen+i*8: self.cri.o_data.eq(header.data[:])
+                 {hdrlen+i*8: self.source.data.eq(record_raw.data[:])
                   for i in range(512//8)}),
 
             self.source.stb.eq(stream_slicer.source_stb),
@@ -195,14 +195,14 @@ class TimeOffset(Module, AutoCSR):
         self.sync += \
             If(pipe_ce,
                 self.source.payload.connect(self.sink.payload,
-                                            exclude={"timestamp"}),
+                                            leave_out={"timestamp"}),
                 self.source.payload.timestamp.eq(self.sink.payload.timestamp
                                                  + self.time_offset.storage),
                 self.source.stb.eq(self.sink.stb)
             )
         self.comb += [
-            self.pipe_ce.eq(self.source.ack | ~self.source.stb),
-            self.sink.ack.eq(self.pipe_ce)
+            pipe_ce.eq(self.source.ack | ~self.source.stb),
+            self.sink.ack.eq(pipe_ce)
         ]
 
 
@@ -239,12 +239,12 @@ class CRIMaster(Module, AutoCSR):
             bit = i + 1
             self.sync += [
                 If(error_set[i],
-                    self.error_status[bit].eq(1),
+                    self.error_status.status[bit].eq(1),
                     self.error_channel.status.eq(self.sink.channel),
                     self.error_timestamp.status.eq(self.sink.timestamp),
                     self.error_address.status.eq(self.sink.address)
                 ),
-                If(rcsr.re, self.error_status[bit].eq(0))
+                If(rcsr.re, self.error_status.status[bit].eq(0))
             ]
 
         self.comb += [
@@ -301,6 +301,7 @@ class DMA(Module):
         self.submodules.slicer = RecordSlicer(len(membus.dat_w))
         self.submodules.time_offset = TimeOffset()
         self.submodules.cri_master = CRIMaster()
+        self.cri = self.cri_master.cri
 
         self.comb += [
             self.dma.source.connect(self.slicer.sink),

From d4cb1eb998323ad5731659b8180bf04f28501707 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Thu, 1 Dec 2016 16:31:00 +0800
Subject: [PATCH 130/134] kc705: integrate DMA

---
 artiq/gateware/targets/kc705.py              | 16 +++++++++++-----
 artiq/gateware/targets/kc705_drtio_master.py | 15 ++++++++++-----
 2 files changed, 21 insertions(+), 10 deletions(-)

diff --git a/artiq/gateware/targets/kc705.py b/artiq/gateware/targets/kc705.py
index 7a9bde597..b06f4753c 100755
--- a/artiq/gateware/targets/kc705.py
+++ b/artiq/gateware/targets/kc705.py
@@ -101,10 +101,11 @@ _ams101_dac = [
 
 class _NIST_Ions(MiniSoC, AMPSoC):
     mem_map = {
-        "timer_kernel":  0x10000000, # (shadow @0x90000000)
-        "rtio":          0x20000000, # (shadow @0xa0000000)
-        "i2c":           0x30000000, # (shadow @0xb0000000)
-        "mailbox":       0x70000000  # (shadow @0xf0000000)
+        "timer_kernel":  0x10000000,
+        "rtio":          0x20000000,
+        "rtio_dma":      0x30000000,
+        "i2c":           0x50000000,
+        "mailbox":       0x70000000
     }
     mem_map.update(MiniSoC.mem_map)
 
@@ -143,8 +144,13 @@ class _NIST_Ions(MiniSoC, AMPSoC):
         self.submodules.rtio_crg = _RTIOCRG(self.platform, self.crg.cd_sys.clk)
         self.csr_devices.append("rtio_crg")
         self.submodules.rtio_core = rtio.Core(rtio_channels)
-        self.submodules.rtio = rtio.KernelInitiator(self.rtio_core.cri)
+        self.submodules.rtio = rtio.KernelInitiator()
+        self.submodules.rtio_dma = rtio.DMA(self.get_native_sdram_if())
         self.register_kernel_cpu_csrdevice("rtio")
+        self.register_kernel_cpu_csrdevice("rtio_dma")
+        self.submodules.cri_con = rtio.CRIInterconnectShared(
+            [self.rtio.cri, self.rtio_dma.cri],
+            [self.rtio_core.cri])
         self.submodules.rtio_moninj = rtio.MonInj(rtio_channels)
         self.csr_devices.append("rtio_moninj")
 
diff --git a/artiq/gateware/targets/kc705_drtio_master.py b/artiq/gateware/targets/kc705_drtio_master.py
index 5dd5e192a..342ef62bf 100755
--- a/artiq/gateware/targets/kc705_drtio_master.py
+++ b/artiq/gateware/targets/kc705_drtio_master.py
@@ -17,9 +17,10 @@ from artiq import __version__ as artiq_version
 
 class Master(MiniSoC, AMPSoC):
     mem_map = {
-        "timer_kernel":  0x10000000, # (shadow @0x90000000)
-        "rtio":          0x20000000, # (shadow @0xa0000000)
-        "mailbox":       0x70000000  # (shadow @0xf0000000)
+        "timer_kernel":  0x10000000,
+        "rtio":          0x20000000,
+        "rtio_dma":      0x30000000,
+        "mailbox":       0x70000000
     }
     mem_map.update(MiniSoC.mem_map)
 
@@ -56,9 +57,13 @@ class Master(MiniSoC, AMPSoC):
             rtio_channels.append(rtio.Channel.from_phy(phy))
         self.submodules.rtio_core = rtio.Core(rtio_channels, 3)
 
-        self.submodules.cridec = rtio.CRIDecoder([self.drtio.cri, self.rtio_core.cri])
-        self.submodules.rtio = rtio.KernelInitiator(self.cridec.master)
+        self.submodules.rtio = rtio.KernelInitiator()
+        self.submodules.rtio_dma = rtio.DMA(self.get_native_sdram_if())
         self.register_kernel_cpu_csrdevice("rtio")
+        self.register_kernel_cpu_csrdevice("rtio_dma")
+        self.submodules.cri_con = rtio.CRIInterconnectShared(
+            [self.rtio.cri, self.rtio_dma.cri],
+            [self.drtio.cri, self.rtio_core.cri])
 
 
 def main():

From 3931d8097bdd9e619583bf56a318efee8f2d4e39 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Thu, 1 Dec 2016 16:43:46 +0800
Subject: [PATCH 131/134] rtio: fix DMA TimeOffset stream.connect

---
 artiq/gateware/rtio/dma.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/artiq/gateware/rtio/dma.py b/artiq/gateware/rtio/dma.py
index 210a352d5..244e0be5c 100644
--- a/artiq/gateware/rtio/dma.py
+++ b/artiq/gateware/rtio/dma.py
@@ -194,8 +194,8 @@ class TimeOffset(Module, AutoCSR):
         pipe_ce = Signal()
         self.sync += \
             If(pipe_ce,
-                self.source.payload.connect(self.sink.payload,
-                                            leave_out={"timestamp"}),
+                self.sink.payload.connect(self.source.payload,
+                                          leave_out={"timestamp"}),
                 self.source.payload.timestamp.eq(self.sink.payload.timestamp
                                                  + self.time_offset.storage),
                 self.source.stb.eq(self.sink.stb)

From 6353f6d5909ec4d04450798d88c6cbe70de6ddfc Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Fri, 2 Dec 2016 17:22:22 +0800
Subject: [PATCH 132/134] drtio: support different configurations and speeds

---
 .../gateware/drtio/transceiver/gtx_7series.py | 17 ++++--
 artiq/gateware/targets/kc705_drtio_master.py  | 56 ++++++++++++++++---
 .../gateware/targets/kc705_drtio_satellite.py | 56 ++++++++++++++++---
 3 files changed, 107 insertions(+), 22 deletions(-)

diff --git a/artiq/gateware/drtio/transceiver/gtx_7series.py b/artiq/gateware/drtio/transceiver/gtx_7series.py
index 74ee4f1af..6c0a3e789 100644
--- a/artiq/gateware/drtio/transceiver/gtx_7series.py
+++ b/artiq/gateware/drtio/transceiver/gtx_7series.py
@@ -7,11 +7,10 @@ from misoc.interconnect.csr import *
 from artiq.gateware.drtio.transceiver.gtx_7series_init import *
 
 
-class GTX_1000BASE_BX10(Module):
-    rtio_clk_freq = 62.5e6
-
-    # The transceiver clock on clock_pads must be 62.5MHz
-    # when clock_div2=False, and 125MHz when clock_div2=True.
+class GTX_20X(Module):
+    # The transceiver clock on clock_pads must be at the RTIO clock
+    # frequency when clock_div2=False, and 2x that frequency when
+    # clock_div2=True.
     def __init__(self, clock_pads, tx_pads, rx_pads, sys_clk_freq,
                  clock_div2=False):
         self.submodules.encoder = ClockDomainsRenamer("rtio")(
@@ -192,6 +191,14 @@ class GTX_1000BASE_BX10(Module):
         ]
 
 
+class GTX_1000BASE_BX10(GTX_20X):
+    rtio_clk_freq = 62.5e6
+
+
+class GTX_3G(GTX_20X):
+    rtio_clk_freq = 150e6
+
+
 class RXSynchronizer(Module, AutoCSR):
     """Delays the data received in the rtio_rx by a configurable amount
     so that it meets s/h in the rtio domain, and recapture it in the rtio
diff --git a/artiq/gateware/targets/kc705_drtio_master.py b/artiq/gateware/targets/kc705_drtio_master.py
index 342ef62bf..8fb5cd1a4 100755
--- a/artiq/gateware/targets/kc705_drtio_master.py
+++ b/artiq/gateware/targets/kc705_drtio_master.py
@@ -3,6 +3,7 @@
 import argparse
 
 from migen import *
+from migen.build.generic_platform import *
 
 from misoc.targets.kc705 import MiniSoC, soc_kc705_args, soc_kc705_argdict
 from misoc.integration.builder import builder_args, builder_argdict
@@ -15,6 +16,14 @@ from artiq.gateware.drtio import DRTIOMaster
 from artiq import __version__ as artiq_version
 
 
+fmc_clock_io = [
+    ("ad9154_refclk", 0,
+        Subsignal("p", Pins("HPC:GBTCLK0_M2C_P")),
+        Subsignal("n", Pins("HPC:GBTCLK0_M2C_N")),
+    )
+]
+
+
 class Master(MiniSoC, AMPSoC):
     mem_map = {
         "timer_kernel":  0x10000000,
@@ -24,7 +33,7 @@ class Master(MiniSoC, AMPSoC):
     }
     mem_map.update(MiniSoC.mem_map)
 
-    def __init__(self, **kwargs):
+    def __init__(self, cfg, medium, **kwargs):
         MiniSoC.__init__(self,
                          cpu_type="or1k",
                          sdram_controller_type="minicon",
@@ -36,13 +45,36 @@ class Master(MiniSoC, AMPSoC):
 
         platform = self.platform
 
-        self.comb += platform.request("sfp_tx_disable_n").eq(1)
-        self.submodules.transceiver = gtx_7series.GTX_1000BASE_BX10(
-            clock_pads=platform.request("sgmii_clock"),
-            tx_pads=platform.request("sfp_tx"),
-            rx_pads=platform.request("sfp_rx"),
-            sys_clk_freq=self.clk_freq,
-            clock_div2=True)
+        if medium == "sfp":
+            self.comb += platform.request("sfp_tx_disable_n").eq(1)
+            tx_pads = platform.request("sfp_tx")
+            rx_pads = platform.request("sfp_rx")
+        elif medium == "sma":
+            tx_pads = platform.request("user_sma_mgt_tx")
+            rx_pads = platform.request("user_sma_mgt_rx")
+        else:
+            raise ValueError
+
+        if cfg == "simple_gbe":
+            # GTX_1000BASE_BX10 Ethernet compatible, 62.5MHz RTIO clock
+            # simple TTLs
+            self.submodules.transceiver = gtx_7series.GTX_1000BASE_BX10(
+                clock_pads=platform.request("sgmii_clock"),
+                tx_pads=tx_pads,
+                rx_pads=rx_pads,
+                sys_clk_freq=self.clk_freq,
+                clock_div2=True)
+        elif cfg == "sawg_3g":
+            # 3Gb link, 150MHz RTIO clock
+            # with SAWG on local RTIO and AD9154-FMC-EBZ
+            platform.register_extension(fmc_clock_io)
+            self.submodules.transceiver = gtx_7series.GTX_3G(
+                clock_pads=platform.request("ad9154_refclk"),
+                tx_pads=tx_pads,
+                rx_pads=rx_pads,
+                sys_clk_freq=self.clk_freq)
+        else:
+            raise ValueError
         self.submodules.drtio = DRTIOMaster(self.transceiver)
         self.csr_devices.append("drtio")
 
@@ -71,9 +103,15 @@ def main():
         description="ARTIQ with DRTIO on KC705 - Master")
     builder_args(parser)
     soc_kc705_args(parser)
+    parser.add_argument("-c", "--config", default="simple_gbe",
+                        help="configuration: simple_gbe/sawg_3g "
+                             "(default: %(default)s)")
+    parser.add_argument("--medium", default="sfp",
+                        help="medium to use for transceiver link: sfp/sma "
+                             "(default: %(default)s)")
     args = parser.parse_args()
 
-    soc = Master(**soc_kc705_argdict(args))
+    soc = Master(args.config, args.medium, **soc_kc705_argdict(args))
     build_artiq_soc(soc, builder_argdict(args))
 
 
diff --git a/artiq/gateware/targets/kc705_drtio_satellite.py b/artiq/gateware/targets/kc705_drtio_satellite.py
index 32795634f..7a882ab94 100755
--- a/artiq/gateware/targets/kc705_drtio_satellite.py
+++ b/artiq/gateware/targets/kc705_drtio_satellite.py
@@ -1,6 +1,7 @@
 import argparse
 
 from migen import *
+from migen.build.generic_platform import *
 from migen.build.platforms import kc705
 
 from misoc.cores.i2c import *
@@ -12,6 +13,7 @@ from artiq.gateware.drtio.transceiver import gtx_7series
 from artiq.gateware.drtio import DRTIOSatellite
 
 
+# TODO: parameters for sawg_3g
 def get_i2c_program(sys_clk_freq):
     # NOTE: the logical parameters DO NOT MAP to physical values written
     # into registers. They have to be mapped; see the datasheet.
@@ -111,8 +113,16 @@ class Si5324ResetClock(Module):
             )
 
 
+fmc_clock_io = [
+    ("ad9154_refclk", 0,
+        Subsignal("p", Pins("HPC:GBTCLK0_M2C_P")),
+        Subsignal("n", Pins("HPC:GBTCLK0_M2C_N")),
+    )
+]
+
+
 class Satellite(Module):
-    def __init__(self, toolchain="vivado"):
+    def __init__(self, cfg, medium, toolchain):
         self.platform = platform = kc705.Platform(toolchain=toolchain)
 
         rtio_channels = []
@@ -141,12 +151,36 @@ class Satellite(Module):
             sequencer.reset.eq(si5324_reset_clock.si5324_not_ready)
         ]
 
-        self.comb += platform.request("sfp_tx_disable_n").eq(1)
-        self.submodules.transceiver = gtx_7series.GTX_1000BASE_BX10(
-            clock_pads=platform.request("si5324_clkout"),
-            tx_pads=platform.request("sfp_tx"),
-            rx_pads=platform.request("sfp_rx"),
-            sys_clk_freq=sys_clk_freq)
+        if medium == "sfp":
+            self.comb += platform.request("sfp_tx_disable_n").eq(1)
+            tx_pads = platform.request("sfp_tx")
+            rx_pads = platform.request("sfp_rx")
+        elif medium == "sma":
+            tx_pads = platform.request("user_sma_mgt_tx")
+            rx_pads = platform.request("user_sma_mgt_rx")
+        else:
+            raise ValueError
+
+        if cfg == "simple_gbe":
+            # GTX_1000BASE_BX10 Ethernet compatible, 62.5MHz RTIO clock
+            # simple TTLs
+            self.submodules.transceiver = gtx_7series.GTX_1000BASE_BX10(
+                clock_pads=platform.request("sgmii_clock"),
+                tx_pads=tx_pads,
+                rx_pads=rx_pads,
+                sys_clk_freq=sys_clk_freq,
+                clock_div2=True)
+        elif cfg == "sawg_3g":
+            # 3Gb link, 150MHz RTIO clock
+            # with SAWG on local RTIO and AD9154-FMC-EBZ
+            platform.register_extension(fmc_clock_io)
+            self.submodules.transceiver = gtx_7series.GTX_3G(
+                clock_pads=platform.request("ad9154_refclk"),
+                tx_pads=tx_pads,
+                rx_pads=rx_pads,
+                sys_clk_freq=sys_clk_freq)
+        else:
+            raise ValueError
         self.submodules.rx_synchronizer = gtx_7series.RXSynchronizer(
             self.transceiver.rtio_clk_freq)
         self.submodules.drtio = DRTIOSatellite(
@@ -163,9 +197,15 @@ def main():
     parser.add_argument("--output-dir", default="drtiosat_kc705",
                         help="output directory for generated "
                              "source files and binaries")
+    parser.add_argument("-c", "--config", default="simple_gbe",
+                        help="configuration: simple_gbe/sawg_3g "
+                             "(default: %(default)s)")
+    parser.add_argument("--medium", default="sfp",
+                        help="medium to use for transceiver link: sfp/sma "
+                             "(default: %(default)s)")
     args = parser.parse_args()
 
-    top = Satellite(args.toolchain)
+    top = Satellite(args.config, args.medium, args.toolchain)
     top.build(build_dir=args.output_dir)
 
 if __name__ == "__main__":

From 4b97b9f8ce1d55e9b66f489f2ded235d7fbdf392 Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Sat, 3 Dec 2016 22:17:29 +0800
Subject: [PATCH 133/134] drtio: add clock constraints

---
 artiq/gateware/drtio/transceiver/gtx_7series.py | 8 ++++++--
 artiq/gateware/targets/kc705_drtio_master.py    | 2 ++
 artiq/gateware/targets/kc705_drtio_satellite.py | 2 ++
 3 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/artiq/gateware/drtio/transceiver/gtx_7series.py b/artiq/gateware/drtio/transceiver/gtx_7series.py
index 6c0a3e789..2c5897a91 100644
--- a/artiq/gateware/drtio/transceiver/gtx_7series.py
+++ b/artiq/gateware/drtio/transceiver/gtx_7series.py
@@ -11,8 +11,9 @@ class GTX_20X(Module):
     # The transceiver clock on clock_pads must be at the RTIO clock
     # frequency when clock_div2=False, and 2x that frequency when
     # clock_div2=True.
-    def __init__(self, clock_pads, tx_pads, rx_pads, sys_clk_freq,
-                 clock_div2=False):
+    def __init__(self, platform,
+                 clock_pads, tx_pads, rx_pads,
+                 sys_clk_freq, clock_div2=False):
         self.submodules.encoder = ClockDomainsRenamer("rtio")(
             Encoder(2, True))
         self.decoders = [ClockDomainsRenamer("rtio_rx")(
@@ -175,6 +176,9 @@ class GTX_20X(Module):
             Instance("BUFG", i_I=rxoutclk, o_O=self.cd_rtio_rx.clk),
             AsyncResetSynchronizer(self.cd_rtio_rx, rx_reset_deglitched)
         ]
+        platform.add_period_constraint(txoutclk, 1e9/self.rtio_clk_freq)
+        platform.add_period_constraint(rxoutclk, 1e9/self.rtio_clk_freq)
+        platform.add_false_path_constraints(txoutclk, rxoutclk)
 
         self.comb += [
             txdata.eq(Cat(self.encoder.output[0], self.encoder.output[1])),
diff --git a/artiq/gateware/targets/kc705_drtio_master.py b/artiq/gateware/targets/kc705_drtio_master.py
index 8fb5cd1a4..4f0ed088b 100755
--- a/artiq/gateware/targets/kc705_drtio_master.py
+++ b/artiq/gateware/targets/kc705_drtio_master.py
@@ -59,6 +59,7 @@ class Master(MiniSoC, AMPSoC):
             # GTX_1000BASE_BX10 Ethernet compatible, 62.5MHz RTIO clock
             # simple TTLs
             self.submodules.transceiver = gtx_7series.GTX_1000BASE_BX10(
+                platform=platform,
                 clock_pads=platform.request("sgmii_clock"),
                 tx_pads=tx_pads,
                 rx_pads=rx_pads,
@@ -69,6 +70,7 @@ class Master(MiniSoC, AMPSoC):
             # with SAWG on local RTIO and AD9154-FMC-EBZ
             platform.register_extension(fmc_clock_io)
             self.submodules.transceiver = gtx_7series.GTX_3G(
+                platform=platform,
                 clock_pads=platform.request("ad9154_refclk"),
                 tx_pads=tx_pads,
                 rx_pads=rx_pads,
diff --git a/artiq/gateware/targets/kc705_drtio_satellite.py b/artiq/gateware/targets/kc705_drtio_satellite.py
index 7a882ab94..f331c6dca 100755
--- a/artiq/gateware/targets/kc705_drtio_satellite.py
+++ b/artiq/gateware/targets/kc705_drtio_satellite.py
@@ -165,6 +165,7 @@ class Satellite(Module):
             # GTX_1000BASE_BX10 Ethernet compatible, 62.5MHz RTIO clock
             # simple TTLs
             self.submodules.transceiver = gtx_7series.GTX_1000BASE_BX10(
+                platform=platform,
                 clock_pads=platform.request("sgmii_clock"),
                 tx_pads=tx_pads,
                 rx_pads=rx_pads,
@@ -175,6 +176,7 @@ class Satellite(Module):
             # with SAWG on local RTIO and AD9154-FMC-EBZ
             platform.register_extension(fmc_clock_io)
             self.submodules.transceiver = gtx_7series.GTX_3G(
+                platform=platform,
                 clock_pads=platform.request("ad9154_refclk"),
                 tx_pads=tx_pads,
                 rx_pads=rx_pads,

From 5d145ff912d268e9ffe205ef64915134ad2a669e Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Sat, 3 Dec 2016 23:03:01 +0800
Subject: [PATCH 134/134] drtio: add false paths between sys and transceiver
 clocks

---
 .../gateware/drtio/transceiver/gtx_7series.py | 23 +++++++++----------
 artiq/gateware/targets/kc705_drtio_master.py  |  9 ++++++--
 .../gateware/targets/kc705_drtio_satellite.py | 10 ++++++--
 3 files changed, 26 insertions(+), 16 deletions(-)

diff --git a/artiq/gateware/drtio/transceiver/gtx_7series.py b/artiq/gateware/drtio/transceiver/gtx_7series.py
index 2c5897a91..958e1d541 100644
--- a/artiq/gateware/drtio/transceiver/gtx_7series.py
+++ b/artiq/gateware/drtio/transceiver/gtx_7series.py
@@ -11,9 +11,8 @@ class GTX_20X(Module):
     # The transceiver clock on clock_pads must be at the RTIO clock
     # frequency when clock_div2=False, and 2x that frequency when
     # clock_div2=True.
-    def __init__(self, platform,
-                 clock_pads, tx_pads, rx_pads,
-                 sys_clk_freq, clock_div2=False):
+    def __init__(self, clock_pads, tx_pads, rx_pads, sys_clk_freq,
+                 clock_div2=False):
         self.submodules.encoder = ClockDomainsRenamer("rtio")(
             Encoder(2, True))
         self.decoders = [ClockDomainsRenamer("rtio_rx")(
@@ -22,6 +21,11 @@ class GTX_20X(Module):
 
         self.rx_ready = Signal()
 
+        # transceiver direct clock outputs
+        # useful to specify clock constraints in a way palatable to Vivado
+        self.txoutclk = Signal()
+        self.rxoutclk = Signal()
+
         # # #
 
         refclk = Signal()
@@ -50,9 +54,7 @@ class GTX_20X(Module):
         self.comb += tx_init.cplllock.eq(cplllock), \
                      rx_init.cplllock.eq(cplllock)
 
-        txoutclk = Signal()
         txdata = Signal(20)
-        rxoutclk = Signal()
         rxdata = Signal(20)
         self.specials += \
             Instance("GTXE2_CHANNEL",
@@ -88,7 +90,7 @@ class GTX_20X(Module):
                 # TX clock
                 p_TXBUF_EN="FALSE",
                 p_TX_XCLK_SEL="TXUSR",
-                o_TXOUTCLK=txoutclk,
+                o_TXOUTCLK=self.txoutclk,
                 i_TXSYSCLKSEL=0b00,
                 i_TXOUTCLKSEL=0b11,
 
@@ -134,7 +136,7 @@ class GTX_20X(Module):
                 i_RXDDIEN=1,
                 i_RXSYSCLKSEL=0b00,
                 i_RXOUTCLKSEL=0b010,
-                o_RXOUTCLK=rxoutclk,
+                o_RXOUTCLK=self.rxoutclk,
                 i_RXUSRCLK=ClockSignal("rtio_rx"),
                 i_RXUSRCLK2=ClockSignal("rtio_rx"),
                 p_RXCDR_CFG=0x03000023FF10100020,
@@ -165,7 +167,7 @@ class GTX_20X(Module):
         self.sync += tx_reset_deglitched.eq(~tx_init.done)
         self.clock_domains.cd_rtio = ClockDomain()
         self.specials += [
-            Instance("BUFG", i_I=txoutclk, o_O=self.cd_rtio.clk),
+            Instance("BUFG", i_I=self.txoutclk, o_O=self.cd_rtio.clk),
             AsyncResetSynchronizer(self.cd_rtio, tx_reset_deglitched)
         ]
         rx_reset_deglitched = Signal()
@@ -173,12 +175,9 @@ class GTX_20X(Module):
         self.sync.rtio += rx_reset_deglitched.eq(~rx_init.done)
         self.clock_domains.cd_rtio_rx = ClockDomain()
         self.specials += [
-            Instance("BUFG", i_I=rxoutclk, o_O=self.cd_rtio_rx.clk),
+            Instance("BUFG", i_I=self.rxoutclk, o_O=self.cd_rtio_rx.clk),
             AsyncResetSynchronizer(self.cd_rtio_rx, rx_reset_deglitched)
         ]
-        platform.add_period_constraint(txoutclk, 1e9/self.rtio_clk_freq)
-        platform.add_period_constraint(rxoutclk, 1e9/self.rtio_clk_freq)
-        platform.add_false_path_constraints(txoutclk, rxoutclk)
 
         self.comb += [
             txdata.eq(Cat(self.encoder.output[0], self.encoder.output[1])),
diff --git a/artiq/gateware/targets/kc705_drtio_master.py b/artiq/gateware/targets/kc705_drtio_master.py
index 4f0ed088b..8459c2ff4 100755
--- a/artiq/gateware/targets/kc705_drtio_master.py
+++ b/artiq/gateware/targets/kc705_drtio_master.py
@@ -59,7 +59,6 @@ class Master(MiniSoC, AMPSoC):
             # GTX_1000BASE_BX10 Ethernet compatible, 62.5MHz RTIO clock
             # simple TTLs
             self.submodules.transceiver = gtx_7series.GTX_1000BASE_BX10(
-                platform=platform,
                 clock_pads=platform.request("sgmii_clock"),
                 tx_pads=tx_pads,
                 rx_pads=rx_pads,
@@ -70,7 +69,6 @@ class Master(MiniSoC, AMPSoC):
             # with SAWG on local RTIO and AD9154-FMC-EBZ
             platform.register_extension(fmc_clock_io)
             self.submodules.transceiver = gtx_7series.GTX_3G(
-                platform=platform,
                 clock_pads=platform.request("ad9154_refclk"),
                 tx_pads=tx_pads,
                 rx_pads=rx_pads,
@@ -80,6 +78,13 @@ class Master(MiniSoC, AMPSoC):
         self.submodules.drtio = DRTIOMaster(self.transceiver)
         self.csr_devices.append("drtio")
 
+        rtio_clk_period = 1e9/self.transceiver.rtio_clk_freq
+        platform.add_period_constraint(self.transceiver.txoutclk, rtio_clk_period)
+        platform.add_period_constraint(self.transceiver.rxoutclk, rtio_clk_period)
+        platform.add_false_path_constraints(
+            self.crg.cd_sys.clk,
+            self.transceiver.txoutclk, self.transceiver.rxoutclk)
+
         rtio_channels = []
         for i in range(8):
             phy = ttl_simple.Output(platform.request("user_led", i))
diff --git a/artiq/gateware/targets/kc705_drtio_satellite.py b/artiq/gateware/targets/kc705_drtio_satellite.py
index f331c6dca..ee4a2163a 100755
--- a/artiq/gateware/targets/kc705_drtio_satellite.py
+++ b/artiq/gateware/targets/kc705_drtio_satellite.py
@@ -165,7 +165,6 @@ class Satellite(Module):
             # GTX_1000BASE_BX10 Ethernet compatible, 62.5MHz RTIO clock
             # simple TTLs
             self.submodules.transceiver = gtx_7series.GTX_1000BASE_BX10(
-                platform=platform,
                 clock_pads=platform.request("sgmii_clock"),
                 tx_pads=tx_pads,
                 rx_pads=rx_pads,
@@ -176,7 +175,6 @@ class Satellite(Module):
             # with SAWG on local RTIO and AD9154-FMC-EBZ
             platform.register_extension(fmc_clock_io)
             self.submodules.transceiver = gtx_7series.GTX_3G(
-                platform=platform,
                 clock_pads=platform.request("ad9154_refclk"),
                 tx_pads=tx_pads,
                 rx_pads=rx_pads,
@@ -188,6 +186,14 @@ class Satellite(Module):
         self.submodules.drtio = DRTIOSatellite(
             self.transceiver, self.rx_synchronizer, rtio_channels)
 
+        rtio_clk_period = 1e9/self.transceiver.rtio_clk_freq
+        platform.add_period_constraint(self.transceiver.txoutclk, rtio_clk_period)
+        platform.add_period_constraint(self.transceiver.rxoutclk, rtio_clk_period)
+        platform.add_false_path_constraints(
+            sys_clock_pads,
+            self.transceiver.txoutclk, self.transceiver.rxoutclk)
+
+
     def build(self, *args, **kwargs):
         self.platform.build(self, *args, **kwargs)