cxp upconn fw: connect tx command to phy

upconn: refactor buffer printout into one function
upconn: refactor buffer write function
upconn: packet writter proto

src/gateware/cxp.py

@@ -1,16 +1,14 @@
 from migen import *
 from misoc.interconnect.csr import *
 from misoc.interconnect import stream
-from misoc.cores.liteeth_mini.mac.crc import LiteEthMACCRCEngine
 
 from cxp_downconn import CXP_DownConn
-from cxp_upconn import CXP_UpConn
+from cxp_upconn import CXP_UpConn_PHY
+from cxp_pipeline import *
 
 class CXP(Module, AutoCSR):
     def __init__(self, refclk, downconn_pads, upconn_pads, sys_clk_freq, debug_sma, pmod_pads):
-        self.submodules.crc = CXP_CRC(8)
+        self.submodules.upconn = UpConn_Interface(upconn_pads, sys_clk_freq, debug_sma, pmod_pads)
-
-        self.submodules.upconn = UpConn_Packets(upconn_pads, sys_clk_freq, debug_sma, pmod_pads)
 
         self.submodules.downconn = CXP_DownConn(refclk, downconn_pads, sys_clk_freq, debug_sma, pmod_pads)
         # TODO: support the option high speed upconn
@@ -18,125 +16,137 @@ class CXP(Module, AutoCSR):
         # TODO: add link layer
 
         
-class UpConn_Packets(Module, AutoCSR):
+class UpConn_Interface(Module, AutoCSR):
     def __init__(self, upconn_pads, sys_clk_freq, debug_sma, pmod_pads, fifos_depth=64):
-
+        self.clk_reset = CSRStorage(reset=1)
-        # increment after ack
-        # for CXP 2.0 or latest, command packet need to includet tags
-        # section 9.6.1.2 (CXP-001-2021)  
-        self.tag_counts = Signal(max=0xFF)
-        self.use_tag = Signal()
-        
         self.bitrate2x_enable = CSRStorage()
         self.tx_enable = CSRStorage()
+        self.tx_busy = CSRStatus()
         self.encoded_data = CSRStatus(10)
 
         # # #
 
-        self.submodules.upconn = upconn = CXP_UpConn(upconn_pads, sys_clk_freq, debug_sma, pmod_pads, fifos_depth)
+        layout = [("data", 8), ("k", 1)]
 
-        self.comb += [
+        self.submodules.upconn_phy = upconn_phy = CXP_UpConn_PHY(upconn_pads, sys_clk_freq, debug_sma, pmod_pads, layout, fifos_depth)
-            upconn.bitrate2x_enable.eq(self.bitrate2x_enable.storage),
+
-            upconn.tx_enable.eq(self.tx_enable.storage),
+        self.sync += [
+            upconn_phy.bitrate2x_enable.eq(self.bitrate2x_enable.storage),
+            upconn_phy.tx_enable.eq(self.tx_enable.storage),
+            upconn_phy.clk_reset.eq(self.clk_reset.re),
+            self.tx_busy.status.eq(upconn_phy.tx_busy),
         ]
         self.sync += [
-            self.encoded_data.status.eq(upconn.scheduler.encoder.output),
+            self.encoded_data.status.eq(upconn_phy.scheduler.encoder.output),
         ]
 
 
         # Packet FIFOs with transmission priority
-        # 0: Trigger packet
+        # NOTE: 0 Trigger packet
+        self.submodules.trig = trig = TX_Trigger(layout)
+        
+        # DEBUG: INPUT
+        self.trig_stb = CSR()
+        self.trig_delay = CSRStorage(8) 
+        self.linktrigger = CSRStorage(2)
+
+        self.sync += [
+            trig.trig_stb.eq(self.trig_stb.re),
+            trig.delay.eq(self.trig_delay.storage),
+            trig.linktrig_mode.eq(self.linktrigger.storage),
+        ]
+
+        # DEBUG: OUTPUT
+        self.submodules.trig_out = trig_out = stream.SyncFIFO(layout, 64)
+        self.comb += trig.source.connect(trig_out.sink)
+        self.trig_inc = CSR()
+        self.trig_dout_pak = CSRStatus(8)
+        self.trig_kout_pak = CSRStatus()
+        self.trig_dout_valid = CSRStatus()
+
+        self.sync += [
+            # output
+            trig_out.source.ack.eq(self.trig_inc.re),
+            self.trig_dout_pak.status.eq(trig_out.source.data),
+            self.trig_kout_pak.status.eq(trig_out.source.k),
+            self.trig_dout_valid.status.eq(trig_out.source.stb),
+        ]
+
         self.symbol0 = CSR(9)
         self.sync += [
-            upconn.tx_fifos.sink_stb[0].eq(self.symbol0.re),
+            upconn_phy.tx_fifos.sink[0].stb.eq(self.symbol0.re),
-            upconn.tx_fifos.sink_data[0].eq(self.symbol0.r[:8]),
+            upconn_phy.tx_fifos.sink[0].data.eq(self.symbol0.r[:8]),
-            upconn.tx_fifos.sink_k[0].eq(self.symbol0.r[8]),
+            upconn_phy.tx_fifos.sink[0].k.eq(self.symbol0.r[8]),
         ]
 
         
-        # 1: IO acknowledgment for trigger packet
+        # NOTE: 1 IO acknowledgment for trigger packet
+
+        self.submodules.trig_ack = trig_ack = Trigger_ACK(layout)
+
+        # DEBUG: INPUT
+        self.ack = CSR()
+        self.sync += [
+            trig_ack.ack.eq(self.ack.re),
+        ]
+
+        # DEBUG: OUTPUT
+        self.submodules.trig_ack_out = trig_ack_out = stream.SyncFIFO(layout, 64)
+        self.comb += trig_ack.source.connect(trig_ack_out.sink)
+        self.trig_ack_inc = CSR()
+        self.trig_ack_dout_pak = CSRStatus(8)
+        self.trig_ack_kout_pak = CSRStatus()
+        self.trig_ack_dout_valid = CSRStatus()
+
+        self.sync += [
+            # output
+            trig_ack_out.source.ack.eq(self.trig_ack_inc.re),
+            self.trig_ack_dout_pak.status.eq(trig_ack_out.source.data),
+            self.trig_ack_kout_pak.status.eq(trig_ack_out.source.k),
+            self.trig_ack_dout_valid.status.eq(trig_ack_out.source.stb),
+        ]
+
         self.symbol1 = CSR(9)
         self.sync += [
-            upconn.tx_fifos.sink_stb[1].eq(self.symbol1.re),
+            upconn_phy.tx_fifos.sink[1].stb.eq(self.symbol1.re),
-            upconn.tx_fifos.sink_data[1].eq(self.symbol1.r[:8]),
+            upconn_phy.tx_fifos.sink[1].data.eq(self.symbol1.r[:8]),
-            upconn.tx_fifos.sink_k[1].eq(self.symbol1.r[8]),
+            upconn_phy.tx_fifos.sink[1].k.eq(self.symbol1.r[8]),
         ]
         
-        # 2: All other packets
+        # NOTE: 2 All other packets
         # Control is not timing dependent, all the link layer is done in firmware
 
         # Table 54 (CXP-001-2021)
         # Largest CXP register is 8 byte
 
-        self.symbol2 = CSR(9)
+        # increment after ack
-        self.sync += [
+        # for CXP 2.0 or latest, command packet need to includet tags
-            upconn.tx_fifos.sink_stb[2].eq(self.symbol2.re),
+        # section 9.6.1.2 (CXP-001-2021)  
-            upconn.tx_fifos.sink_data[2].eq(self.symbol2.r[:8]),
+        # tags implementation is on firmware
-            upconn.tx_fifos.sink_k[2].eq(self.symbol2.r[8]),
-        ]
 
-class CXP_Packet(Module):
+        self.submodules.command = command = TX_Command_Packet(layout)
-    def __init__(self, max_packet_length):
+        self.comb += command.source.connect(upconn_phy.tx_fifos.sink[2])
-        pass
 
+        # DEBUG: OUTPUT
+        # self.submodules.command_out = command_out = stream.SyncFIFO(layout, 64)
+        # self.comb += command.source.connect(command_out.sink)
 
-class CXP_CRC(Module, AutoCSR):
+        # self.command_inc = CSR()
-    def __init__(self, data_width):
+        # self.command_dout_pak = CSRStatus(8)
-        # Section 9.2.2.2 (CXP-001-2021)
+        # self.command_kout_pak = CSRStatus()
-        crc_width = 32
+        # self.command_dout_valid = CSRStatus()
-        polynom = 0x04C11DB7
-        seed = 2**crc_width-1
 
-        self.d = Signal(data_width)
+        # self.sync += [
-        self.stb = Signal()
+        #     # output
-        self.reset = Signal()
+        #     command_out.source.ack.eq(self.command_inc.re),
-        self.val = Signal(crc_width, reset=seed)
+        #     self.command_dout_pak.status.eq(command_out.source.data),
+        #     self.command_kout_pak.status.eq(command_out.source.k),
+        #     self.command_dout_valid.status.eq(command_out.source.stb),
+        # ]
 
-
+        # self.symbol2 = CSR(9)
-        # # #
+        # self.sync += [
-        
+        #     upconn_phy.tx_fifos.sink[2].stb.eq(self.symbol2.re),
-        self.submodules.engine = LiteEthMACCRCEngine(data_width, crc_width, polynom)
+        #     upconn_phy.tx_fifos.sink[2].data.eq(self.symbol2.r[:8]),
-
+        #     upconn_phy.tx_fifos.sink[2].k.eq(self.symbol2.r[8]),
-        self.sync += [
+        # ]
-            self.val.eq(self.engine.next),
-            If(self.stb,
-                self.engine.data.eq(self.d), 
-
-                If(self.reset,
-                    # because the seed is non zero, even if the data is 0x00, the engine output will be change :<
-                    self.engine.last.eq(seed), 
-                    # clear reset bit
-                    self.reset.eq(0),
-                ).Else(
-                    self.engine.last.eq(self.val), 
-                )
-            ),
-        ]
-
-        # DEBUG: remove those csr
-        # TODO: do char bit reverse outside of this submodule
-
-        p0 = Signal(8)
-        p1 = Signal(8)
-        p2 = Signal(8)
-        p3 = Signal(8)
-        self.comb += [
-            p3.eq(self.engine.next[:8][::-1]),
-            p2.eq(self.engine.next[8:16][::-1]),
-            p1.eq(self.engine.next[16:24][::-1]),
-            p0.eq(self.engine.next[24:32][::-1]),
-        ]
-
-        self.data = CSR(data_width)
-        self.en = CSR()
-        self.value = CSRStatus(crc_width)
-        self.processed = CSRStatus(crc_width)
-
-        self.sync += [
-            self.d.eq(self.data.r),
-            self.stb.eq(self.data.re),
-            If(self.en.re, self.reset.eq(1)),
-
-            self.value.status.eq(self.engine.next),
-            self.processed.status.eq(Cat(p3, p2, p1, p0)),
-        ]

src/gateware/cxp_pipeline.py

@@ -0,0 +1,272 @@
+from migen import *
+from misoc.interconnect.csr import *
+from misoc.interconnect import stream
+from misoc.cores.liteeth_mini.mac.crc import LiteEthMACCRCEngine, LiteEthMACCRCChecker
+
+def K(x, y):
+    return ((y << 5) | x)
+
+class Code_Source(Module):
+    def __init__(self, layout, counts=4):
+
+        self.source = stream.Endpoint(layout)
+        self.stb = Signal()
+        self.data = Signal.like(self.source.data)
+        self.k = Signal.like(self.source.k)
+
+        # # #
+
+        cnt = Signal(max=counts)
+        clr_cnt = Signal()
+        inc_cnt = Signal()
+
+        self.sync += [
+            If(clr_cnt,
+                cnt.eq(cnt.reset),
+            ).Elif(inc_cnt,
+                cnt.eq(cnt + 1),
+            )
+        ]
+
+        self.submodules.fsm = fsm = FSM(reset_state="IDLE")
+        
+        fsm.act("IDLE",
+            clr_cnt.eq(1),
+            If(self.stb,
+               NextState("WRITE")
+            )
+        )
+
+        fsm.act("WRITE",
+            self.source.stb.eq(1),
+            self.source.data.eq(self.data),
+            self.source.k.eq(self.k),
+            If(cnt == counts - 1,
+                self.source.eop.eq(1),
+                If(self.source.ack, NextState("IDLE"))
+            ).Else(
+                inc_cnt.eq(self.source.ack)
+            )
+        )
+
+
+class Code_Inserter(Module):
+    def __init__(self, layout, insert_infront=True, counts=4):
+        self.sink = sink = stream.Endpoint(layout)
+        self.source = source = stream.Endpoint(layout)
+
+        self.data = Signal.like(sink.data)
+        self.k = Signal.like(sink.k)
+
+        # # #
+        assert counts > 0
+
+        cnt = Signal() if counts == 1 else Signal(max=counts)
+        clr_cnt = Signal()
+        inc_cnt = Signal()
+
+        self.sync += [
+            If(clr_cnt,
+                cnt.eq(cnt.reset),
+            ).Elif(inc_cnt,
+                cnt.eq(cnt + 1),
+            )
+        ]
+
+        self.submodules.fsm = fsm = FSM(reset_state="IDLE")
+
+        if insert_infront:
+            fsm.act("IDLE",
+                sink.ack.eq(1),
+                clr_cnt.eq(1),
+                If(sink.stb,
+                    sink.ack.eq(0),
+                    NextState("INSERT"),
+                )
+            )
+
+            fsm.act("INSERT",
+                sink.ack.eq(0),
+                source.stb.eq(1),
+                source.data.eq(self.data),
+                source.k.eq(self.k),
+                If(cnt == counts - 1,
+                    If(source.ack, NextState("COPY"))
+                ).Else(
+                    inc_cnt.eq(source.ack)
+                )
+            )
+
+            fsm.act("COPY",
+                sink.connect(source),
+                If(sink.stb & sink.eop & source.ack,
+                    NextState("IDLE"),
+                )
+            )
+
+        else:
+            fsm.act("IDLE",
+                sink.ack.eq(1),
+                clr_cnt.eq(1),
+                If(sink.stb,
+                    sink.ack.eq(0),
+                    NextState("COPY"),
+                )
+            )
+
+            fsm.act("COPY",
+                sink.connect(source),
+                source.eop.eq(0),
+                If(sink.stb & sink.eop & source.ack,
+                    NextState("INSERT"),
+                )
+            )
+
+            fsm.act("INSERT",
+                sink.ack.eq(0),
+                source.stb.eq(1),
+                source.data.eq(self.data),
+                source.k.eq(self.k),
+                If(cnt == counts - 1,
+                    source.eop.eq(1),
+                    If(source.ack, NextState("IDLE"))
+                ).Else(
+                    inc_cnt.eq(source.ack)
+                ),
+            )
+
+
+class Packet_Wrapper(Module):
+    def __init__(self, layout):
+        self.submodules.pak_start = pak_start = Code_Inserter(layout) 
+        self.submodules.pak_end = pak_end = Code_Inserter(layout, insert_infront=False) 
+
+        self.sink = pak_start.sink
+        self.source = pak_end.source
+
+        self.comb += [
+            pak_start.data.eq(K(27, 7)),
+            pak_start.k.eq(1),
+            pak_end.data.eq(K(29, 7)),
+            pak_end.k.eq(1),
+
+            pak_start.source.connect(pak_end.sink),
+        ]
+        
+@ResetInserter()
+@CEInserter()
+class CXPCRC32(Module):
+    # Section 9.2.2.2 (CXP-001-2021)
+    width = 32
+    polynom = 0x04C11DB7
+    seed = 2**width-1
+    check = 0x00000000
+    def __init__(self, data_width):
+        self.data = Signal(data_width)
+        self.value = Signal(self.width)
+        self.error = Signal()
+
+        # # #
+
+        self.submodules.engine = LiteEthMACCRCEngine(data_width, self.width, self.polynom)
+        reg = Signal(self.width, reset=self.seed)
+        self.sync += reg.eq(self.engine.next)
+        self.comb += [
+            self.engine.data.eq(self.data),
+            self.engine.last.eq(reg),
+
+            self.value.eq(reg[::-1]),
+            self.error.eq(self.engine.next != self.check)
+        ]
+
+class CXPCRC32Checker(LiteEthMACCRCChecker):
+    def __init__(self, layout):
+        LiteEthMACCRCChecker.__init__(self, CXPCRC32, layout)
+
+class TX_Trigger(Module, AutoCSR):
+    def __init__(self, layout):
+        self.trig_stb = Signal()
+        self.delay = Signal(8) 
+        self.linktrig_mode = Signal(max=4)
+
+        # # #
+
+        self.submodules.code_src = code_src = Code_Source(layout, counts=3)
+        self.comb += [
+            code_src.stb.eq(self.trig_stb),
+            code_src.data.eq(self.delay),
+            code_src.k.eq(0)
+        ]
+
+        self.submodules.inserter_once = inserter_once = Code_Inserter(layout, counts=1)
+        self.submodules.inserter_twice = inserter_twice = Code_Inserter(layout, counts=2)
+        self.comb += [
+            inserter_once.k.eq(1),
+            inserter_twice.k.eq(1),
+            If((self.linktrig_mode == 0) | (self.linktrig_mode == 2),
+                inserter_once.data.eq(K(28, 2)),
+                inserter_twice.data.eq(K(28, 4)),
+            ).Else(
+                inserter_once.data.eq(K(28, 4)),
+                inserter_twice.data.eq(K(28, 2)),
+            )
+        ]
+        
+        tx_pipeline = [ code_src, inserter_twice, inserter_once]
+
+        for s, d in zip(tx_pipeline, tx_pipeline[1:]):
+            self.comb += s.source.connect(d.sink)
+
+        self.source = tx_pipeline[-1].source
+
+class Trigger_ACK(Module):
+    def __init__(self, layout):
+        self.ack = Signal()
+
+        # # #
+
+        # Section 9.3.2 (CXP-001-2021)
+        # Send 4x K28.6 and 4x 0x01 as trigger packet ack
+        self.submodules.code_src = code_src = Code_Source(layout)
+        self.submodules.k_code_inserter = k_code_inserter = Code_Inserter(layout)
+        self.comb += [
+            code_src.stb.eq(self.ack),
+            code_src.data.eq(0x01),
+            code_src.k.eq(0),
+            k_code_inserter.data.eq(K(28, 6)),
+            k_code_inserter.k.eq(1),
+
+            code_src.source.connect(k_code_inserter.sink)
+        ]
+
+        self.source = k_code_inserter.source
+
+class TX_Command_Packet(Module, AutoCSR):
+    def __init__(self, layout):
+        self.len = CSRStorage(6)
+        self.data = CSR(8)
+        self.writeable = CSRStatus()
+
+        # # #
+
+        self.submodules.pak_wrp = pak_wrp = Packet_Wrapper(layout)
+        self.source = pak_wrp.source
+
+        len = Signal(6, reset=1)
+        self.sync += [
+            self.writeable.status.eq(pak_wrp.sink.ack),
+            If(pak_wrp.sink.ack,pak_wrp.sink.stb.eq(0)),
+            If(self.data.re,
+                pak_wrp.sink.stb.eq(1),
+                pak_wrp.sink.data.eq(self.data.r),
+
+                pak_wrp.sink.k.eq(0),
+                If(len == self.len.storage,
+                    pak_wrp.sink.eop.eq(1),
+                    len.eq(len.reset),
+                ).Else(
+                    pak_wrp.sink.eop.eq(0),
+                    len.eq(len + 1),
+                ),
+            )
+        ]

src/gateware/cxp_upconn.py

@@ -15,6 +15,7 @@ IDLE_CHARS = Array([
     [0xB5, 0], #D21.5
 ])
 
+@ResetInserter()
 class UpConn_ClockGen(Module):
     def __init__(self, sys_clk_freq):
         self.clk = Signal()
@@ -55,6 +56,8 @@ class UpConn_ClockGen(Module):
 
         ]
 
+@ResetInserter()
+@CEInserter()
 class SERDES_10bits(Module):
     def __init__(self, pad):
         self.oe = Signal()
@@ -89,8 +92,10 @@ class SERDES_10bits(Module):
             )
         ]
 
+@ResetInserter()
+@CEInserter()
 class Packets_Scheduler(Module):
-    def __init__(self, tx_fifos):
+    def __init__(self, tx_fifos, debug_buf):
         self.tx_enable = Signal()
 
         self.oe = Signal()
@@ -121,6 +126,13 @@ class Packets_Scheduler(Module):
                 NextValue(encoder.d, IDLE_CHARS[0][0]),
                 NextValue(encoder.k, IDLE_CHARS[0][1]),
                 NextState("START_TX"),
+
+                # DEBUG:
+                If(debug_buf.sink_ack,
+                    NextValue(debug_buf.sink_stb, 1),
+                    NextValue(debug_buf.sink_data, IDLE_CHARS[0][0]),
+                    NextValue(debug_buf.sink_k, IDLE_CHARS[0][1]),
+                )
             )
         )
 
@@ -139,6 +151,13 @@ class Packets_Scheduler(Module):
                     tx_fifos.source_ack[0].eq(1),
                     encoder.d.eq(tx_fifos.source_data[0]),
                     encoder.k.eq(tx_fifos.source_k[0]),
+
+                    # DEBUG:
+                    If(debug_buf.sink_ack,
+                        debug_buf.sink_stb.eq(1),
+                        debug_buf.sink_data.eq(tx_fifos.source_data[0]),
+                        debug_buf.sink_k.eq(tx_fifos.source_k[0]),
+                    )
                 ).Else(
                     If(tx_charcount == 3,
                         tx_charcount.eq(0),
@@ -153,6 +172,13 @@ class Packets_Scheduler(Module):
                             tx_fifos.source_ack[tx_fifos.pe.o].eq(1),
                             encoder.d.eq(tx_fifos.source_data[tx_fifos.pe.o]),
                             encoder.k.eq(tx_fifos.source_k[tx_fifos.pe.o]),
+
+                            # DEBUG:
+                            If(debug_buf.sink_ack,
+                                debug_buf.sink_stb.eq(1),
+                                debug_buf.sink_data.eq(tx_fifos.source_data[tx_fifos.pe.o]),
+                                debug_buf.sink_k.eq(tx_fifos.source_k[tx_fifos.pe.o]),
+                            )
                         ).Else(
                             # Section 9.2.5.1 (CXP-001-2021)
                             # IDLE word shall be transmitted at least once every 10,000 words, but should not be inserted into trigger packet
@@ -161,6 +187,13 @@ class Packets_Scheduler(Module):
 
                             encoder.d.eq(IDLE_CHARS[0][0]),
                             encoder.k.eq(IDLE_CHARS[0][1]),
+
+                            # DEBUG:
+                            If(debug_buf.sink_ack,
+                                debug_buf.sink_stb.eq(1),
+                                debug_buf.sink_data.eq(IDLE_CHARS[0][0]),
+                                debug_buf.sink_k.eq(IDLE_CHARS[0][1]),
+                            )
                         )
                     ).Else(
                         tx_charcount.eq(tx_charcount + 1),
@@ -168,9 +201,23 @@ class Packets_Scheduler(Module):
                             tx_fifos.source_ack[priorities].eq(1),
                             encoder.d.eq(tx_fifos.source_data[priorities]),
                             encoder.k.eq(tx_fifos.source_k[priorities]),
+
+                            # DEBUG:
+                            If(debug_buf.sink_ack,
+                                debug_buf.sink_stb.eq(1),
+                                debug_buf.sink_data.eq(tx_fifos.source_data[priorities]),
+                                debug_buf.sink_k.eq(tx_fifos.source_k[priorities]),
+                            )
                         ).Else(
                             encoder.d.eq(IDLE_CHARS[tx_charcount + 1][0]),
                             encoder.k.eq(IDLE_CHARS[tx_charcount + 1][1]),
+
+                            # DEBUG:
+                            If(debug_buf.sink_ack,
+                                debug_buf.sink_stb.eq(1),
+                                debug_buf.sink_data.eq(IDLE_CHARS[tx_charcount + 1][0]),
+                                debug_buf.sink_k.eq(IDLE_CHARS[tx_charcount + 1][1]),
+                            )
                         )
                     ),       
                 ),
@@ -178,68 +225,114 @@ class Packets_Scheduler(Module):
         ]
 
 class TxFIFOs(Module):
-    def __init__(self, nfifos, fifo_depth):
+    def __init__(self, layout, nfifos, fifo_depth):
 
-        self.sink_full = Signal(nfifos) 
+        self.sink = []
-        self.sink_stb = Signal(nfifos)
-        self.sink_data = [Signal(8) for _ in range(nfifos)]
-        self.sink_k = [Signal() for _ in range(nfifos)]
 
+        self.source_stb = Signal(nfifos)
         self.source_ack = Array(Signal() for _ in range(nfifos))
         self.source_data = Array(Signal(8) for _ in range(nfifos))
         self.source_k = Array(Signal() for _ in range(nfifos))
 
         # # #
 
-        not_empty_reg = Signal(nfifos)
-
         for i in range(nfifos):
-            fifo = stream.SyncFIFO([("data", 8), ("k", 1)], fifo_depth)
+            fifo = stream.SyncFIFO(layout, fifo_depth)
-
             setattr(self.submodules, "tx_fifo" + str(i), fifo)
 
-            self.sync += [
+            self.sink += [fifo.sink]
-                fifo.sink.stb.eq(self.sink_stb[i]),
-                self.sink_full[i].eq(fifo.sink.ack),
-                fifo.sink.data.eq(self.sink_data[i]),
-                fifo.sink.k.eq(self.sink_k[i]),
 
+            self.sync += [
                 If(self.source_ack[i], 
                     # reset ack after asserted 
+                    # since upconn clk run much slower, the ack will be high for longer than expected which will result in data loss
                     self.source_ack[i].eq(0),
                     fifo.source.ack.eq(1),
                 ).Else(
                     fifo.source.ack.eq(0),
                 ),
 
-                not_empty_reg[i].eq(fifo.source.stb),
+                self.source_stb[i].eq(fifo.source.stb),
                 self.source_data[i].eq(fifo.source.data),
                 self.source_k[i].eq(fifo.source.k),
             ]
 
         # FIFOs transmission priority
         self.submodules.pe = PriorityEncoder(nfifos)
-        self.comb += self.pe.i.eq(not_empty_reg)
+        self.comb += self.pe.i.eq(self.source_stb)
 
-class CXP_UpConn(Module):
+class Debug_buffer(Module,AutoCSR):
-    def __init__(self, pad, sys_clk_freq, debug_sma, pmod_pads, fifo_depth, nfifos=3):
+    def __init__(self, layout):
+
+        self.sink_stb = Signal()
+        self.sink_ack = Signal()
+        self.sink_data = Signal(8)
+        self.sink_k = Signal()
+
+        # # #
+
+        self.submodules.buf_out = buf_out = stream.SyncFIFO(layout, 128)
+
+        self.sync += [
+            If(self.sink_stb, 
+                # reset ack after asserted 
+                # since upconn clk run much slower, the stb will be high for longer than expected which will result in multiple data entry
+                self.sink_stb.eq(0),
+                buf_out.sink.stb.eq(1),
+            ).Else(
+                buf_out.sink.stb.eq(0),
+            ),
+
+            self.sink_ack.eq(buf_out.sink.ack),
+            buf_out.sink.data.eq(self.sink_data),
+            buf_out.sink.k.eq(self.sink_k),
+            
+        ]
+
+        self.inc = CSR()
+        self.dout_pak = CSRStatus(8)
+        self.kout_pak = CSRStatus()
+        self.dout_valid = CSRStatus()
+
+        self.sync += [
+            # output
+            buf_out.source.ack.eq(self.inc.re),
+            self.dout_pak.status.eq(buf_out.source.data),
+            self.kout_pak.status.eq(buf_out.source.k),
+            self.dout_valid.status.eq(buf_out.source.stb),
+        ]
+
+
+class CXP_UpConn_PHY(Module, AutoCSR):
+    def __init__(self, pad, sys_clk_freq, debug_sma, pmod_pads, layout, fifo_depth, nfifos=3):
         self.bitrate2x_enable = Signal()
+        self.clk_reset = Signal()
+
         self.tx_enable = Signal()
+        self.tx_busy = Signal()
 
         # # #
 
         self.submodules.cg = cg = UpConn_ClockGen(sys_clk_freq)
-        self.submodules.tx_fifos = tx_fifos = TxFIFOs(nfifos, fifo_depth)
+        self.submodules.tx_fifos = tx_fifos = TxFIFOs(layout, nfifos, fifo_depth)
 
-        self.submodules.scheduler = scheduler = CEInserter()(Packets_Scheduler(tx_fifos))
+        # DEBUG:
-        self.submodules.serdes = serdes = CEInserter()(SERDES_10bits(pad))
+        self.submodules.debug_buf = debug_buf = Debug_buffer(layout)
+
+        self.submodules.scheduler = scheduler = Packets_Scheduler(tx_fifos, debug_buf)
+        self.submodules.serdes = serdes = SERDES_10bits(pad)
 
         self.comb += [
+            self.tx_busy.eq(tx_fifos.source_stb != 0),
+
+            cg.reset.eq(self.clk_reset),
             cg.freq2x_enable.eq(self.bitrate2x_enable),
 
+            scheduler.reset.eq(self.clk_reset),
             scheduler.ce.eq(cg.clk),
             scheduler.tx_enable.eq(self.tx_enable),
 
+            serdes.reset.eq(self.clk_reset),
             serdes.ce.eq(cg.clk_10x),
             serdes.d.eq(scheduler.encoder.output),
             serdes.oe.eq(scheduler.oe),
@@ -259,23 +352,31 @@ class CXP_UpConn(Module):
            # because of clk delay
            p0.eq(scheduler.tx_charcount == 2),
            p3.eq(scheduler.tx_charcount  == 1),
+
         ] 
         self.specials += [
             # # debug sma
-            Instance("OBUF", i_I=cg.clk, o_O=debug_sma.p_tx),
+            # Instance("OBUF", i_I=cg.clk, o_O=debug_sma.p_tx),
-            Instance("OBUF", i_I=cg.clk_10x, o_O=debug_sma.n_rx),
+            # Instance("OBUF", i_I=cg.clk_10x, o_O=debug_sma.n_rx),
+
+
 
             # # pmod 0-7 pin
-            Instance("OBUF", i_I=serdes.o, o_O=pmod_pads[0]),
+            # Instance("OBUF", i_I=serdes.o, o_O=pmod_pads[0]),
-            Instance("OBUF", i_I=cg.clk_10x, o_O=pmod_pads[1]),
+            # Instance("OBUF", i_I=cg.clk_10x, o_O=pmod_pads[1]),
-            Instance("OBUF", i_I=~tx_fifos.pe.n, o_O=pmod_pads[2]),
+            # Instance("OBUF", i_I=~tx_fifos.pe.n, o_O=pmod_pads[2]),
-            Instance("OBUF", i_I=prioity_0, o_O=pmod_pads[3]),
+            # Instance("OBUF", i_I=prioity_0, o_O=pmod_pads[3]),
-            Instance("OBUF", i_I=word_bound, o_O=pmod_pads[4]),
+            # Instance("OBUF", i_I=word_bound, o_O=pmod_pads[4]),
-            Instance("OBUF", i_I=scheduler.idling, o_O=pmod_pads[5]),
+            # Instance("OBUF", i_I=debug_buf.buf_out.sink.stb, o_O=pmod_pads[4]),
-            # Instance("OBUF", i_I=tx_fifos.source_ack[0], o_O=pmod[6]),
+            # Instance("OBUF", i_I=debug_buf.buf_out.sink.ack, o_O=pmod_pads[5]),
-            # Instance("OBUF", i_I=tx_fifos.source_ack[2], o_O=pmod[6]),
+            # Instance("OBUF", i_I=debug_buf.buf_out.source.stb, o_O=pmod_pads[6]),
-            # Instance("OBUF", i_I=tx_fifos.source_ack[1], o_O=pmod[7]),
+            # Instance("OBUF", i_I=debug_buf.buf_out.source.ack, o_O=pmod_pads[7]),
-            Instance("OBUF", i_I=p0, o_O=pmod_pads[6]),
+
-            Instance("OBUF", i_I=p3, o_O=pmod_pads[7]),
+            # Instance("OBUF", i_I=scheduler.idling, o_O=pmod_pads[5]),
+            # # Instance("OBUF", i_I=tx_fifos.source_ack[0], o_O=pmod[6]),
+            # # Instance("OBUF", i_I=tx_fifos.source_ack[2], o_O=pmod[6]),
+            # # Instance("OBUF", i_I=tx_fifos.source_ack[1], o_O=pmod[7]),
+            # Instance("OBUF", i_I=p0, o_O=pmod_pads[6]),
+            # Instance("OBUF", i_I=p3, o_O=pmod_pads[7]),
         ]
 

src/libboard_artiq/src/cxp_upconn.rs

@@ -1,95 +1,195 @@
+use core_io::{Error as IoError, Write};
+use crc::crc32;
 use embedded_hal::prelude::_embedded_hal_blocking_delay_DelayUs;
+use io::Cursor;
 use libboard_zynq::{println, timer::GlobalTimer};
 
 use crate::pl::csr;
 
-pub fn crc_test() {
+pub fn trigger_test(timer: &mut GlobalTimer, linktrig_mode: u8) {
-    let arr = [
+    const LEN: usize = 4 * 8;
-        0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, // CXP CRC-32
+    let mut pak_arr: [u8; LEN] = [0; LEN];
-        0x56, 0x86, 0x5D, 0x6f,
-    ];
-    let mut crc: u32; // seed = 0xFFFFFFFF
 
     unsafe {
-        csr::cxp::crc_en_write(1);
+        csr::cxp::upconn_trig_delay_write(0x05);
+        csr::cxp::upconn_linktrigger_write(linktrig_mode);
+        csr::cxp::upconn_trig_stb_write(1); // send trig
+        timer.delay_us(1);
 
-        for a in arr.iter() {
+        let mut i: usize = 0;
-            csr::cxp::crc_data_write(*a);
+        while csr::cxp::upconn_trig_dout_valid_read() == 1 {
-            crc = csr::cxp::crc_value_read();
+            pak_arr[i] = csr::cxp::upconn_trig_dout_pak_read();
-            println!("input = {:#04x}", *a);
+            // println!("received {:#04X}", pak_arr[i]);
-            // println!("CRC NOT(val.reverse) = {:#010x}", !crc.reverse_bits());
+            csr::cxp::upconn_trig_inc_write(1);
-            // since the input bit are reversed when entering the crc engine, the output char need to be reversed to cancel out on the receiver side
+            i += 1;
-            // println!("CRC CXP = {:#010x}", crc);
-            println!("CRC processed = {:#010x}", csr::cxp::crc_processed_read())
         }
+
+        println!("trigger packet | linktrigger = {}", linktrig_mode);
+        print_packet(&pak_arr);
+    }
+}
+
+pub fn trigger_ack_test(timer: &mut GlobalTimer) {
+    const LEN: usize = 4 * 8;
+    let mut pak_arr: [u8; LEN] = [0; LEN];
+
+    unsafe {
+        csr::cxp::upconn_ack_write(1); // send IO ack
+
+        let mut i: usize = 0;
+        while csr::cxp::upconn_trig_ack_dout_valid_read() == 1 {
+            pak_arr[i] = csr::cxp::upconn_trig_ack_dout_pak_read();
+            // println!("received {:#04X}", pak_arr[i]);
+            csr::cxp::upconn_trig_ack_inc_write(1);
+            i += 1;
+        }
+
+        println!("trigger ack packet");
+        print_packet(&pak_arr);
     }
 }
 
 pub fn tx_test(timer: &mut GlobalTimer) {
-    // the 8bit shift is k symbol
+    const LEN: usize = 4 * 20;
-    // const K28_1: u16 = 0x3C | (1 << 8);
+    let mut pak_arr: [u8; LEN] = [0; LEN];
-    // const K28_5: u16 = 0xBC | (1 << 8);
-    const D01_1: u16 = 0x21;
-    const D31_1: u16 = 0x3F;
-
-    const LEN: usize = 200;
-    let mut arr: [u16; LEN] = [0; LEN];
 
     unsafe {
-        csr::cxp::upconn_bitrate2x_enable_write(1);
+        csr::cxp::upconn_clk_reset_write(1);
+        // csr::cxp::upconn_bitrate2x_enable_write(1);
+        csr::cxp::upconn_clk_reset_write(0);
 
-        loop {
+        send(&Packet::ControlU32Reg(Command::Read { addr: 0x00 })).expect("Cannot send CoaXpress packet");
-            // TODO: verify the char & word boundary thingy
-            for _ in 0..12 {
-                csr::cxp::upconn_symbol2_write(D31_1);
-            }
 
         csr::cxp::upconn_tx_enable_write(1);
-            timer.delay_us(1);
+        timer.delay_us(20);
-
-            for _ in 0..3 {
-                csr::cxp::upconn_symbol0_write(D01_1);
-            }
-
-            for i in 0..LEN {
-                arr[i] = get_encoded();
-            }
-            let mut last_encoded: u16 = 0;
-            for i in 0..LEN {
-                if last_encoded != arr[i] {
-                    match arr[i] {
-                        0b1010111001 | 0b0101001001 => {
-                            println!("D31.1")
-                        }
-                        0b0111011001 | 0b1000101001 => {
-                            println!("D01.1")
-                        }
-                        0b1101010010 | 0b0010101101 => {
-                            println!("D04.4")
-                        }
-                        0b0011111010 | 0b1100000101 => {
-                            println!("K28.5 start idling....")
-                        }
-                        0b0011111001 | 0b1100000110 => {
-                            println!("K28.1 idling...")
-                        }
-                        0b1010101010 => {
-                            println!("D21.5 END idle")
-                        }
-                        _ => {
-                            println!("encoded = {:#012b}", arr[i])
-                        }
-                    }
-                    last_encoded = arr[i]
-                }
-            }
-            println!("-------------------------------------");
-
         csr::cxp::upconn_tx_enable_write(0);
-            timer.delay_us(2_000_000);
+
+        // Collect data
+        let mut i: usize = 0;
+        while csr::cxp::upconn_upconn_phy_debug_buf_dout_valid_read() == 1 {
+            pak_arr[i] = csr::cxp::upconn_upconn_phy_debug_buf_dout_pak_read();
+            // println!("received {:#04X}", pak_arr[i]);
+            csr::cxp::upconn_upconn_phy_debug_buf_inc_write(1);
+            i += 1;
+            if i == LEN {
+                break;
             }
         }
-    fn get_encoded() -> u16 {
+
-        unsafe { csr::cxp::upconn_encoded_data_read().reverse_bits() >> 6 }
+        print_packet(&pak_arr);
     }
 }
+
+pub enum Command<T> {
+    Read { addr: u32 },
+    Write { addr: u32, data: T },
+    ReadWithTag { addr: u32, tag: u8 },
+    WriteWithTag { addr: u32, data: T, tag: u8 },
+}
+
+pub enum Packet {
+    ControlU32Reg(Command<u32>),
+    ControlU64Reg(Command<u64>),
+}
+
+impl Packet {
+    pub fn write_to<W>(&self, writer: &mut W) -> Result<(), IoError>
+    where W: Write {
+        match self {
+            Packet::ControlU32Reg(cmd) => match cmd {
+                Command::Read { addr } => {
+                    writer.write(&[0x02; 4])?;
+                    writer.write(&[0x00, 0x00, 0x00, 0x04])?;
+                    writer.write(&addr.to_be_bytes())?;
+                }
+                Command::Write { addr, data } => {
+                    writer.write(&[0x02; 4])?;
+                    writer.write(&[0x01, 0x00, 0x00, 0x04])?;
+                    writer.write(&addr.to_be_bytes())?;
+                    writer.write(&data.to_be_bytes())?;
+                }
+                Command::ReadWithTag { addr, tag } => {
+                    writer.write(&[0x05; 4])?;
+                    writer.write(&[*tag; 4])?;
+                    writer.write(&[0x00, 0x00, 0x00, 0x04])?;
+                    writer.write(&addr.to_be_bytes())?;
+                }
+                Command::WriteWithTag { addr, data, tag } => {
+                    writer.write(&[0x05; 4])?;
+                    writer.write(&[*tag; 4])?;
+                    writer.write(&[0x01, 0x00, 0x00, 0x04])?;
+                    writer.write(&addr.to_be_bytes())?;
+                    writer.write(&data.to_be_bytes())?;
+                }
+            },
+            Packet::ControlU64Reg(cmd) => match cmd {
+                Command::Read { addr } => {
+                    writer.write(&[0x02; 4])?;
+                    writer.write(&[0x00, 0x00, 0x00, 0x08])?;
+                    writer.write(&addr.to_be_bytes())?;
+                }
+                Command::Write { addr, data } => {
+                    writer.write(&[0x02; 4])?;
+                    writer.write(&[0x01, 0x00, 0x00, 0x08])?;
+                    writer.write(&addr.to_be_bytes())?;
+                    writer.write(&data.to_be_bytes())?;
+                }
+                Command::ReadWithTag { addr, tag } => {
+                    writer.write(&[0x05; 4])?;
+                    writer.write(&[*tag; 4])?;
+                    writer.write(&[0x00, 0x00, 0x00, 0x08])?;
+                    writer.write(&addr.to_be_bytes())?;
+                }
+                Command::WriteWithTag { addr, data, tag } => {
+                    writer.write(&[0x05; 4])?;
+                    writer.write(&[*tag; 4])?;
+                    writer.write(&[0x01, 0x00, 0x00, 0x08])?;
+                    writer.write(&addr.to_be_bytes())?;
+                    writer.write(&data.to_be_bytes())?;
+                }
+            },
+        }
+        Ok(())
+    }
+}
+
+pub fn send(packet: &Packet) -> Result<(), IoError> {
+    const LEN: usize = 4 * 20;
+    let mut buffer: [u8; LEN] = [0; LEN];
+    let mut writer = Cursor::new(&mut buffer[..]);
+
+    packet.write_to(&mut writer)?;
+
+    // Section 9.2.2.2 (CXP-001-2021)
+    // CoaXpress use the polynomial of IEEE-802.3 (Ethernet) CRC but the checksum calculation is different
+    // Also, the calculation does not include the first 4 bytes of packet_type
+    let checksum = crc32::checksum_ieee(&writer.get_ref()[4..writer.position()]);
+    writer.write(&(!checksum).to_le_bytes())?;
+
+    unsafe {
+        let len = writer.position();
+        csr::cxp::upconn_command_len_write(len as u8);
+        for data in writer.get_ref()[..len].iter() {
+            while csr::cxp::upconn_command_writeable_read() == 0 {}
+            csr::cxp::upconn_command_data_write(*data);
+        }
+    }
+
+    Ok(())
+}
+
+fn print_packet(pak: &[u8]) {
+    println!("pak = [");
+    for i in 0..(pak.len() / 4) {
+        println!(
+            "{:#03}     {:#04X} {:#04X} {:#04X} {:#04X},",
+            i + 1,
+            pak[i * 4],
+            pak[i * 4 + 1],
+            pak[i * 4 + 2],
+            pak[i * 4 + 3]
+        )
+    }
+    println!("]");
+    println!("============================================");
+}