cxp downconn firmware: GTX setup

testing: add loopmode mode & tsusrclk mmcm drp bitbang
testing: add IDLE word printout
downconn: add QPLL and GTX setup
downconn: add DRP to support all CXP linerate up to 12.5Gbps

flake.lock

@@ -11,11 +11,11 @@
         "src-pythonparser": "src-pythonparser"
       },
       "locked": {
-        "lastModified": 1725373154,
-        "narHash": "sha256-fq9EW9fDWrV0v1vNj7ZqDNpNYx8+OxoFdPwpvkPf67g=",
+        "lastModified": 1724210813,
+        "narHash": "sha256-OqQdE2lC0jKNS2fFq0Fda1nBpyT8ijmSXqdkO8xeOJ8=",
         "ref": "refs/heads/master",
-        "rev": "0c1ffa9f4f6a3e7864459923ec4b9cc45f16327a",
-        "revCount": 9005,
+        "rev": "61e96b37f9c4345e2d7bf71d47ba0b5e947de83e",
+        "revCount": 8985,
         "type": "git",
         "url": "https://github.com/m-labs/artiq.git"
       },
@@ -102,11 +102,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1724224976,
-        "narHash": "sha256-Z/ELQhrSd7bMzTO8r7NZgi9g5emh+aRKoCdaAv5fiO0=",
+        "lastModified": 1723362943,
+        "narHash": "sha256-dFZRVSgmJkyM0bkPpaYRtG/kRMRTorUIDj8BxoOt1T4=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "c374d94f1536013ca8e92341b540eba4c22f9c62",
+        "rev": "a58bc8ad779655e790115244571758e8de055e3d",
         "type": "github"
       },
       "original": {
@@ -169,11 +169,11 @@
     "src-migen": {
       "flake": false,
       "locked": {
-        "lastModified": 1724304798,
-        "narHash": "sha256-tQ02N0eXY5W/Z7CrOy3Cu4WjDZDQWb8hYlzsFzr3Mus=",
+        "lastModified": 1721561053,
+        "narHash": "sha256-z3LRhNmKZrjr6rFD0yxtccSa/SWvFIYmb+G/D5d2Jd8=",
         "owner": "m-labs",
         "repo": "migen",
-        "rev": "832a7240ba32af9cbd4fdd519ddcb4f912534726",
+        "rev": "9279e8623f8433bc4f23ac51e5e2331bfe544417",
         "type": "github"
       },
       "original": {

src/gateware/cxp.py

@@ -2,7 +2,7 @@ from migen import *
 from misoc.interconnect.csr import *
 from misoc.interconnect import stream
 
-from cxp_downconn import CXP_DownConn_PHY
+from cxp_downconn import CXP_DownConn
 from cxp_upconn import CXP_UpConn_PHY
 from cxp_pipeline import *
 
@@ -10,20 +10,11 @@ class CXP(Module, AutoCSR):
     def __init__(self, refclk, downconn_pads, upconn_pads, sys_clk_freq, debug_sma, pmod_pads):
         self.submodules.upconn = UpConn_Interface(upconn_pads, sys_clk_freq, debug_sma, pmod_pads)
 
-        self.submodules.downconn = DownConn_Interface(refclk, downconn_pads, sys_clk_freq, debug_sma, pmod_pads)
+        self.submodules.downconn = CXP_DownConn(refclk, downconn_pads, sys_clk_freq, debug_sma, pmod_pads)
         # TODO: support the option high speed upconn
 
         # TODO: add link layer
 
-class DownConn_Interface(Module, AutoCSR):
-    def __init__(self, refclk, downconn_pads, sys_clk_freq, debug_sma, pmod_pads):
-
-        # # #
-
-        self.submodules.phy = phy = CXP_DownConn_PHY(refclk, downconn_pads, sys_clk_freq, debug_sma, pmod_pads)
-        self.gtxs = phy.gtxs
-
-
         
 class UpConn_Interface(Module, AutoCSR):
     def __init__(self, upconn_pads, sys_clk_freq, debug_sma, pmod_pads):
@@ -32,24 +23,24 @@ class UpConn_Interface(Module, AutoCSR):
         self.tx_enable = CSRStorage()
         self.tx_busy = CSRStatus()
 
-        self.tx_testmode_en = CSRStorage()
-
         # # #
 
-        self.submodules.phy = phy = CXP_UpConn_PHY(upconn_pads, sys_clk_freq, debug_sma, pmod_pads)
+        layout = [("data", 8), ("k", 1)]
+
+        self.submodules.upconn_phy = upconn_phy = CXP_UpConn_PHY(upconn_pads, sys_clk_freq, debug_sma, pmod_pads, layout)
 
         self.sync += [
-            phy.bitrate2x_enable.eq(self.bitrate2x_enable.storage),
-            phy.tx_enable.eq(self.tx_enable.storage),
-            phy.clk_reset.eq(self.clk_reset.re),
-            self.tx_busy.status.eq(phy.tx_busy),
+            upconn_phy.bitrate2x_enable.eq(self.bitrate2x_enable.storage),
+            upconn_phy.tx_enable.eq(self.tx_enable.storage),
+            upconn_phy.clk_reset.eq(self.clk_reset.re),
+            self.tx_busy.status.eq(upconn_phy.tx_busy),
         ]
 
 
         # Packet FIFOs with transmission priority
         # 0: Trigger packet
-        self.submodules.trig = trig = TX_Trigger()
-        self.comb += trig.source.connect(phy.sinks[0])
+        self.submodules.trig = trig = TX_Trigger(layout)
+        self.comb += trig.source.connect(upconn_phy.sinks[0])
         
         # DEBUG: INPUT
         self.trig_stb = CSR()
@@ -64,8 +55,8 @@ class UpConn_Interface(Module, AutoCSR):
 
         
         # 1: IO acknowledgment for trigger packet
-        self.submodules.trig_ack = trig_ack = Trigger_ACK()
-        self.comb += trig_ack.source.connect(phy.sinks[1])
+        self.submodules.trig_ack = trig_ack = Trigger_ACK(layout)
+        self.comb += trig_ack.source.connect(upconn_phy.sinks[1])
 
         # DEBUG: INPUT
         self.ack = CSR()
@@ -74,16 +65,5 @@ class UpConn_Interface(Module, AutoCSR):
 
         # 2: All other packets
         # Control is not timing dependent, all the link layer is done in firmware
-        self.submodules.command = command = TX_Command_Packet()
-        self.submodules.testseq = testseq = TX_Test_Packet()
-
-
-        self.submodules.mux = mux = stream.Multiplexer(upconn_layout, 2)
-
-        self.comb += [
-            command.source.connect(mux.sink0),
-            testseq.source.connect(mux.sink1),
-            mux.sel.eq(self.tx_testmode_en.storage),
-
-            mux.source.connect(phy.sinks[2])
-        ]
+        self.submodules.command = command = TX_Command_Packet(layout)
+        self.comb += command.source.connect(upconn_phy.sinks[2])

src/gateware/cxp_downconn.py

@@ -10,7 +10,7 @@ from artiq.gateware.drtio.transceiver.gtx_7series_init import *
 from functools import reduce
 from operator import add
 
-class CXP_DownConn_PHY(Module, AutoCSR):
+class CXP_DownConn(Module, AutoCSR):
     def __init__(self, refclk, pads, sys_clk_freq, debug_sma, pmod_pads):
         nconn = len(pads)
         self.rx_start_init = CSRStorage()

src/gateware/cxp_pipeline.py

@@ -3,11 +3,6 @@ from misoc.interconnect.csr import *
 from misoc.interconnect import stream
 from misoc.cores.liteeth_mini.mac.crc import LiteEthMACCRCEngine, LiteEthMACCRCChecker
 
-upconn_dw = 8
-upconn_layout = [("data", upconn_dw), ("k", upconn_dw//8)]
-
-
-
 def K(x, y):
     return ((y << 5) | x)
 
@@ -57,11 +52,11 @@ class Code_Source(Module):
 
 class Code_Inserter(Module):
     def __init__(self, layout, insert_infront=True, counts=4):
-        self.sink = stream.Endpoint(layout)
-        self.source = stream.Endpoint(layout)
+        self.sink = sink = stream.Endpoint(layout)
+        self.source = source = stream.Endpoint(layout)
 
-        self.data = Signal.like(self.sink.data)
-        self.k = Signal.like(self.sink.k)
+        self.data = Signal.like(sink.data)
+        self.k = Signal.like(sink.k)
 
         # # #
         assert counts > 0
@@ -82,61 +77,61 @@ class Code_Inserter(Module):
 
         if insert_infront:
             fsm.act("IDLE",
-                self.sink.ack.eq(1),
+                sink.ack.eq(1),
                 clr_cnt.eq(1),
-                If(self.sink.stb,
-                    self.sink.ack.eq(0),
+                If(sink.stb,
+                    sink.ack.eq(0),
                     NextState("INSERT"),
                 )
             )
 
             fsm.act("INSERT",
-                self.sink.ack.eq(0),
-                self.source.stb.eq(1),
-                self.source.data.eq(self.data),
-                self.source.k.eq(self.k),
+                sink.ack.eq(0),
+                source.stb.eq(1),
+                source.data.eq(self.data),
+                source.k.eq(self.k),
                 If(cnt == counts - 1,
-                    If(self.source.ack, NextState("COPY"))
+                    If(source.ack, NextState("COPY"))
                 ).Else(
-                    inc_cnt.eq(self.source.ack)
+                    inc_cnt.eq(source.ack)
                 )
             )
 
             fsm.act("COPY",
-                self.sink.connect(self.source),
-                If(self.sink.stb & self.sink.eop & self.source.ack,
+                sink.connect(source),
+                If(sink.stb & sink.eop & source.ack,
                     NextState("IDLE"),
                 )
             )
 
         else:
             fsm.act("IDLE",
-                self.sink.ack.eq(1),
+                sink.ack.eq(1),
                 clr_cnt.eq(1),
-                If(self.sink.stb,
-                    self.sink.ack.eq(0),
+                If(sink.stb,
+                    sink.ack.eq(0),
                     NextState("COPY"),
                 )
             )
 
             fsm.act("COPY",
-                self.sink.connect(self.source),
-                self.source.eop.eq(0),
-                If(self.sink.stb & self.sink.eop & self.source.ack,
+                sink.connect(source),
+                source.eop.eq(0),
+                If(sink.stb & sink.eop & source.ack,
                     NextState("INSERT"),
                 )
             )
 
             fsm.act("INSERT",
-                self.sink.ack.eq(0),
-                self.source.stb.eq(1),
-                self.source.data.eq(self.data),
-                self.source.k.eq(self.k),
+                sink.ack.eq(0),
+                source.stb.eq(1),
+                source.data.eq(self.data),
+                source.k.eq(self.k),
                 If(cnt == counts - 1,
-                    self.source.eop.eq(1),
-                    If(self.source.ack, NextState("IDLE"))
+                    source.eop.eq(1),
+                    If(source.ack, NextState("IDLE"))
                 ).Else(
-                    inc_cnt.eq(self.source.ack)
+                    inc_cnt.eq(source.ack)
                 ),
             )
 
@@ -189,22 +184,22 @@ class CXPCRC32Checker(LiteEthMACCRCChecker):
         LiteEthMACCRCChecker.__init__(self, CXPCRC32, layout)
 
 class TX_Trigger(Module, AutoCSR):
-    def __init__(self):
+    def __init__(self, layout):
         self.trig_stb = Signal()
-        self.delay = Signal(upconn_dw) 
+        self.delay = Signal(8) 
         self.linktrig_mode = Signal(max=4)
 
         # # #
 
-        self.submodules.code_src = code_src = Code_Source(upconn_layout, counts=3)
+        self.submodules.code_src = code_src = Code_Source(layout, counts=3)
         self.comb += [
             code_src.stb.eq(self.trig_stb),
             code_src.data.eq(self.delay),
             code_src.k.eq(0)
         ]
 
-        self.submodules.inserter_once = inserter_once = Code_Inserter(upconn_layout, counts=1)
-        self.submodules.inserter_twice = inserter_twice = Code_Inserter(upconn_layout, counts=2)
+        self.submodules.inserter_once = inserter_once = Code_Inserter(layout, counts=1)
+        self.submodules.inserter_twice = inserter_twice = Code_Inserter(layout, counts=2)
         self.comb += [
             inserter_once.k.eq(1),
             inserter_twice.k.eq(1),
@@ -225,15 +220,15 @@ class TX_Trigger(Module, AutoCSR):
         self.source = tx_pipeline[-1].source
 
 class Trigger_ACK(Module):
-    def __init__(self):
+    def __init__(self, layout):
         self.ack = Signal()
 
         # # #
 
         # Section 9.3.2 (CXP-001-2021)
         # Send 4x K28.6 and 4x 0x01 as trigger packet ack
-        self.submodules.code_src = code_src = Code_Source(upconn_layout)
-        self.submodules.k_code_inserter = k_code_inserter = Code_Inserter(upconn_layout)
+        self.submodules.code_src = code_src = Code_Source(layout)
+        self.submodules.k_code_inserter = k_code_inserter = Code_Inserter(layout)
         self.comb += [
             code_src.stb.eq(self.ack),
             code_src.data.eq(0x01),
@@ -247,17 +242,20 @@ class Trigger_ACK(Module):
         self.source = k_code_inserter.source
 
 class TX_Command_Packet(Module, AutoCSR):
-    def __init__(self):
+    def __init__(self, layout):
         self.len = CSRStorage(6)
-        self.data = CSR(upconn_dw)
+        self.data = CSR(8)
         self.writeable = CSRStatus()
 
         # # #
 
+        # TODO: use RAM instead of FIFO ?
         # Section 12.1.2 (CXP-001-2021)
         # Max control packet size is 128 bytes
-        self.submodules.fifo = fifo = stream.SyncFIFO(upconn_layout, 128)
-        self.submodules.pak_wrp = pak_wrp = Packet_Wrapper(upconn_layout)
+        
+        # NOTE: The firmware will lock up if there is not enough space for the packet
+        self.submodules.fifo = fifo = stream.SyncFIFO(layout, 128)
+        self.submodules.pak_wrp = pak_wrp = Packet_Wrapper(layout)
         self.source = pak_wrp.source
 
         self.comb += fifo.source.connect(pak_wrp.sink)
@@ -280,69 +278,3 @@ class TX_Command_Packet(Module, AutoCSR):
                 ),
             )
         ]
-
-class TX_Test_Packet(Module, AutoCSR):
-    def __init__(self):
-
-        self.stb = CSR()
-        self.busy = CSRStatus()
-
-        # # #
-
-        testdata_src = stream.Endpoint(upconn_layout)
-
-        
-        # Section 9.9.2 (CXP-001-2021)
-        # 0x00, 0x01 ... 0xFF need to be send 16 times 
-        # cnt[8:12] is used to count up 16 times while cnt[:8] is the data 
-        cnt = Signal(max=0x1000)
-        clr_cnt = Signal()
-        inc_cnt = Signal()
-
-        self.sync += [
-            If(clr_cnt,
-                cnt.eq(cnt.reset),
-            ).Elif(inc_cnt,
-                cnt.eq(cnt + 1),
-            ),
-        ]
-
-        self.submodules.fsm = fsm = FSM(reset_state="IDLE")
-        
-        fsm.act("IDLE",
-            clr_cnt.eq(1),
-            If(self.stb.re,
-               NextState("WRITE")
-            )
-        )
-
-        fsm.act("WRITE",
-            testdata_src.stb.eq(1),
-            testdata_src.data.eq(cnt[:8]),
-            testdata_src.k.eq(0),
-            If(cnt == 0xFFF,
-                testdata_src.eop.eq(1),
-                If(testdata_src.ack, NextState("IDLE"))
-            ).Else(
-                inc_cnt.eq(testdata_src.ack)
-            )
-        )
-
-        self.submodules.pak_type_inserter = pak_type_inserter = Code_Inserter(upconn_layout)
-        self.submodules.pak_wrp = pak_wrp = Packet_Wrapper(upconn_layout)
-        self.comb += [
-            pak_type_inserter.data.eq(0x04),
-            pak_type_inserter.k.eq(0x04),
-
-            testdata_src.connect(pak_type_inserter.sink),
-            pak_type_inserter.source.connect(pak_wrp.sink),
-        ]
-
-        self.source = pak_wrp.source
-
-        self.sync += \
-            If(self.stb.re,
-                self.busy.status.eq(1),
-            ).Elif(self.source.eop & self.source.ack,
-                self.busy.status.eq(0)
-            )

src/gateware/cxp_upconn.py

@@ -7,8 +7,6 @@ from misoc.cores.code_8b10b import SingleEncoder
 from misoc.interconnect import stream
 from misoc.interconnect.csr import *
 
-from cxp_pipeline import upconn_layout
-
 IDLE_CHARS = Array([
     #[char, k]
     [0xBC, 1], #K28.5
@@ -95,27 +93,23 @@ class SERDES_10bits(Module):
         ]
 
 @ResetInserter()
-class Transmit_Scheduler(Module):
-    def __init__(self, interface, debug_buf):
+@CEInserter()
+class Packets_Scheduler(Module):
+    def __init__(self, tx_fifos, debug_buf):
         self.tx_enable = Signal()
 
         self.oe = Signal()
-        self.ce = Signal()
 
         # # #
 
-        self.submodules.startup_fsm = startup_fsm = CEInserter()(FSM(reset_state="WAIT_TX_ENABLE"))
-        self.submodules.encoder = encoder = CEInserter()(SingleEncoder(True))
-        self.comb += [
-            startup_fsm.ce.eq(self.ce),
-            encoder.ce.eq(self.ce),
-        ]
+        self.submodules.startup_fsm = startup_fsm = FSM(reset_state="WAIT_TX_ENABLE")
+        self.submodules.encoder = encoder = SingleEncoder(True)
 
         tx_charcount = Signal(max=4)
         tx_wordcount = Signal(max=10000)
 
         idling = Signal()
-        priorities = Signal.like(interface.pe.o)
+        priorities = Signal.like(tx_fifos.pe.o)
 
         # DEBUG:
         self.idling = Signal()
@@ -149,25 +143,20 @@ class Transmit_Scheduler(Module):
             )
         )
 
-        # hold ack for only one sys clk cycle to prevent data loss
-        for ack in interface.sink_ack:
-            self.sync += ack.eq(0)
-
         self.sync += [
-            debug_buf.sink_stb.eq(0),
-            If(self.oe & self.ce,
+            If(self.oe,
                 encoder.disp_in.eq(encoder.disp_out),
-                If((~interface.pe.n) & (interface.pe.o == 0),
+                If((~tx_fifos.pe.n) & (tx_fifos.pe.o == 0),
                     # trigger packets are inserted at char boundary and don't contribute to word count
-                    interface.sink_ack[0].eq(1),
-                    encoder.d.eq(interface.sink_data[0]),
-                    encoder.k.eq(interface.sink_k[0]),
+                    tx_fifos.source_ack[0].eq(1),
+                    encoder.d.eq(tx_fifos.source_data[0]),
+                    encoder.k.eq(tx_fifos.source_k[0]),
 
                     # DEBUG:
                     If(debug_buf.sink_ack,
                         debug_buf.sink_stb.eq(1),
-                        debug_buf.sink_data.eq(interface.sink_data[0]),
-                        debug_buf.sink_k.eq(interface.sink_k[0]),
+                        debug_buf.sink_data.eq(tx_fifos.source_data[0]),
+                        debug_buf.sink_k.eq(tx_fifos.source_k[0]),
                     )
                 ).Else(
                     If(tx_charcount == 3,
@@ -175,20 +164,20 @@ class Transmit_Scheduler(Module):
 
                         # Section 9.2.4 (CXP-001-2021)
                         # other priorities packets are inserted at word boundary
-                        If((~interface.pe.n) & (tx_wordcount != 9999),
+                        If((~tx_fifos.pe.n) & (tx_wordcount != 9999),
                             idling.eq(0),
-                            priorities.eq(interface.pe.o),
+                            priorities.eq(tx_fifos.pe.o),
                             tx_wordcount.eq(tx_wordcount + 1),
 
-                            interface.sink_ack[interface.pe.o].eq(1),
-                            encoder.d.eq(interface.sink_data[interface.pe.o]),
-                            encoder.k.eq(interface.sink_k[interface.pe.o]),
+                            tx_fifos.source_ack[tx_fifos.pe.o].eq(1),
+                            encoder.d.eq(tx_fifos.source_data[tx_fifos.pe.o]),
+                            encoder.k.eq(tx_fifos.source_k[tx_fifos.pe.o]),
 
                             # DEBUG:
                             If(debug_buf.sink_ack,
                                 debug_buf.sink_stb.eq(1),
-                                debug_buf.sink_data.eq(interface.sink_data[interface.pe.o]),
-                                debug_buf.sink_k.eq(interface.sink_k[interface.pe.o]),
+                                debug_buf.sink_data.eq(tx_fifos.source_data[tx_fifos.pe.o]),
+                                debug_buf.sink_k.eq(tx_fifos.source_k[tx_fifos.pe.o]),
                             )
                         ).Else(
                             # Section 9.2.5.1 (CXP-001-2021)
@@ -209,16 +198,15 @@ class Transmit_Scheduler(Module):
                     ).Else(
                         tx_charcount.eq(tx_charcount + 1),
                         If(~idling,
-                            tx_wordcount.eq(tx_wordcount + 1),
-                            interface.sink_ack[priorities].eq(1),
-                            encoder.d.eq(interface.sink_data[priorities]),
-                            encoder.k.eq(interface.sink_k[priorities]),
+                            tx_fifos.source_ack[priorities].eq(1),
+                            encoder.d.eq(tx_fifos.source_data[priorities]),
+                            encoder.k.eq(tx_fifos.source_k[priorities]),
 
                             # DEBUG:
                             If(debug_buf.sink_ack,
                                 debug_buf.sink_stb.eq(1),
-                                debug_buf.sink_data.eq(interface.sink_data[priorities]),
-                                debug_buf.sink_k.eq(interface.sink_k[priorities]),
+                                debug_buf.sink_data.eq(tx_fifos.source_data[priorities]),
+                                debug_buf.sink_k.eq(tx_fifos.source_k[priorities]),
                             )
                         ).Else(
                             encoder.d.eq(IDLE_CHARS[tx_charcount + 1][0]),
@@ -238,10 +226,11 @@ class Transmit_Scheduler(Module):
 
 class PHY_Interface(Module):
     def __init__(self, layout, nsink):
-        sink_stb = Signal(nsink)
-        self.sink_ack = Array(Signal() for _ in range(nsink))
-        self.sink_data = Array(Signal(8) for _ in range(nsink))
-        self.sink_k = Array(Signal() for _ in range(nsink))
+
+        self.source_stb = Signal(nsink)
+        self.source_ack = Array(Signal() for _ in range(nsink))
+        self.source_data = Array(Signal(8) for _ in range(nsink))
+        self.source_k = Array(Signal() for _ in range(nsink))
 
         # # #
 
@@ -250,19 +239,28 @@ class PHY_Interface(Module):
             sink = stream.Endpoint(layout)
             self.sinks += [sink]
 
-            self.comb += [
-                sink.ack.eq(self.sink_ack[i]),
-                sink_stb[i].eq(sink.stb),
-                self.sink_data[i].eq(sink.data),
-                self.sink_k[i].eq(sink.k),
+            self.sync += [
+                If(self.source_ack[i], 
+                    # reset ack after asserted 
+                    # since upconn clk run much slower, the ack will be high for longer than expected which will result in data loss
+                    self.source_ack[i].eq(0),
+                    sink.ack.eq(1),
+                ).Else(
+                    sink.ack.eq(0),
+                ),
+
+                self.source_stb[i].eq(sink.stb),
+                self.source_data[i].eq(sink.data),
+                self.source_k[i].eq(sink.k),
             ]
 
         # FIFOs transmission priority
         self.submodules.pe = PriorityEncoder(nsink)
-        self.comb += self.pe.i.eq(sink_stb)
+        self.comb += self.pe.i.eq(self.source_stb)
 
 class Debug_buffer(Module,AutoCSR):
     def __init__(self, layout):
+
         self.sink_stb = Signal()
         self.sink_ack = Signal()
         self.sink_data = Signal(8)
@@ -273,10 +271,19 @@ class Debug_buffer(Module,AutoCSR):
         self.submodules.buf_out = buf_out = stream.SyncFIFO(layout, 128)
 
         self.sync += [
-            buf_out.sink.stb.eq(self.sink_stb),
+            If(self.sink_stb, 
+                # reset ack after asserted 
+                # since upconn clk run much slower, the stb will be high for longer than expected which will result in multiple data entry
+                self.sink_stb.eq(0),
+                buf_out.sink.stb.eq(1),
+            ).Else(
+                buf_out.sink.stb.eq(0),
+            ),
+
             self.sink_ack.eq(buf_out.sink.ack),
             buf_out.sink.data.eq(self.sink_data),
             buf_out.sink.k.eq(self.sink_k),
+            
         ]
 
         self.inc = CSR()
@@ -294,7 +301,7 @@ class Debug_buffer(Module,AutoCSR):
 
 
 class CXP_UpConn_PHY(Module, AutoCSR):
-    def __init__(self, pad, sys_clk_freq, debug_sma, pmod_pads, nsink=3):
+    def __init__(self, pad, sys_clk_freq, debug_sma, pmod_pads, layout, nsink=3):
         self.bitrate2x_enable = Signal()
         self.clk_reset = Signal()
 
@@ -304,18 +311,18 @@ class CXP_UpConn_PHY(Module, AutoCSR):
         # # #
 
         self.submodules.cg = cg = UpConn_ClockGen(sys_clk_freq)
-        self.submodules.interface = interface = PHY_Interface(upconn_layout, nsink)
+        self.submodules.interface = interface = PHY_Interface(layout, nsink)
 
         self.sinks = interface.sinks
 
         # DEBUG:
-        self.submodules.debug_buf = debug_buf = Debug_buffer(upconn_layout)
+        self.submodules.debug_buf = debug_buf = Debug_buffer(layout)
 
-        self.submodules.scheduler = scheduler = Transmit_Scheduler(interface, debug_buf)
+        self.submodules.scheduler = scheduler = Packets_Scheduler(interface, debug_buf)
         self.submodules.serdes = serdes = SERDES_10bits(pad)
 
         self.comb += [
-            self.tx_busy.eq(~interface.pe.n),
+            self.tx_busy.eq(interface.source_stb != 0),
 
             cg.reset.eq(self.clk_reset),
             cg.freq2x_enable.eq(self.bitrate2x_enable),
@@ -348,8 +355,8 @@ class CXP_UpConn_PHY(Module, AutoCSR):
         ] 
         self.specials += [
             # # debug sma
-            Instance("OBUF", i_I=serdes.o, o_O=debug_sma.p_tx),
-            Instance("OBUF", i_I=cg.clk_10x, o_O=debug_sma.n_rx),
+            # Instance("OBUF", i_I=cg.clk, o_O=debug_sma.p_tx),
+            # Instance("OBUF", i_I=cg.clk_10x, o_O=debug_sma.n_rx),
 
 
 

src/libboard_artiq/src/cxp_downconn.rs

@@ -26,25 +26,25 @@ pub fn loopback_testing(timer: &mut GlobalTimer, speed: CXP_SPEED) {
         timer.delay_us(50_000);
         info!(
             "tx_phaligndone = {} | rx_phaligndone = {}",
-            csr::cxp::downconn_phy_txinit_phaligndone_read(),
-            csr::cxp::downconn_phy_rxinit_phaligndone_read(),
+            csr::cxp::downconn_txinit_phaligndone_read(),
+            csr::cxp::downconn_rxinit_phaligndone_read(),
         );
 
         // enable txdata tranmission thought MGTXTXP, required by PMA loopback
-        csr::cxp::downconn_phy_txenable_write(1);
+        csr::cxp::downconn_txenable_write(1);
 
         info!("waiting for rx to align...");
-        while csr::cxp::downconn_phy_rx_ready_read() != 1 {}
+        while csr::cxp::downconn_rx_ready_read() != 1 {}
         info!("rx ready!");
 
         loop {
             // for _ in 0..20 {
             // NOTE: raw bits
-            // let data0 = csr::cxp::downconn_phy_rxdata_0_read();
-            // let data1 = csr::cxp::downconn_phy_rxdata_1_read();
-            // let data2 = csr::cxp::downconn_phy_rxdata_2_read();
-            // let data3 = csr::cxp::downconn_phy_rxdata_3_read();
-            // let rxready = csr::cxp::downconn_phy_rx_ready_read();
+            // let data0 = csr::cxp::downconn_rxdata_0_read();
+            // let data1 = csr::cxp::downconn_rxdata_1_read();
+            // let data2 = csr::cxp::downconn_rxdata_2_read();
+            // let data3 = csr::cxp::downconn_rxdata_3_read();
+            // let rxready = csr::cxp::downconn_rx_ready_read();
             // timer.delay_us(100);
             // if data0 == 0b0101111100 || data0 == 0b1010000011 {
             //     println!(
@@ -72,24 +72,24 @@ pub fn loopback_testing(timer: &mut GlobalTimer, speed: CXP_SPEED) {
 
             timer.delay_us(1_000_000);
             // NOTE: raw bits
-            // let data0 = csr::cxp::downconn_phy_rxdata_0_read();
-            // let data1 = csr::cxp::downconn_phy_rxdata_1_read();
-            // let data2 = csr::cxp::downconn_phy_rxdata_2_read();
-            // let data3 = csr::cxp::downconn_phy_rxdata_3_read();
+            // let data0 = csr::cxp::downconn_rxdata_0_read();
+            // let data1 = csr::cxp::downconn_rxdata_1_read();
+            // let data2 = csr::cxp::downconn_rxdata_2_read();
+            // let data3 = csr::cxp::downconn_rxdata_3_read();
             // println!(
             //     "0b{:010b} {:010b} {:010b} {:010b}",
             //     data0, data1, data2, data3
             // );
 
             // NOTE:decode data
-            // let data0_k = csr::cxp::downconn_phy_decoded_k_0_read();
-            // let data1_k = csr::cxp::downconn_phy_decoded_k_1_read();
-            // let data2_k = csr::cxp::downconn_phy_decoded_k_2_read();
-            // let data3_k = csr::cxp::downconn_phy_decoded_k_3_read();
-            let data0_decoded = csr::cxp::downconn_phy_decoded_data_0_read();
-            let data1_decoded = csr::cxp::downconn_phy_decoded_data_1_read();
-            let data2_decoded = csr::cxp::downconn_phy_decoded_data_2_read();
-            let data3_decoded = csr::cxp::downconn_phy_decoded_data_3_read();
+            // let data0_k = csr::cxp::downconn_decoded_k_0_read();
+            // let data1_k = csr::cxp::downconn_decoded_k_1_read();
+            // let data2_k = csr::cxp::downconn_decoded_k_2_read();
+            // let data3_k = csr::cxp::downconn_decoded_k_3_read();
+            let data0_decoded = csr::cxp::downconn_decoded_data_0_read();
+            let data1_decoded = csr::cxp::downconn_decoded_data_1_read();
+            let data2_decoded = csr::cxp::downconn_decoded_data_2_read();
+            let data3_decoded = csr::cxp::downconn_decoded_data_3_read();
             println!(
                 "{:#04x} {:#04x} {:#04x} {:#04x}",
                 data0_decoded, data1_decoded, data2_decoded, data3_decoded,
@@ -108,24 +108,24 @@ pub fn loopback_testing(timer: &mut GlobalTimer, speed: CXP_SPEED) {
 pub fn setup(timer: &mut GlobalTimer) {
     unsafe {
         info!("turning on pmc loopback mode...");
-        csr::cxp::downconn_phy_loopback_mode_write(0b010); // Near-End PMA Loopback
+        csr::cxp::downconn_loopback_mode_write(0b010); // Near-End PMA Loopback
 
         // QPLL setup
-        csr::cxp::downconn_phy_qpll_reset_write(1);
+        csr::cxp::downconn_qpll_reset_write(1);
         info!("waiting for QPLL/CPLL to lock...");
-        while csr::cxp::downconn_phy_qpll_locked_read() != 1 {}
+        while csr::cxp::downconn_qpll_locked_read() != 1 {}
         info!("QPLL locked");
 
         // tx/rx setup
-        csr::cxp::downconn_phy_tx_start_init_write(1);
-        csr::cxp::downconn_phy_rx_start_init_write(1);
+        csr::cxp::downconn_tx_start_init_write(1);
+        csr::cxp::downconn_rx_start_init_write(1);
 
         info!("waiting for tx & rx setup...");
         timer.delay_us(50_000);
         info!(
             "tx_phaligndone = {} | rx_phaligndone = {}",
-            csr::cxp::downconn_phy_txinit_phaligndone_read(),
-            csr::cxp::downconn_phy_rxinit_phaligndone_read(),
+            csr::cxp::downconn_txinit_phaligndone_read(),
+            csr::cxp::downconn_rxinit_phaligndone_read(),
         );
     }
 
@@ -153,15 +153,15 @@ pub mod cxp_gtx {
         change_cdr_cfg(speed);
 
         unsafe {
-            csr::cxp::downconn_phy_qpll_reset_write(1);
+            csr::cxp::downconn_qpll_reset_write(1);
             info!("waiting for QPLL/CPLL to lock...");
-            while csr::cxp::downconn_phy_qpll_locked_read() != 1 {}
+            while csr::cxp::downconn_qpll_locked_read() != 1 {}
             info!("QPLL locked");
         }
 
         unsafe {
-            csr::cxp::downconn_phy_tx_restart_write(1);
-            csr::cxp::downconn_phy_rx_restart_write(1);
+            csr::cxp::downconn_tx_restart_write(1);
+            csr::cxp::downconn_rx_restart_write(1);
         }
     }
 
@@ -245,19 +245,19 @@ pub mod cxp_gtx {
     fn gtx_read(address: u16) -> u16 {
         // DEBUG:
         unsafe {
-            csr::cxp::downconn_phy_gtx_daddr_write(address);
-            csr::cxp::downconn_phy_gtx_dread_write(1);
-            while csr::cxp::downconn_phy_gtx_dready_read() != 1 {}
-            csr::cxp::downconn_phy_gtx_dout_read()
+            csr::cxp::downconn_gtx_daddr_write(address);
+            csr::cxp::downconn_gtx_dread_write(1);
+            while csr::cxp::downconn_gtx_dready_read() != 1 {}
+            csr::cxp::downconn_gtx_dout_read()
         }
     }
 
     fn gtx_write(address: u16, value: u16) {
         unsafe {
-            csr::cxp::downconn_phy_gtx_daddr_write(address);
-            csr::cxp::downconn_phy_gtx_din_write(value);
-            csr::cxp::downconn_phy_gtx_din_stb_write(1);
-            while csr::cxp::downconn_phy_gtx_dready_read() != 1 {}
+            csr::cxp::downconn_gtx_daddr_write(address);
+            csr::cxp::downconn_gtx_din_write(value);
+            csr::cxp::downconn_gtx_din_stb_write(1);
+            while csr::cxp::downconn_gtx_dready_read() != 1 {}
         }
     }
 
@@ -265,19 +265,19 @@ pub mod cxp_gtx {
     fn qpll_read(address: u8) -> u16 {
         // DEBUG:
         unsafe {
-            csr::cxp::downconn_phy_qpll_daddr_write(address);
-            csr::cxp::downconn_phy_qpll_dread_write(1);
-            while csr::cxp::downconn_phy_qpll_dready_read() != 1 {}
-            csr::cxp::downconn_phy_qpll_dout_read()
+            csr::cxp::downconn_qpll_daddr_write(address);
+            csr::cxp::downconn_qpll_dread_write(1);
+            while csr::cxp::downconn_qpll_dready_read() != 1 {}
+            csr::cxp::downconn_qpll_dout_read()
         }
     }
 
     fn qpll_write(address: u8, value: u16) {
         unsafe {
-            csr::cxp::downconn_phy_qpll_daddr_write(address);
-            csr::cxp::downconn_phy_qpll_din_write(value);
-            csr::cxp::downconn_phy_qpll_din_stb_write(1);
-            while csr::cxp::downconn_phy_qpll_dready_read() != 1 {}
+            csr::cxp::downconn_qpll_daddr_write(address);
+            csr::cxp::downconn_qpll_din_write(value);
+            csr::cxp::downconn_qpll_din_stb_write(1);
+            while csr::cxp::downconn_qpll_dready_read() != 1 {}
         }
     }
 }
@@ -301,43 +301,43 @@ pub mod txusrclk {
 
     fn one_clock_cycle() {
         unsafe {
-            csr::cxp::downconn_phy_pll_dclk_write(1);
-            csr::cxp::downconn_phy_pll_dclk_write(0);
+            csr::cxp::downconn_pll_dclk_write(1);
+            csr::cxp::downconn_pll_dclk_write(0);
         }
     }
 
     fn set_addr(address: u8) {
         unsafe {
-            csr::cxp::downconn_phy_pll_daddr_write(address);
+            csr::cxp::downconn_pll_daddr_write(address);
         }
     }
 
     fn set_data(value: u16) {
         unsafe {
-            csr::cxp::downconn_phy_pll_din_write(value);
+            csr::cxp::downconn_pll_din_write(value);
         }
     }
 
     fn set_enable(en: bool) {
         unsafe {
             let val = if en { 1 } else { 0 };
-            csr::cxp::downconn_phy_pll_den_write(val);
+            csr::cxp::downconn_pll_den_write(val);
         }
     }
 
     fn set_write_enable(en: bool) {
         unsafe {
             let val = if en { 1 } else { 0 };
-            csr::cxp::downconn_phy_pll_dwen_write(val);
+            csr::cxp::downconn_pll_dwen_write(val);
         }
     }
 
     fn get_data() -> u16 {
-        unsafe { csr::cxp::downconn_phy_pll_dout_read() }
+        unsafe { csr::cxp::downconn_pll_dout_read() }
     }
 
     fn drp_ready() -> bool {
-        unsafe { csr::cxp::downconn_phy_pll_dready_read() == 1 }
+        unsafe { csr::cxp::downconn_pll_dready_read() == 1 }
     }
 
     #[allow(dead_code)]
@@ -374,7 +374,7 @@ pub mod txusrclk {
     fn reset(rst: bool) {
         unsafe {
             let val = if rst { 1 } else { 0 };
-            csr::cxp::downconn_phy_txpll_reset_write(val)
+            csr::cxp::downconn_txpll_reset_write(val)
         }
     }
 
@@ -411,7 +411,7 @@ pub mod txusrclk {
             // wait for the pll to lock
             timer.delay_us(100);
 
-            let locked = unsafe { csr::cxp::downconn_phy_txpll_locked_read() == 1 };
+            let locked = unsafe { csr::cxp::downconn_txpll_locked_read() == 1 };
             info!("txusrclk locked = {}", locked);
         }
     }
@@ -540,5 +540,3 @@ pub mod txusrclk {
         }
     }
 }
-
-// TODO: add recv like in drtioaux

src/libboard_artiq/src/cxp_proto.rs

@@ -1,193 +0,0 @@
-use core_io::{Error as IoError, Write};
-use crc::crc32;
-use embedded_hal::prelude::_embedded_hal_blocking_delay_DelayUs;
-use io::Cursor;
-use libboard_zynq::{println, timer::GlobalTimer};
-
-use crate::pl::csr;
-
-const MAX_PACKET: usize = 128;
-const DATA_MAXSIZE: usize = /*max size*/MAX_PACKET - /*Tag*/4 - /*Op code & length*/4 - /*addr*/4 - /*CRC*/4 ;
-
-#[derive(Debug)]
-pub enum Error {
-    BufferError,
-    LinkDown,
-}
-
-impl From<IoError> for Error {
-    fn from(_: IoError) -> Error {
-        Error::BufferError
-    }
-}
-
-pub enum Packet {
-    CtrlRead {
-        addr: u32,
-        length: u8,
-    },
-    CtrlWrite {
-        addr: u32,
-        length: u8,
-        data: [u8; DATA_MAXSIZE],
-    }, // max register size is 8 bytes
-    CtrlReadWithTag {
-        tag: u8,
-        addr: u32,
-        length: u8,
-    },
-    CtrlWriteWithTag {
-        tag: u8,
-        addr: u32,
-        length: u8,
-        data: [u8; DATA_MAXSIZE],
-    }, // max register size is 8 bytes
-    EventAck {
-        packet_tag: u8,
-    },
-    TestPacket,
-}
-
-impl Packet {
-    pub fn write_to(&self, writer: &mut Cursor<&mut [u8]>) -> Result<(), Error> {
-        // CoaXpress use big endian
-        match *self {
-            Packet::CtrlRead { addr, length } => {
-                writer.write(&[0x02; 4])?;
-                writer.write(&[0x00, 0x00, 0x00, length])?;
-                writer.write(&addr.to_be_bytes())?;
-            }
-            Packet::CtrlWrite { addr, length, data } => {
-                writer.write(&[0x02; 4])?;
-                writer.write(&[0x01, 0x00, 0x00, length])?;
-                writer.write(&addr.to_be_bytes())?;
-                writer.write(&data[0..length as usize])?;
-            }
-            Packet::CtrlReadWithTag { tag, addr, length } => {
-                writer.write(&[0x05; 4])?;
-                writer.write(&[tag; 4])?;
-                writer.write(&[0x00, 0x00, 0x00, length])?;
-                writer.write(&addr.to_be_bytes())?;
-            }
-            Packet::CtrlWriteWithTag {
-                tag,
-                addr,
-                length,
-                data,
-            } => {
-                writer.write(&[0x05; 4])?;
-                writer.write(&[tag; 4])?;
-                writer.write(&[0x01, 0x00, 0x00, length])?;
-                writer.write(&addr.to_be_bytes())?;
-                writer.write(&data[0..length as usize])?;
-            }
-            Packet::EventAck { packet_tag } => {
-                writer.write(&[0x08; 4])?;
-                writer.write(&[packet_tag; 4])?;
-            }
-            _ => {}
-        }
-        // Section 9.2.2.2 (CXP-001-2021)
-        // Only Control packet need CRC32 appended in the end of the packet
-        // CoaXpress use the polynomial of IEEE-802.3 (Ethernet) CRC but the checksum calculation is different
-        // Also, the calculation does not include the first 4 bytes of packet_type
-        match *self {
-            Packet::CtrlRead { .. }
-            | Packet::CtrlWrite { .. }
-            | Packet::CtrlReadWithTag { .. }
-            | Packet::CtrlWriteWithTag { .. } => {
-                let checksum = crc32::checksum_ieee(&writer.get_ref()[4..writer.position()]);
-                writer.write(&(!checksum).to_le_bytes())?;
-            }
-            _ => {}
-        }
-        Ok(())
-    }
-}
-
-pub fn send(packet: &Packet) -> Result<(), Error> {
-    if unsafe { csr::cxp::upconn_tx_enable_read() } == 0 {
-        Err(Error::LinkDown)?
-    }
-
-    match *packet {
-        Packet::TestPacket => send_test_packet(),
-        _ => send_data_packet(packet),
-    }
-}
-
-fn send_data_packet(packet: &Packet) -> Result<(), Error> {
-    let mut buffer: [u8; MAX_PACKET] = [0; MAX_PACKET];
-    let mut writer = Cursor::new(&mut buffer[..]);
-
-    packet.write_to(&mut writer)?;
-
-    unsafe {
-        let len = writer.position();
-        csr::cxp::upconn_command_len_write(len as u8);
-        for data in writer.get_ref()[..len].iter() {
-            while csr::cxp::upconn_command_writeable_read() == 0 {}
-            csr::cxp::upconn_command_data_write(*data);
-        }
-    }
-    Ok(())
-}
-
-fn send_test_packet() -> Result<(), Error> {
-    unsafe {
-        while csr::cxp::upconn_tx_busy_read() == 1 {}
-        csr::cxp::upconn_tx_testmode_en_write(1);
-        // timer.delay_us(2);
-        csr::cxp::upconn_testseq_stb_write(1);
-        while csr::cxp::upconn_testseq_busy_read() == 1 {}
-        csr::cxp::upconn_tx_testmode_en_write(0);
-    }
-    Ok(())
-}
-
-pub fn write_u32(addr: u32, data: u32) -> Result<(), Error> {
-    let mut data_slice: [u8; DATA_MAXSIZE] = [0; DATA_MAXSIZE];
-    data_slice[..4].clone_from_slice(&data.to_be_bytes());
-    send(&Packet::CtrlWrite {
-        addr,
-        length: 4,
-        data: data_slice,
-    })?;
-
-    Ok(())
-}
-
-pub fn read_u32(addr: u32) -> Result<(), Error> {
-    send(&Packet::CtrlRead { addr, length: 4 })?;
-
-    Ok(())
-}
-
-pub fn write_u64(addr: u32, data: u64) -> Result<(), Error> {
-    let mut data_slice: [u8; DATA_MAXSIZE] = [0; DATA_MAXSIZE];
-    data_slice[..8].clone_from_slice(&data.to_be_bytes());
-    send(&Packet::CtrlWrite {
-        addr,
-        length: 8,
-        data: data_slice,
-    })?;
-
-    Ok(())
-}
-
-// DEBUG: use only
-pub fn print_packet(pak: &[u8]) {
-    println!("pak = [");
-    for i in 0..(pak.len() / 4) {
-        println!(
-            "{:#03}     {:#04X} {:#04X} {:#04X} {:#04X},",
-            i + 1,
-            pak[i * 4],
-            pak[i * 4 + 1],
-            pak[i * 4 + 2],
-            pak[i * 4 + 3]
-        )
-    }
-    println!("]");
-    println!("============================================");
-}

src/libboard_artiq/src/cxp_upconn.rs

@@ -4,11 +4,22 @@ use embedded_hal::prelude::_embedded_hal_blocking_delay_DelayUs;
 use io::Cursor;
 use libboard_zynq::{println, timer::GlobalTimer};
 
-pub use crate::cxp_proto;
 use crate::pl::csr;
 
+#[derive(Debug)]
+pub enum Error {
+    BufferError,
+    LinkDown,
+}
+
+impl From<IoError> for Error {
+    fn from(_: IoError) -> Error {
+        Error::BufferError
+    }
+}
+
 pub fn tx_test(timer: &mut GlobalTimer) {
-    const LEN: usize = 4 * 50;
+    const LEN: usize = 4 * 20;
     let mut pak_arr: [u8; LEN] = [0; LEN];
 
     unsafe {
@@ -16,36 +27,154 @@ pub fn tx_test(timer: &mut GlobalTimer) {
         // csr::cxp::upconn_bitrate2x_enable_write(1);
         csr::cxp::upconn_clk_reset_write(0);
 
-        // read_u32(0x00).expect("Cannot Write CoaXpress Register");
-        csr::cxp::upconn_tx_enable_write(1);
-        timer.delay_us(2); // send one word
-        cxp_proto::send(&cxp_proto::Packet::EventAck { packet_tag: 0x04 }).expect("Cannot send CoaXpress packet");
-        // cxp_proto::send(&cxp_proto::Packet::TestPacket).expect("Cannot send CoaXpress packet");
+        send(&Packet::ControlU32Reg(Command::Read { addr: 0x00 })).expect("Cannot send CoaXpress packet");
 
-        // timer.delay_us(2);
+        csr::cxp::upconn_tx_enable_write(1);
+
+        timer.delay_us(2);
         // DEBUG: Trigger packet
-        // let linktrig_mode: u8 = 0x01;
-        // csr::cxp::upconn_trig_delay_write(0x05);
-        // csr::cxp::upconn_linktrigger_write(linktrig_mode);
-        // csr::cxp::upconn_trig_stb_write(1); // send trig
+        let linktrig_mode: u8 = 0x01;
+        csr::cxp::upconn_trig_delay_write(0x05);
+        csr::cxp::upconn_linktrigger_write(linktrig_mode);
+        csr::cxp::upconn_trig_stb_write(1); // send trig
 
         // DEBUG: Trigger ACK packet
         // csr::cxp::upconn_ack_write(1);
         timer.delay_us(20);
+
         csr::cxp::upconn_tx_enable_write(0);
 
         // Collect data
         let mut i: usize = 0;
-        while csr::cxp::upconn_phy_debug_buf_dout_valid_read() == 1 {
-            pak_arr[i] = csr::cxp::upconn_phy_debug_buf_dout_pak_read();
+        while csr::cxp::upconn_upconn_phy_debug_buf_dout_valid_read() == 1 {
+            pak_arr[i] = csr::cxp::upconn_upconn_phy_debug_buf_dout_pak_read();
             // println!("received {:#04X}", pak_arr[i]);
-            csr::cxp::upconn_phy_debug_buf_inc_write(1);
+            csr::cxp::upconn_upconn_phy_debug_buf_inc_write(1);
             i += 1;
             if i == LEN {
                 break;
             }
         }
 
-        cxp_proto::print_packet(&pak_arr);
+        print_packet(&pak_arr);
     }
 }
+
+pub enum Command<T> {
+    Read { addr: u32 },
+    Write { addr: u32, data: T },
+    ReadWithTag { addr: u32, tag: u8 },
+    WriteWithTag { addr: u32, data: T, tag: u8 },
+}
+
+pub enum Packet {
+    ControlU32Reg(Command<u32>),
+    ControlU64Reg(Command<u64>),
+}
+
+impl Packet {
+    pub fn write_to<W>(&self, writer: &mut W) -> Result<(), Error>
+    where W: Write {
+        match self {
+            Packet::ControlU32Reg(cmd) => match cmd {
+                Command::Read { addr } => {
+                    writer.write(&[0x02; 4])?;
+                    writer.write(&[0x00, 0x00, 0x00, 0x04])?;
+                    writer.write(&addr.to_be_bytes())?;
+                }
+                Command::Write { addr, data } => {
+                    writer.write(&[0x02; 4])?;
+                    writer.write(&[0x01, 0x00, 0x00, 0x04])?;
+                    writer.write(&addr.to_be_bytes())?;
+                    writer.write(&data.to_be_bytes())?;
+                }
+                Command::ReadWithTag { addr, tag } => {
+                    writer.write(&[0x05; 4])?;
+                    writer.write(&[*tag; 4])?;
+                    writer.write(&[0x00, 0x00, 0x00, 0x04])?;
+                    writer.write(&addr.to_be_bytes())?;
+                }
+                Command::WriteWithTag { addr, data, tag } => {
+                    writer.write(&[0x05; 4])?;
+                    writer.write(&[*tag; 4])?;
+                    writer.write(&[0x01, 0x00, 0x00, 0x04])?;
+                    writer.write(&addr.to_be_bytes())?;
+                    writer.write(&data.to_be_bytes())?;
+                }
+            },
+            Packet::ControlU64Reg(cmd) => match cmd {
+                Command::Read { addr } => {
+                    writer.write(&[0x02; 4])?;
+                    writer.write(&[0x00, 0x00, 0x00, 0x08])?;
+                    writer.write(&addr.to_be_bytes())?;
+                }
+                Command::Write { addr, data } => {
+                    writer.write(&[0x02; 4])?;
+                    writer.write(&[0x01, 0x00, 0x00, 0x08])?;
+                    writer.write(&addr.to_be_bytes())?;
+                    writer.write(&data.to_be_bytes())?;
+                }
+                Command::ReadWithTag { addr, tag } => {
+                    writer.write(&[0x05; 4])?;
+                    writer.write(&[*tag; 4])?;
+                    writer.write(&[0x00, 0x00, 0x00, 0x08])?;
+                    writer.write(&addr.to_be_bytes())?;
+                }
+                Command::WriteWithTag { addr, data, tag } => {
+                    writer.write(&[0x05; 4])?;
+                    writer.write(&[*tag; 4])?;
+                    writer.write(&[0x01, 0x00, 0x00, 0x08])?;
+                    writer.write(&addr.to_be_bytes())?;
+                    writer.write(&data.to_be_bytes())?;
+                }
+            },
+        }
+        Ok(())
+    }
+}
+
+pub fn send(packet: &Packet) -> Result<(), Error> {
+    // DEBUG: remove the comment out section
+    // if unsafe { csr::cxp::upconn_tx_enable_read() } == 0 {
+    //     Err(Error::LinkDown)?
+    // }
+
+    const LEN: usize = 4 * 20;
+    let mut buffer: [u8; LEN] = [0; LEN];
+    let mut writer = Cursor::new(&mut buffer[..]);
+
+    packet.write_to(&mut writer)?;
+
+    // Section 9.2.2.2 (CXP-001-2021)
+    // CoaXpress use the polynomial of IEEE-802.3 (Ethernet) CRC but the checksum calculation is different
+    // Also, the calculation does not include the first 4 bytes of packet_type
+    let checksum = crc32::checksum_ieee(&writer.get_ref()[4..writer.position()]);
+    writer.write(&(!checksum).to_le_bytes())?;
+
+    unsafe {
+        let len = writer.position();
+        csr::cxp::upconn_command_len_write(len as u8);
+        for data in writer.get_ref()[..len].iter() {
+            while csr::cxp::upconn_command_writeable_read() == 0 {}
+            csr::cxp::upconn_command_data_write(*data);
+        }
+    }
+
+    Ok(())
+}
+
+fn print_packet(pak: &[u8]) {
+    println!("pak = [");
+    for i in 0..(pak.len() / 4) {
+        println!(
+            "{:#03}     {:#04X} {:#04X} {:#04X} {:#04X},",
+            i + 1,
+            pak[i * 4],
+            pak[i * 4 + 1],
+            pak[i * 4 + 2],
+            pak[i * 4 + 3]
+        )
+    }
+    println!("]");
+    println!("============================================");
+}

src/libboard_artiq/src/lib.rs

@@ -47,8 +47,6 @@ pub mod cxp_downconn;
 #[cfg(has_cxp)]
 pub mod cxp_upconn;
 
-pub mod cxp_proto;
-
 pub fn identifier_read(buf: &mut [u8]) -> &str {
     unsafe {
         pl::csr::identifier::address_write(0);

src/runtime/src/main.rs

@@ -150,16 +150,5 @@ pub fn main_core0() {
 
     task::spawn(ksupport::report_async_rtio_errors());
 
-    use libboard_artiq::{cxp_downconn, cxp_upconn};
-    cxp_downconn::setup(&mut timer);
-    // cxp_downconn::loopback_testing(&mut timer, cxp_downconn::CXP_SPEED::CXP_1);
-    // cxp_downconn::loopback_testing(&mut timer, cxp_downconn::CXP_SPEED::CXP_2);
-    // cxp_downconn::loopback_testing(&mut timer, cxp_downconn::CXP_SPEED::CXP_3);
-    // cxp_downconn::loopback_testing(&mut timer, cxp_downconn::CXP_SPEED::CXP_5);
-    // cxp_downconn::loopback_testing(&mut timer, cxp_downconn::CXP_SPEED::CXP_6);
-    // cxp_downconn::loopback_testing(&mut timer, cxp_downconn::CXP_SPEED::CXP_10);
-    // cxp_downconn::loopback_testing(&mut timer, cxp_downconn::CXP_SPEED::CXP_12);
-    cxp_upconn::tx_test(&mut timer);
-
     comms::main(timer, cfg);
 }