cxp coredevice driver: init

zc706: update fn names

zc706: use cxp_gtx_rxi clk

cxp_kernel.py

@@ -0,0 +1,58 @@
+"""
+Non-realtime drivers for CXP.
+"""
+
+# TODO: add api calls for CTRL packet similar i2c
+# TODO: add timing critical trigger ack
+
+
+from artiq.language.core import syscall, kernel
+from artiq.language.types import TBool, TInt32, TNone
+from artiq.coredevice.rtio import rtio_output
+from artiq.experiment import *
+
+
+class CoaXPress:
+    def __init__(self, channel, core_device="core"):
+        # __device_mgr is private 
+        # self.core = dmgr.get(core_device)
+
+        # you can get the channel via `print(len(rtio_channels))` before calling
+        # `rtio_channels.append(rtio.Channel.from_phy(cxp_interface))`
+        self.channel = channel
+        # the first 8 bits is reserved for the rtlink.OInterface.addr not for channel no.
+        self.target_o = channel << 8
+
+    @staticmethod
+    def get_rtio_channels(channel, **kwargs):
+        return [(channel, None)]
+
+    @kernel
+    def trigger(self, linktrig, trigdelay):
+        rtio_output(self.target_o, linktrig | trigdelay << 1)
+
+
+@syscall(flags={"nounwind", "nowrite"})
+def cxp_readu32(channel: TInt32, addr: TInt32) -> TNone:
+    raise NotImplementedError("syscall not simulated")
+
+@syscall(flags={"nounwind", "nowrite"})
+def cxp_writeu32(channel: TInt32, addr: TInt32, val: TInt32) -> TNone:
+    raise NotImplementedError("syscall not simulated")
+
+
+
+class IdleKernel(EnvExperiment):
+    def build(self):
+        self.setattr_device("core")
+        self.setattr_device("led0")
+
+        # declare the class before using it in kernel
+        self.cxp = CoaXPress(0x0)
+
+    @kernel
+    def run(self):
+        self.core.reset()
+        # cxp_readu32(0, 3)
+        # cxp_writeu32(0, 0, 0xABCD)
+        self.cxp.trigger(1, 10)

src/gateware/cxp.py

@@ -2,6 +2,8 @@ from migen import *
 from migen.genlib.cdc import MultiReg, PulseSynchronizer
 from misoc.interconnect.csr import *
 
+from artiq.gateware.rtio import rtlink
+
 from cxp_downconn import CXP_DownConn_PHYS
 from cxp_upconn import CXP_UpConn_PHYS
 from cxp_pipeline import *
@@ -10,16 +12,14 @@ from cxp_pipeline import *
 class CXP_PHYS(Module, AutoCSR):
     def __init__(self, refclk, upconn_pads, downconn_pads, sys_clk_freq, debug_sma, pmod_pads):
         assert len(upconn_pads) == len(downconn_pads)
+
         self.submodules.upconn = CXP_UpConn_PHYS(upconn_pads, sys_clk_freq, debug_sma, pmod_pads)
         self.submodules.downconn = CXP_DownConn_PHYS(refclk, downconn_pads, sys_clk_freq, debug_sma, pmod_pads)
 
 @FullMemoryWE()
 class CXP_Interface(Module, AutoCSR):
     def __init__(self, upconn_phy, downconn_phy, debug_sma, pmod_pads):
-        # TODO: add rtio interface io 
-        
         self.submodules.upconn = UpConn_Interface(upconn_phy, debug_sma, pmod_pads)
-
         self.submodules.downconn = DownConn_Interface(downconn_phy, debug_sma, pmod_pads)
 
     def get_tx_port(self):
@@ -43,6 +43,34 @@ class CXP_Interface(Module, AutoCSR):
     def get_loopback_tx_mem_size(self):
         return self.downconn.bootstrap_loopback.mem.depth*self.downconn.bootstrap_loopback.mem.width // 8
 
+class CXP_Master(CXP_Interface):
+    def __init__(self, upconn_phy, downconn_phy, debug_sma, pmod_pads):
+        CXP_Interface.__init__(self, upconn_phy, downconn_phy, debug_sma, pmod_pads)
+        nbit_trigdelay = 8
+        nbit_linktrig = 1
+
+        self.rtlink = rtlink.Interface(
+            rtlink.OInterface(nbit_trigdelay + nbit_linktrig),
+            rtlink.IInterface(word_dw, timestamped=False)
+        )
+
+        self.sync.rio += [
+            If(self.rtlink.o.stb,
+                self.upconn.trig.delay.eq(self.rtlink.o.data[nbit_linktrig:]),
+                self.upconn.trig.linktrig_mode.eq(self.rtlink.o.data[:nbit_linktrig]),
+            ),
+            self.upconn.trig.stb.eq(self.rtlink.o.stb),
+        ]
+
+        # DEBUG: out
+        self.specials += Instance("OBUF", i_I=self.rtlink.o.stb, o_O=debug_sma.p_tx),
+        # self.specials += Instance("OBUF", i_I=self.rtlink.o.stb, o_O=debug_sma.n_rx),
+
+class CXP_Extension(CXP_Interface):
+    def __init__(self, upconn_phy, downconn_phy, debug_sma, pmod_pads):
+        CXP_Interface.__init__(self, upconn_phy, downconn_phy, debug_sma, pmod_pads)
+
+
 class DownConn_Interface(Module, AutoCSR):
     def __init__(self, phy, debug_sma, pmod_pads):
         self.rx_start_init = CSRStorage()
@@ -95,22 +123,14 @@ class DownConn_Interface(Module, AutoCSR):
 
         self.comb += gtx.dclk.eq(ClockSignal("sys"))
         self.sync += [
-            gtx.den.eq(0),
-            gtx.dwen.eq(0),
-            If(self.gtx_dread.re,
-                gtx.den.eq(1),
-                gtx.daddr.eq(self.gtx_daddr.storage),
-            ).Elif(self.gtx_din_stb.re,
-                gtx.den.eq(1),
-                gtx.dwen.eq(1),
             gtx.daddr.eq(self.gtx_daddr.storage),
+            gtx.den.eq(self.gtx_dread.re | self.gtx_din_stb.re),
+            gtx.dwen.eq(self.gtx_din_stb.re),
             gtx.din.eq(self.gtx_din.storage),
-            ),
             If(gtx.dready,
                 self.gtx_dready.w.eq(1),
                 self.gtx_dout.status.eq(gtx.dout),
-            ),
+            ).Elif(self.gtx_dready.re,
-            If(self.gtx_dready.re,
                 self.gtx_dready.w.eq(0),
             ),
         ]
@@ -254,7 +274,7 @@ class DownConn_Interface(Module, AutoCSR):
         ]
         self.sync += [
             self.pending_packet.w.eq(self.read_ptr.status != bootstrap.write_ptr_sys),
-            If(self.rx_restart.re,
+            If(~gtx.rx_ready,
                 self.read_ptr.status.eq(0),
             ).Elif(self.pending_packet.re & self.pending_packet.w, 
                 self.read_ptr.status.eq(self.read_ptr.status + 1),
@@ -296,7 +316,7 @@ class DownConn_Interface(Module, AutoCSR):
         ]
 
         self.specials += [
-            Instance("OBUF", i_I=phy.gtx.cd_cxp_gtx_rx.clk, o_O=debug_sma.p_tx),
+            # Instance("OBUF", i_I=phy.gtx.cd_cxp_gtx_rx.clk, o_O=debug_sma.p_tx),
             # Instance("OBUF", i_I=, o_O=debug_sma.p_rx),
             # # pmod 0-7 pin
             Instance("OBUF", i_I=bootstrap.test_err, o_O=pmod_pads[0]),
@@ -312,20 +332,6 @@ class DownConn_Interface(Module, AutoCSR):
         
 class UpConn_Interface(Module, AutoCSR):
     def __init__(self, phy, debug_sma, pmod_pads):
-        self.clk_reset = CSRStorage(reset=1)
-        self.bitrate2x_enable = CSRStorage()
-        self.tx_enable = CSRStorage()
-        self.tx_mux = CSRStorage()
-
-        # # #
-
-        self.sync += [
-            phy.bitrate2x_enable.eq(self.bitrate2x_enable.storage),
-            phy.tx_enable.eq(self.tx_enable.storage),
-            phy.clk_reset.eq(self.clk_reset.re),
-        ]
-
-
         # Transmission Pipeline
         #
         #              32                                                  32           8
@@ -350,13 +356,13 @@ class UpConn_Interface(Module, AutoCSR):
         # # DEBUG: INPUT
         self.trig_stb = CSR()
         self.trig_delay = CSRStorage(8) 
-        self.linktrigger = CSRStorage(2)
+        self.linktrigger = CSRStorage()
 
-        self.sync += [
+        # self.sync += [
-            trig.stb.eq(self.trig_stb.re),
+        #     trig.stb.eq(self.trig_stb.re),
-            trig.delay.eq(self.trig_delay.storage),
+        #     trig.delay.eq(self.trig_delay.storage),
-            trig.linktrig_mode.eq(self.linktrigger.storage),
+        #     trig.linktrig_mode.eq(self.linktrigger.storage),
-        ]
+        # ]
 
         
         # 1: IO acknowledgment for trigger packet

src/gateware/cxp_downconn.py

@@ -40,6 +40,15 @@ class CXP_DownConn_PHYS(Module, AutoCSR):
         # checkout channel interfaces & drtio_gtx
         # GTPTXPhaseAlignement for inspiration
 
+        # Connect slave i's `cxp_gtx_rx` clock to `cxp_gtx_rxi` clock
+        for rx in self.rx_phys:
+            name = "cd_cxp_gtx_rx" + str(i)
+            setattr(self.clock_domains, name, ClockDomain(name=name))
+            self.comb += [
+                getattr(self, name).clk.eq(rx.gtx.cd_cxp_gtx_rx.clk),
+                getattr(self, name).rst.eq(rx.gtx.cd_cxp_gtx_rx.rst)
+            ]
+
 
 class Receiver(Module):
     def __init__(self, qpll, pad, sys_clk_freq, tx_mode, rx_mode, debug_sma, pmod_pads):
@@ -51,8 +60,8 @@ class Receiver(Module):
             self.source.stb.eq(0),
             If(gtx.rx_ready & self.source.ack & ~((gtx.decoders[0].d == 0xBC) & (gtx.decoders[0].k == 1)),
                 self.source.stb.eq(1),
-                self.source.data.eq(Cat(gtx.decoders[0].d, gtx.decoders[1].d, gtx.decoders[2].d, gtx.decoders[3].d)),
+                self.source.data.eq(Cat(gtx.decoders[i].d for i in range(4))),
-                self.source.k.eq(Cat(gtx.decoders[0].k, gtx.decoders[1].k, gtx.decoders[2].k, gtx.decoders[3].k)),
+                self.source.k.eq(Cat(gtx.decoders[i].k for i in range(4))),
             )
         ]
 

src/gateware/cxp_pipeline.py

@@ -47,7 +47,7 @@ class Packet_Wrapper(Module):
             self.sink.ack.eq(0),
             self.source.stb.eq(1),
             self.source.data.eq(Replicate(KCode["pak_start"], 4)),
-            self.source.k.eq(0b1111),
+            self.source.k.eq(Replicate(1, 4)),
             If(self.source.ack, NextState("COPY")),
         )
 
@@ -63,7 +63,7 @@ class Packet_Wrapper(Module):
             self.sink.ack.eq(0),
             self.source.stb.eq(1),
             self.source.data.eq(Replicate(KCode["pak_end"], 4)),
-            self.source.k.eq(0b1111),
+            self.source.k.eq(Replicate(1, 4)),
             self.source.eop.eq(1),
             If(self.source.ack, NextState("IDLE")),
         )
@@ -72,7 +72,7 @@ class TX_Trigger(Module):
     def __init__(self):
         self.stb = Signal()
         self.delay = Signal(char_width) 
-        self.linktrig_mode = Signal(max=4)
+        self.linktrig_mode = Signal()
 
         # # #
 
@@ -85,14 +85,14 @@ class TX_Trigger(Module):
         trig_packet = [Signal(char_width), Signal(char_width), Signal(char_width), self.delay, self.delay, self.delay]
         trig_packet_k = [1, 1, 1, 0, 0, 0]
         self.comb += [
-            If((self.linktrig_mode == 0) | (self.linktrig_mode == 2),
+            If(self.linktrig_mode,
-                trig_packet[0].eq(KCode["trig_indic_28_2"]),
-                trig_packet[1].eq(KCode["trig_indic_28_4"]),
-                trig_packet[2].eq(KCode["trig_indic_28_4"]),
-            ).Else(
                 trig_packet[0].eq(KCode["trig_indic_28_4"]),
                 trig_packet[1].eq(KCode["trig_indic_28_2"]),
                 trig_packet[2].eq(KCode["trig_indic_28_2"]),
+            ).Else(
+                trig_packet[0].eq(KCode["trig_indic_28_2"]),
+                trig_packet[1].eq(KCode["trig_indic_28_4"]),
+                trig_packet[2].eq(KCode["trig_indic_28_4"]),
             ),
         ]
         
@@ -140,7 +140,7 @@ class Idle_Word_Inserter(Module):
             self.sink.ack.eq(0),
             self.source.stb.eq(1),
             self.source.data.eq(Cat(KCode["idle_comma"], KCode["idle_alignment"], KCode["idle_alignment"], C(0xB5, char_width))),
-            self.source.k.eq(0b1110),
+            self.source.k.eq(Cat(1, 1, 1, 0)),
             If(self.source.ack, NextState("COPY")),
         )
 
@@ -166,7 +166,7 @@ class Trigger_ACK_Inserter(Module):
             self.sink.ack.eq(0),
             self.source.stb.eq(1),
             self.source.data.eq(Replicate(KCode["io_ack"], 4)),
-            self.source.k.eq(0b1111),
+            self.source.k.eq(Replicate(1, 4)),
             If(self.source.ack, NextState("WRITE_ACK1")),
         )
 
@@ -174,7 +174,7 @@ class Trigger_ACK_Inserter(Module):
             self.sink.ack.eq(0),
             self.source.stb.eq(1),
             self.source.data.eq(Replicate(C(0x01, char_width), 4)),
-            self.source.k.eq(0b0000),
+            self.source.k.eq(Replicate(0, 4)),
             If(self.source.ack, NextState("COPY")),
         )
 
@@ -240,14 +240,14 @@ class TX_Bootstrap(Module, AutoCSR):
         fsm.act("WRITE_TEST_PACKET_TYPE",
             self.source.stb.eq(1),
             self.source.data.eq(Replicate(C(0x04, char_width), 4)),
-            self.source.k.eq(0b0000),
+            self.source.k.eq(Replicate(0, 4)),
             If(self.source.ack,NextState("WRITE_TEST_COUNTER"))
         )
 
         fsm.act("WRITE_TEST_COUNTER",
             self.source.stb.eq(1),
             self.source.data.eq(Cat(cnt[:8], cnt[:8]+1, cnt[:8]+2, cnt[:8]+3)),
-            self.source.k.eq(0b0000),
+            self.source.k.eq(Cat(0, 0, 0, 0)),
             If(self.source.ack,
                 If(cnt == 0xFFF-3,
                     self.source.eop.eq(1),
@@ -405,6 +405,7 @@ class RX_Bootstrap(Module):
         self.comb += [
             mem_port.adr[:addr_nbits].eq(addr),
             mem_port.adr[addr_nbits:].eq(write_ptr),
+            mem_port.dat_w.eq(self.sink.data),
         ]
 
         # For control ack, event packet
@@ -416,9 +417,7 @@ class RX_Bootstrap(Module):
                     NextState("MOVE_BUFFER_PTR"),
                 ).Else(
                     mem_port.we.eq(1),
-                    mem_port.dat_w.eq(self.sink.data),
                     NextValue(addr, addr + 1),
-
                     If(addr == buffer_depth - 1,
                         # discard the packet
                         self.buffer_err.eq(1),

src/gateware/cxp_upconn.py

@@ -180,7 +180,7 @@ class Transmitter(Module, AutoCSR):
         self.specials += [
             # # debug sma
             # Instance("OBUF", i_I=serdes.o, o_O=debug_sma.p_tx),
-            # Instance("OBUF", i_I=cg.clk_10x, o_O=debug_sma.n_rx),
+            Instance("OBUF", i_I=serdes.o, o_O=debug_sma.n_rx),
 
 
 
@@ -205,8 +205,20 @@ class Transmitter(Module, AutoCSR):
 
 class CXP_UpConn_PHYS(Module, AutoCSR):
     def __init__(self, pads, sys_clk_freq, debug_sma, pmod_pads):
+        self.clk_reset = CSR()
+        self.bitrate2x_enable = CSRStorage()
+        self.tx_enable = CSRStorage()
+
+        # # #
+
+
         self.tx_phys = []
         for i, pad in enumerate(pads):
             tx = Transmitter(pad, sys_clk_freq, debug_sma, pmod_pads)
             self.tx_phys.append(tx)
             setattr(self.submodules, "tx"+str(i), tx)
+            self.sync += [
+                tx.clk_reset.eq(self.clk_reset.re),
+                tx.bitrate2x_enable.eq(self.bitrate2x_enable.storage),
+                tx.tx_enable.eq(self.tx_enable.storage),
+            ]

src/gateware/zc706.py

@@ -704,6 +704,7 @@ class CXP_FMC():
         self.csr_devices.append("cxp_phys")
 
 
+        rtio_channels = []
         cxp_csr_group = []
         cxp_tx_mem_group = []
         cxp_rx_mem_group = []
@@ -711,13 +712,25 @@ class CXP_FMC():
         for i, (tx, rx) in enumerate(zip(cxp_phys.upconn.tx_phys, cxp_phys.downconn.rx_phys)):
             cxp_name = "cxp" + str(i)
 
-            # TODO: cdr = ClockDomainsRenamer({"cxp_gtx_rx": "cxp_gtx_rx" + str(i)})
+            cdr = ClockDomainsRenamer({"cxp_gtx_rx": "cxp_gtx_rx" + str(i)})
+            
+            if i == 0:
+                cxp_interface = cdr(cxp.CXP_Master(tx, rx, debug_sma_pad, pmod_pads))
+
+                # Add rtlink for Master Connection only
+                print("CoaXPress at RTIO channel 0x{:06x}".format(len(rtio_channels)))
+                rtio_channels.append(rtio.Channel.from_phy(cxp_interface))
+
+            else:
+                cxp_interface = cdr(cxp.CXP_Extension(tx, rx, debug_sma_pad, pmod_pads))
 
-            cxp_interface = cxp.CXP_Interface(tx, rx, debug_sma_pad, pmod_pads)
             setattr(self.submodules, cxp_name, cxp_interface)
             self.csr_devices.append(cxp_name)
             cxp_csr_group.append(cxp_name)
 
+
+
+            # Add memory group
             rx_mem_name = "cxp_rx" + str(i) + "_mem"
             rx_mem_size = cxp_interface.get_rx_mem_size()
             cxp_rx_mem_group.append(rx_mem_name)
@@ -751,7 +764,6 @@ class CXP_FMC():
             # constraint the CLK path 
             platform.add_false_path_constraints(self.sys_crg.cd_sys.clk, rx.gtx.cd_cxp_gtx_tx.clk, rx.gtx.cd_cxp_gtx_rx.clk)
 
-        rtio_channels = []
         # FIXME remove this placeholder RTIO channel
         # There are too few RTIO channels and cannot be compiled (adr width issue of the lane distributor)
         # see https://github.com/m-labs/artiq/pull/2158 for similar issue

src/libboard_artiq/src/cxp_downconn.rs

@@ -6,7 +6,7 @@ use crate::{cxp_phys, cxp_proto, pl::csr::CXP};
 
 pub fn loopback_testing(channel: usize, timer: &mut GlobalTimer, speed: cxp_phys::CXP_SPEED) {
     println!("==============================================================================");
-    cxp_phys::change_linerate(channel, timer, speed);
+    cxp_phys::change_linerate(speed);
 
     unsafe {
         info!("waiting for tx&rx setup...");

src/libboard_artiq/src/cxp_phys.rs

@@ -4,6 +4,8 @@ use log::info;
 
 use crate::pl::{csr, csr::CXP};
 
+const CHANNEL_LEN: usize = csr::CXP_LEN;
+
 #[derive(Clone, Copy, Debug)]
 #[allow(non_camel_case_types)]
 pub enum CXP_SPEED {
@@ -19,36 +21,35 @@ pub enum CXP_SPEED {
 pub fn setup(timer: &mut GlobalTimer) {
     down_conn::setup(timer);
     up_conn::setup();
+    change_linerate(CXP_SPEED::CXP_1);
 }
 
-pub fn change_linerate(channel: usize, timer: &mut GlobalTimer, speed: CXP_SPEED) {
+pub fn change_linerate(speed: CXP_SPEED) {
-    info!("Changing channel {}'s datarate to {:?}", channel, speed);
+    info!("Changing all channels datarate to {:?}", speed);
-    down_conn::change_linerate(channel, timer, speed);
+    down_conn::change_linerate(speed);
-    up_conn::change_linerate(channel, speed);
+    up_conn::change_linerate(speed);
 }
 
 mod up_conn {
     use super::*;
 
     pub fn setup() {
-        // TODO: do a for loop for channel?
-        let channel: usize = 0;
         unsafe {
-            change_linerate(channel, CXP_SPEED::CXP_1);
+            csr::cxp_phys::upconn_tx_enable_write(1);
-            (CXP[channel].upconn_tx_enable_write)(1);
         }
     }
 
-    pub fn change_linerate(channel: usize, speed: CXP_SPEED) {
+    pub fn change_linerate(speed: CXP_SPEED) {
         unsafe {
-            (CXP[channel].upconn_clk_reset_write)(1);
             match speed {
-                CXP_SPEED::CXP_10 | CXP_SPEED::CXP_12 => {
+                CXP_SPEED::CXP_1 | CXP_SPEED::CXP_2 | CXP_SPEED::CXP_3 | CXP_SPEED::CXP_5 | CXP_SPEED::CXP_6 => {
-                    (CXP[channel].upconn_bitrate2x_enable_write)(1);
+                    csr::cxp_phys::upconn_bitrate2x_enable_write(0);
+                }
+                CXP_SPEED::CXP_10 | CXP_SPEED::CXP_12 => {
+                    csr::cxp_phys::upconn_bitrate2x_enable_write(1);
                 }
-                _ => {}
             };
-            (CXP[channel].upconn_clk_reset_write)(0);
+            csr::cxp_phys::upconn_clk_reset_write(1);
         }
     }
 }
@@ -57,11 +58,11 @@ mod down_conn {
     use super::*;
 
     pub fn setup(timer: &mut GlobalTimer) {
-        // TODO: do a for loop for channel?
-        let channel: usize = 0;
         unsafe {
             info!("turning on pmc loopback mode...");
+            for channel in 0..CHANNEL_LEN {
                 (CXP[channel].downconn_loopback_mode_write)(0b010); // Near-End PMA Loopback
+            }
 
             // QPLL setup
             csr::cxp_phys::downconn_qpll_reset_write(1);
@@ -69,43 +70,46 @@ mod down_conn {
             while csr::cxp_phys::downconn_qpll_locked_read() != 1 {}
             info!("QPLL locked");
 
+            for channel in 0..CHANNEL_LEN {
                 // tx/rx setup
                 (CXP[channel].downconn_tx_start_init_write)(1);
                 (CXP[channel].downconn_rx_start_init_write)(1);
+            }
 
+            // DEBUG: printout
             info!("waiting for tx & rx setup...");
             timer.delay_us(50_000);
+            for channel in 0..CHANNEL_LEN {
                 info!(
                     "tx_phaligndone = {} | rx_phaligndone = {}",
                     (CXP[channel].downconn_txinit_phaligndone_read)(),
                     (CXP[channel].downconn_rxinit_phaligndone_read)(),
                 );
             }
-
+        }
-        change_linerate(channel, timer, CXP_SPEED::CXP_1);
     }
 
-    pub fn change_linerate(channel: usize, timer: &mut GlobalTimer, speed: CXP_SPEED) {
+    pub fn change_linerate(speed: CXP_SPEED) {
         // DEBUG: DRP pll for TXUSRCLK = freq(linerate)/20
         let settings = txusrclk::get_txusrclk_config(speed);
-        txusrclk::setup(channel, timer, settings);
+        txusrclk::setup(settings);
 
         change_qpll_fb_divider(speed);
-        change_gtx_divider(channel, speed);
+        change_gtx_divider(speed);
-        change_cdr_cfg(channel, speed);
+        change_cdr_cfg(speed);
 
         unsafe {
             csr::cxp_phys::downconn_qpll_reset_write(1);
             info!("waiting for QPLL/CPLL to lock...");
             while csr::cxp_phys::downconn_qpll_locked_read() != 1 {}
             info!("QPLL locked");
-        }
 
-        unsafe {
+            for channel in 0..CHANNEL_LEN {
                 (CXP[channel].downconn_tx_restart_write)(1);
                 (CXP[channel].downconn_rx_restart_write)(1);
             }
         }
+    }
 
     fn change_qpll_fb_divider(speed: CXP_SPEED) {
         let qpll_div_reg = match speed {
@@ -118,7 +122,7 @@ mod down_conn {
         println!("0x36 = {:#06x}", qpll_read(0x36));
     }
 
-    fn change_gtx_divider(channel: usize, speed: CXP_SPEED) {
+    fn change_gtx_divider(speed: CXP_SPEED) {
         let div_reg = match speed {
             CXP_SPEED::CXP_1 => 0x33,                      // RXOUT_DIV = 8
             CXP_SPEED::CXP_2 | CXP_SPEED::CXP_3 => 0x22,   // RXOUT_DIV = 4
@@ -126,12 +130,14 @@ mod down_conn {
             CXP_SPEED::CXP_10 | CXP_SPEED::CXP_12 => 0x00, // RXOUT_DIV = 1
         };
 
-        println!("0x88 = {:#06x}", gtx_read(channel, 0x88));
+        for channel in 0..CHANNEL_LEN {
+            println!("channel {}, 0x88 = {:#06x}", channel, gtx_read(channel, 0x88));
             gtx_write(channel, 0x88, div_reg);
-        println!("0x88 = {:#06x}", gtx_read(channel, 0x88));
+            println!("channel {}, 0x88 = {:#06x}", channel, gtx_read(channel, 0x88));
+        }
     }
 
-    fn change_cdr_cfg(channel: usize, speed: CXP_SPEED) {
+    fn change_cdr_cfg(speed: CXP_SPEED) {
         struct CdrConfig {
             pub cfg_reg0: u16, // addr = 0xA8
             pub cfg_reg1: u16, // addr = 0xA9
@@ -175,12 +181,14 @@ mod down_conn {
             },
         };
 
+        for channel in 0..CHANNEL_LEN {
             gtx_write(channel, 0x0A8, cdr_cfg.cfg_reg0);
             gtx_write(channel, 0x0A9, cdr_cfg.cfg_reg1);
             gtx_write(channel, 0x0AA, cdr_cfg.cfg_reg2);
             gtx_write(channel, 0x0AB, cdr_cfg.cfg_reg3);
             gtx_write(channel, 0x0AC, cdr_cfg.cfg_reg4);
         }
+    }
 
     #[allow(dead_code)]
     fn gtx_read(channel: usize, address: u16) -> u16 {
@@ -224,6 +232,7 @@ mod down_conn {
     pub mod txusrclk {
         use super::*;
 
+        #[derive(Copy, Clone)]
         pub struct PLLSetting {
             pub clkout0_reg1: u16,  //0x08
             pub clkout0_reg2: u16,  //0x09
@@ -317,7 +326,8 @@ mod down_conn {
             }
         }
 
-        pub fn setup(channel: usize, timer: &mut GlobalTimer, settings: PLLSetting) {
+        pub fn setup(settings: PLLSetting) {
+            for channel in 0..CHANNEL_LEN {
                 if false {
                     info!("0x08 = {:#06x}", read(channel, 0x08));
                     info!("0x09 = {:#06x}", read(channel, 0x09));
@@ -347,11 +357,10 @@ mod down_conn {
                     write(channel, 0x4F, settings.filt_reg2);
                     reset(channel, false);
 
-                // wait for the pll to lock
+                    info!("waiting for PLL of txusrclk to lock...");
-                timer.delay_us(100);
+                    while unsafe { (CXP[channel].downconn_txpll_locked_read)() == 0 } {}
-
+                    info!("txusrclk locked :D");
-                let locked = unsafe { (CXP[channel].downconn_txpll_locked_read)() == 1 };
+                }
-                info!("txusrclk locked = {}", locked);
             }
         }
 

src/libboard_artiq/src/cxp_proto.rs

@@ -7,7 +7,7 @@ use io::Cursor;
 use libboard_zynq::println;
 
 use crate::{mem::mem::{CXP_LOOPBACK_MEM, CXP_RX_MEM, CXP_TX_MEM},
-            pl::csr::CXP};
+            pl::{csr, csr::CXP}};
 
 const BUF_LEN: usize = 0x800;
 const DATA_MAXSIZE: usize = 48;
@@ -448,7 +448,7 @@ impl UpConnPacket {
 }
 
 pub fn send(channel: usize, packet: &UpConnPacket) -> Result<(), Error> {
-    if unsafe { (CXP[channel].upconn_tx_enable_read)() } == 0 {
+    if unsafe { csr::cxp_phys::upconn_tx_enable_read() } == 0 {
         Err(Error::LinkDown)?
     }
 
@@ -465,6 +465,8 @@ fn send_data_packet(channel: usize, packet: &UpConnPacket) -> Result<(), Error>
         let mut writer = Cursor::new(slice::from_raw_parts_mut(ptr as *mut u8, BUF_LEN));
 
         packet.write_to(&mut writer)?;
+        println!("TX MEM after writing");
+        print_packet(&writer.get_ref()[0..40]);
 
         (CXP[channel].upconn_bootstrap_tx_word_len_write)(writer.position() as u16 / 4);
         (CXP[channel].upconn_bootstrap_tx_write)(1);

src/libboard_artiq/src/cxp_upconn.rs

@@ -24,7 +24,7 @@ pub fn tx_test(channel: usize, timer: &mut GlobalTimer) {
         // CXP[channel].upconn_ack_write(1);
 
         timer.delay_us(20);
-        (CXP[channel].upconn_tx_enable_write)(0);
+        csr::cxp_phys::upconn_tx_enable_write(0);
 
         // Collect data
         let mut i: usize = 0;

src/runtime/src/main.rs

@@ -158,11 +158,7 @@ pub fn main_core0() {
     // cxp_downconn::loopback_testing(0, &mut timer, cxp_phys::CXP_SPEED::CXP_6);
     // cxp_downconn::loopback_testing(0, &mut timer, cxp_phys::CXP_SPEED::CXP_10);
     // cxp_downconn::loopback_testing(0, &mut timer, cxp_phys::CXP_SPEED::CXP_12);
-    // loop {
-    //     use embedded_hal::prelude::_embedded_hal_blocking_delay_DelayUs;
     // cxp_upconn::tx_test(0, &mut timer);
-    //     timer.delay_us(5_000_000);
-    // }
 
     comms::main(timer, cfg);
 }