cxp downconn firmware: GTX setup

testing: add loopmode mode & tsusrclk mmcm drp bitbang
testing: add IDLE word printout
downconn: add QPLL and GTX setup
downconn: add DRP to support all CXP linerate up to 12.5Gbps

src/gateware/cxp.py

@@ -17,18 +17,17 @@ class CXP(Module, AutoCSR):
 
         
 class UpConn_Interface(Module, AutoCSR):
-    def __init__(self, upconn_pads, sys_clk_freq, debug_sma, pmod_pads, fifos_depth=64):
+    def __init__(self, upconn_pads, sys_clk_freq, debug_sma, pmod_pads):
         self.clk_reset = CSRStorage(reset=1)
         self.bitrate2x_enable = CSRStorage()
         self.tx_enable = CSRStorage()
         self.tx_busy = CSRStatus()
-        self.encoded_data = CSRStatus(10)
 
         # # #
 
         layout = [("data", 8), ("k", 1)]
 
-        self.submodules.upconn_phy = upconn_phy = CXP_UpConn_PHY(upconn_pads, sys_clk_freq, debug_sma, pmod_pads, layout, fifos_depth)
+        self.submodules.upconn_phy = upconn_phy = CXP_UpConn_PHY(upconn_pads, sys_clk_freq, debug_sma, pmod_pads, layout)
 
         self.sync += [
             upconn_phy.bitrate2x_enable.eq(self.bitrate2x_enable.storage),
@@ -36,14 +35,12 @@ class UpConn_Interface(Module, AutoCSR):
             upconn_phy.clk_reset.eq(self.clk_reset.re),
             self.tx_busy.status.eq(upconn_phy.tx_busy),
         ]
-        self.sync += [
-            self.encoded_data.status.eq(upconn_phy.scheduler.encoder.output),
-        ]
 
 
         # Packet FIFOs with transmission priority
-        # NOTE: 0 Trigger packet
+        # 0: Trigger packet
         self.submodules.trig = trig = TX_Trigger(layout)
+        self.comb += trig.source.connect(upconn_phy.sinks[0])
         
         # DEBUG: INPUT
         self.trig_stb = CSR()
@@ -56,97 +53,17 @@ class UpConn_Interface(Module, AutoCSR):
             trig.linktrig_mode.eq(self.linktrigger.storage),
         ]
 
-        # DEBUG: OUTPUT
-        self.submodules.trig_out = trig_out = stream.SyncFIFO(layout, 64)
-        self.comb += trig.source.connect(trig_out.sink)
-        self.trig_inc = CSR()
-        self.trig_dout_pak = CSRStatus(8)
-        self.trig_kout_pak = CSRStatus()
-        self.trig_dout_valid = CSRStatus()
-
-        self.sync += [
-            # output
-            trig_out.source.ack.eq(self.trig_inc.re),
-            self.trig_dout_pak.status.eq(trig_out.source.data),
-            self.trig_kout_pak.status.eq(trig_out.source.k),
-            self.trig_dout_valid.status.eq(trig_out.source.stb),
-        ]
-
-        self.symbol0 = CSR(9)
-        self.sync += [
-            upconn_phy.tx_fifos.sink[0].stb.eq(self.symbol0.re),
-            upconn_phy.tx_fifos.sink[0].data.eq(self.symbol0.r[:8]),
-            upconn_phy.tx_fifos.sink[0].k.eq(self.symbol0.r[8]),
-        ]
-
         
-        # NOTE: 1 IO acknowledgment for trigger packet
-
+        # 1: IO acknowledgment for trigger packet
         self.submodules.trig_ack = trig_ack = Trigger_ACK(layout)
+        self.comb += trig_ack.source.connect(upconn_phy.sinks[1])
 
         # DEBUG: INPUT
         self.ack = CSR()
-        self.sync += [
-            trig_ack.ack.eq(self.ack.re),
-        ]
+        self.sync += trig_ack.ack.eq(self.ack.re),
 
-        # DEBUG: OUTPUT
-        self.submodules.trig_ack_out = trig_ack_out = stream.SyncFIFO(layout, 64)
-        self.comb += trig_ack.source.connect(trig_ack_out.sink)
-        self.trig_ack_inc = CSR()
-        self.trig_ack_dout_pak = CSRStatus(8)
-        self.trig_ack_kout_pak = CSRStatus()
-        self.trig_ack_dout_valid = CSRStatus()
 
-        self.sync += [
-            # output
-            trig_ack_out.source.ack.eq(self.trig_ack_inc.re),
-            self.trig_ack_dout_pak.status.eq(trig_ack_out.source.data),
-            self.trig_ack_kout_pak.status.eq(trig_ack_out.source.k),
-            self.trig_ack_dout_valid.status.eq(trig_ack_out.source.stb),
-        ]
-
-        self.symbol1 = CSR(9)
-        self.sync += [
-            upconn_phy.tx_fifos.sink[1].stb.eq(self.symbol1.re),
-            upconn_phy.tx_fifos.sink[1].data.eq(self.symbol1.r[:8]),
-            upconn_phy.tx_fifos.sink[1].k.eq(self.symbol1.r[8]),
-        ]
-        
-        # NOTE: 2 All other packets
+        # 2: All other packets
         # Control is not timing dependent, all the link layer is done in firmware
-
-        # Table 54 (CXP-001-2021)
-        # Largest CXP register is 8 byte
-
-        # increment after ack
-        # for CXP 2.0 or latest, command packet need to includet tags
-        # section 9.6.1.2 (CXP-001-2021)  
-        # tags implementation is on firmware
-
         self.submodules.command = command = TX_Command_Packet(layout)
-        self.comb += command.source.connect(upconn_phy.tx_fifos.sink[2])
-
-        # DEBUG: OUTPUT
-        # self.submodules.command_out = command_out = stream.SyncFIFO(layout, 64)
-        # self.comb += command.source.connect(command_out.sink)
-
-        # self.command_inc = CSR()
-        # self.command_dout_pak = CSRStatus(8)
-        # self.command_kout_pak = CSRStatus()
-        # self.command_dout_valid = CSRStatus()
-
-        # self.sync += [
-        #     # output
-        #     command_out.source.ack.eq(self.command_inc.re),
-        #     self.command_dout_pak.status.eq(command_out.source.data),
-        #     self.command_kout_pak.status.eq(command_out.source.k),
-        #     self.command_dout_valid.status.eq(command_out.source.stb),
-        # ]
-
-        # self.symbol2 = CSR(9)
-        # self.sync += [
-        #     upconn_phy.tx_fifos.sink[2].stb.eq(self.symbol2.re),
-        #     upconn_phy.tx_fifos.sink[2].data.eq(self.symbol2.r[:8]),
-        #     upconn_phy.tx_fifos.sink[2].k.eq(self.symbol2.r[8]),
-        # ]
+        self.comb += command.source.connect(upconn_phy.sinks[2])

src/gateware/cxp_pipeline.py

@@ -249,23 +249,31 @@ class TX_Command_Packet(Module, AutoCSR):
 
         # # #
 
+        # TODO: use RAM instead of FIFO ?
+        # Section 12.1.2 (CXP-001-2021)
+        # Max control packet size is 128 bytes
+        
+        # NOTE: The firmware will lock up if there is not enough space for the packet
+        self.submodules.fifo = fifo = stream.SyncFIFO(layout, 128)
         self.submodules.pak_wrp = pak_wrp = Packet_Wrapper(layout)
         self.source = pak_wrp.source
 
+        self.comb += fifo.source.connect(pak_wrp.sink)
+
         len = Signal(6, reset=1)
         self.sync += [
-            self.writeable.status.eq(pak_wrp.sink.ack),
-            If(pak_wrp.sink.ack,pak_wrp.sink.stb.eq(0)),
+            self.writeable.status.eq(fifo.sink.ack),
+            If(fifo.sink.ack, fifo.sink.stb.eq(0)),
             If(self.data.re,
-                pak_wrp.sink.stb.eq(1),
-                pak_wrp.sink.data.eq(self.data.r),
+                fifo.sink.stb.eq(1),
+                fifo.sink.data.eq(self.data.r),
 
-                pak_wrp.sink.k.eq(0),
+                fifo.sink.k.eq(0),
                 If(len == self.len.storage,
-                    pak_wrp.sink.eop.eq(1),
+                    fifo.sink.eop.eq(1),
                     len.eq(len.reset),
                 ).Else(
-                    pak_wrp.sink.eop.eq(0),
+                    fifo.sink.eop.eq(0),
                     len.eq(len + 1),
                 ),
             )

src/gateware/cxp_upconn.py

@@ -224,41 +224,38 @@ class Packets_Scheduler(Module):
             )
         ]
 
-class TxFIFOs(Module):
-    def __init__(self, layout, nfifos, fifo_depth):
+class PHY_Interface(Module):
+    def __init__(self, layout, nsink):
 
-        self.sink = []
-
-        self.source_stb = Signal(nfifos)
-        self.source_ack = Array(Signal() for _ in range(nfifos))
-        self.source_data = Array(Signal(8) for _ in range(nfifos))
-        self.source_k = Array(Signal() for _ in range(nfifos))
+        self.source_stb = Signal(nsink)
+        self.source_ack = Array(Signal() for _ in range(nsink))
+        self.source_data = Array(Signal(8) for _ in range(nsink))
+        self.source_k = Array(Signal() for _ in range(nsink))
 
         # # #
 
-        for i in range(nfifos):
-            fifo = stream.SyncFIFO(layout, fifo_depth)
-            setattr(self.submodules, "tx_fifo" + str(i), fifo)
-
-            self.sink += [fifo.sink]
+        self.sinks = []
+        for i in range(nsink):
+            sink = stream.Endpoint(layout)
+            self.sinks += [sink]
 
             self.sync += [
                 If(self.source_ack[i], 
                     # reset ack after asserted 
                     # since upconn clk run much slower, the ack will be high for longer than expected which will result in data loss
                     self.source_ack[i].eq(0),
-                    fifo.source.ack.eq(1),
+                    sink.ack.eq(1),
                 ).Else(
-                    fifo.source.ack.eq(0),
+                    sink.ack.eq(0),
                 ),
 
-                self.source_stb[i].eq(fifo.source.stb),
-                self.source_data[i].eq(fifo.source.data),
-                self.source_k[i].eq(fifo.source.k),
+                self.source_stb[i].eq(sink.stb),
+                self.source_data[i].eq(sink.data),
+                self.source_k[i].eq(sink.k),
             ]
 
         # FIFOs transmission priority
-        self.submodules.pe = PriorityEncoder(nfifos)
+        self.submodules.pe = PriorityEncoder(nsink)
         self.comb += self.pe.i.eq(self.source_stb)
 
 class Debug_buffer(Module,AutoCSR):
@@ -304,7 +301,7 @@ class Debug_buffer(Module,AutoCSR):
 
 
 class CXP_UpConn_PHY(Module, AutoCSR):
-    def __init__(self, pad, sys_clk_freq, debug_sma, pmod_pads, layout, fifo_depth, nfifos=3):
+    def __init__(self, pad, sys_clk_freq, debug_sma, pmod_pads, layout, nsink=3):
         self.bitrate2x_enable = Signal()
         self.clk_reset = Signal()
 
@@ -314,16 +311,18 @@ class CXP_UpConn_PHY(Module, AutoCSR):
         # # #
 
         self.submodules.cg = cg = UpConn_ClockGen(sys_clk_freq)
-        self.submodules.tx_fifos = tx_fifos = TxFIFOs(layout, nfifos, fifo_depth)
+        self.submodules.interface = interface = PHY_Interface(layout, nsink)
+
+        self.sinks = interface.sinks
 
         # DEBUG:
         self.submodules.debug_buf = debug_buf = Debug_buffer(layout)
 
-        self.submodules.scheduler = scheduler = Packets_Scheduler(tx_fifos, debug_buf)
+        self.submodules.scheduler = scheduler = Packets_Scheduler(interface, debug_buf)
         self.submodules.serdes = serdes = SERDES_10bits(pad)
 
         self.comb += [
-            self.tx_busy.eq(tx_fifos.source_stb != 0),
+            self.tx_busy.eq(interface.source_stb != 0),
 
             cg.reset.eq(self.clk_reset),
             cg.freq2x_enable.eq(self.bitrate2x_enable),
@@ -346,7 +345,7 @@ class CXP_UpConn_PHY(Module, AutoCSR):
         p0 = Signal()
         p3 = Signal()
         self.comb += [
-           prioity_0.eq((~tx_fifos.pe.n) & (tx_fifos.pe.o == 0)),
+           prioity_0.eq((~interface.pe.n) & (interface.pe.o == 0)),
            word_bound.eq(scheduler.tx_charcount == 3),
 
            # because of clk delay

src/libboard_artiq/src/cxp_upconn.rs

@@ -6,46 +6,15 @@ use libboard_zynq::{println, timer::GlobalTimer};
 
 use crate::pl::csr;
 
-pub fn trigger_test(timer: &mut GlobalTimer, linktrig_mode: u8) {
-    const LEN: usize = 4 * 8;
-    let mut pak_arr: [u8; LEN] = [0; LEN];
-
-    unsafe {
-        csr::cxp::upconn_trig_delay_write(0x05);
-        csr::cxp::upconn_linktrigger_write(linktrig_mode);
-        csr::cxp::upconn_trig_stb_write(1); // send trig
-        timer.delay_us(1);
-
-        let mut i: usize = 0;
-        while csr::cxp::upconn_trig_dout_valid_read() == 1 {
-            pak_arr[i] = csr::cxp::upconn_trig_dout_pak_read();
-            // println!("received {:#04X}", pak_arr[i]);
-            csr::cxp::upconn_trig_inc_write(1);
-            i += 1;
-        }
-
-        println!("trigger packet | linktrigger = {}", linktrig_mode);
-        print_packet(&pak_arr);
-    }
+#[derive(Debug)]
+pub enum Error {
+    BufferError,
+    LinkDown,
 }
 
-pub fn trigger_ack_test(timer: &mut GlobalTimer) {
-    const LEN: usize = 4 * 8;
-    let mut pak_arr: [u8; LEN] = [0; LEN];
-
-    unsafe {
-        csr::cxp::upconn_ack_write(1); // send IO ack
-
-        let mut i: usize = 0;
-        while csr::cxp::upconn_trig_ack_dout_valid_read() == 1 {
-            pak_arr[i] = csr::cxp::upconn_trig_ack_dout_pak_read();
-            // println!("received {:#04X}", pak_arr[i]);
-            csr::cxp::upconn_trig_ack_inc_write(1);
-            i += 1;
-        }
-
-        println!("trigger ack packet");
-        print_packet(&pak_arr);
+impl From<IoError> for Error {
+    fn from(_: IoError) -> Error {
+        Error::BufferError
     }
 }
 
@@ -61,7 +30,18 @@ pub fn tx_test(timer: &mut GlobalTimer) {
         send(&Packet::ControlU32Reg(Command::Read { addr: 0x00 })).expect("Cannot send CoaXpress packet");
 
         csr::cxp::upconn_tx_enable_write(1);
+
+        timer.delay_us(2);
+        // DEBUG: Trigger packet
+        let linktrig_mode: u8 = 0x01;
+        csr::cxp::upconn_trig_delay_write(0x05);
+        csr::cxp::upconn_linktrigger_write(linktrig_mode);
+        csr::cxp::upconn_trig_stb_write(1); // send trig
+
+        // DEBUG: Trigger ACK packet
+        // csr::cxp::upconn_ack_write(1);
         timer.delay_us(20);
+
         csr::cxp::upconn_tx_enable_write(0);
 
         // Collect data
@@ -93,7 +73,7 @@ pub enum Packet {
 }
 
 impl Packet {
-    pub fn write_to<W>(&self, writer: &mut W) -> Result<(), IoError>
+    pub fn write_to<W>(&self, writer: &mut W) -> Result<(), Error>
     where W: Write {
         match self {
             Packet::ControlU32Reg(cmd) => match cmd {
@@ -153,7 +133,12 @@ impl Packet {
     }
 }
 
-pub fn send(packet: &Packet) -> Result<(), IoError> {
+pub fn send(packet: &Packet) -> Result<(), Error> {
+    // DEBUG: remove the comment out section
+    // if unsafe { csr::cxp::upconn_tx_enable_read() } == 0 {
+    //     Err(Error::LinkDown)?
+    // }
+
     const LEN: usize = 4 * 20;
     let mut buffer: [u8; LEN] = [0; LEN];
     let mut writer = Cursor::new(&mut buffer[..]);