cxp GW: add cxp frame routing pipeline

cxp GW: use roi pipeline

cxp GW: add register for cnt to imprve timinig

cxp GW: add cdr for roi pipeline

cxp GW: add back rx debug buffer

cxp GW: move router pipeline to frame pipeline
cxp GW: refactor to combine rx/tx into phy
cxp GW: fix compilation err
cxp GW: remove unused memorywe
cxp GW: update to use word_width
cxp GW: add rtio interface for frame pipeline
cxp GW: fix extra 1 bits issue
cxp GW: add crc error counter
cxp GW: add fifo for rx pipeline

src/gateware/cxp.py

@@ -1,97 +1,73 @@
 from migen import *
-from migen.genlib.cdc import MultiReg, PulseSynchronizer, BusSynchronizer
+from migen.genlib.cdc import MultiReg, PulseSynchronizer
 from misoc.interconnect.csr import *
 
 from artiq.gateware.rtio import rtlink
 
-from cxp_downconn import CXP_DownConn_PHYS
+from cxp_downconn import CXP_RXPHYs
-from cxp_upconn import CXP_UpConn_PHYS
+from cxp_upconn import CXP_TXPHYs
 from cxp_pipeline import *
 from cxp_frame_pipeline import *
 
 from functools import reduce
 from operator import add
+from types import SimpleNamespace
 
 class CXP_PHYS(Module, AutoCSR):
-    def __init__(self, refclk, upconn_pads, downconn_pads, sys_clk_freq, debug_sma, pmod_pads):
+    def __init__(self, refclk, upconn_pads, downconn_pads, sys_clk_freq, master=0):
         assert len(upconn_pads) == len(downconn_pads)
 
-        self.submodules.upconn = CXP_UpConn_PHYS(upconn_pads, sys_clk_freq, debug_sma, pmod_pads)
+        self.submodules.tx = CXP_TXPHYs(upconn_pads, sys_clk_freq)
-        self.submodules.downconn = CXP_DownConn_PHYS(refclk, downconn_pads, sys_clk_freq, debug_sma, pmod_pads)
+        self.submodules.rx = CXP_RXPHYs(refclk, downconn_pads, sys_clk_freq, master)
 
-@FullMemoryWE()
+        self.phys = []
-class CXP_Interface(Module, AutoCSR):
+        for tx, rx in zip(self.tx.phys, self.rx.phys):
-    def __init__(self, upconn_phy, downconn_phy, debug_sma, pmod_pads):
+            phy = SimpleNamespace()
-        self.submodules.upconn = UpConn_Interface(upconn_phy, debug_sma, pmod_pads)
+            phy.tx, phy.rx = tx, rx
-        self.submodules.downconn = DownConn_Interface(downconn_phy, debug_sma, pmod_pads)
+            self.phys.append(phy)
+        
+
+class CXP_Core(Module, AutoCSR):
+    def __init__(self, phy):
+        self.submodules.tx = TX_Pipeline(phy.tx)
+        self.submodules.rx = RX_Pipeline(phy.rx)
 
     def get_tx_port(self):
-        return self.upconn.bootstrap.mem.get_port(write_capable=True)
+        return self.tx.writer.mem.get_port(write_capable=True)
 
     def get_tx_mem_size(self):
+        # TODO: remove this
         # FIXME: if tx mem size is NOT same as rx, for some reason when rx mem is writen, tx mem cannot be access anymore
         # and each time tx mem is read, CPU will return rx mem instead (fixed by reordering the mem allocation order)
         # FIXME: seems like there are address alignment issue, if tx mem size is 0x800, the mem following the tx mem cannot be read correctly
         # However, if tx mem is 0x2000 (same size as rx mem) the following rx mem can be read correctly
-        return self.upconn.bootstrap.mem.depth*self.upconn.bootstrap.mem.width // 8 # 0x800
+        return self.tx.writer.mem.depth*self.upconn.bootstrap.mem.width // 8 # 0x800
         # return self.downconn.bootstrap.mem.depth*self.downconn.bootstrap.mem.width // 8 # 0x2000
 
     def get_mem_size(self):
-        return word_dw * buffer_count * buffer_depth // 8
+        return word_width * buffer_count * buffer_depth // 8
 
     def get_rx_port(self):
-        return self.downconn.bootstrap.mem.get_port(write_capable=False)
+        return self.rx.reader.mem.get_port(write_capable=False)
 
     def get_rx_mem_size(self):
-        return self.downconn.bootstrap.mem.depth*self.downconn.bootstrap.mem.width // 8
+        # TODO: remove this
+        return self.rx.reader.mem.depth*self.downconn.bootstrap.mem.width // 8
 
-    def get_rx_downconn(self):
+class RX_Pipeline(Module, AutoCSR):
-        return self.downconn
+    def __init__(self, phy):
-
+        self.ready = CSRStatus()
-class CXP_Master(CXP_Interface):
-    def __init__(self, upconn_phy, downconn_phy, debug_sma, pmod_pads):
-        CXP_Interface.__init__(self, upconn_phy, downconn_phy, debug_sma, pmod_pads)
-        nbit_trigdelay = 8
-        nbit_linktrig = 1
-
-        self.rtlink = rtlink.Interface(
-            rtlink.OInterface(nbit_trigdelay + nbit_linktrig),
-            rtlink.IInterface(word_dw, timestamped=False)
-        )
-
-        self.sync.rio += [
-            If(self.rtlink.o.stb,
-                self.upconn.trig.delay.eq(self.rtlink.o.data[nbit_linktrig:]),
-                self.upconn.trig.linktrig_mode.eq(self.rtlink.o.data[:nbit_linktrig]),
-            ),
-            self.upconn.trig.stb.eq(self.rtlink.o.stb),
-        ]
-
-        # DEBUG: out
-        self.specials += Instance("OBUF", i_I=self.rtlink.o.stb, o_O=debug_sma.p_tx),
-        # self.specials += Instance("OBUF", i_I=self.rtlink.o.stb, o_O=debug_sma.n_rx),
-
-class CXP_Extension(CXP_Interface):
-    def __init__(self, upconn_phy, downconn_phy, debug_sma, pmod_pads):
-        CXP_Interface.__init__(self, upconn_phy, downconn_phy, debug_sma, pmod_pads)
-
-
-class DownConn_Interface(Module, AutoCSR):
-    def __init__(self, phy, debug_sma, pmod_pads):
-        self.rx_ready = CSRStatus()
 
         # # #
 
         gtx = phy.gtx
 
         # GTX status
-        self.sync += self.rx_ready.status.eq(gtx.rx_ready)
+        self.sync += self.ready.status.eq(gtx.rx_ready)
 
         # DEBUG: init status
-        self.txinit_phaligndone = CSRStatus()
         self.rxinit_phaligndone = CSRStatus()
         self.comb += [
-            self.txinit_phaligndone.status.eq(gtx.tx_init.Xxphaligndone),
             self.rxinit_phaligndone.status.eq(gtx.rx_init.Xxphaligndone),
         ]
 
@@ -122,8 +98,8 @@ class DownConn_Interface(Module, AutoCSR):
         # Receiver Pipeline WIP
         # 
         #        32             32+8(dchar)
-        # PHY ---/---> dchar   -----/-----> trigger ack ------>  packet  ------> CDC FIFO ------> debug buffer
+        # PHY ---/---> dchar   -----/-----> trigger ack ------>  packet  ------> EOP Marker ------> stream data packet
-        #              decoder              checker              decoder
+        #              decoder              checker              decoder                            with CRC
         #
         cdr = ClockDomainsRenamer("cxp_gtx_rx")
 
@@ -148,59 +124,65 @@ class DownConn_Interface(Module, AutoCSR):
         ]
 
         # Priority level 2 packet - data, test packet
-        self.submodules.bootstrap = bootstrap = cdr(RX_Bootstrap())
+        self.submodules.reader = reader = cdr(Control_Packet_Reader())
 
-        self.bootstrap_decoder_err = CSR()
+        self.reader_decode_err = CSR()
-        self.bootstrap_buffer_err = CSR()
+        self.reader_buffer_err = CSR()
 
         decode_err_ps = PulseSynchronizer("cxp_gtx_rx", "sys")
         buffer_err_ps = PulseSynchronizer("cxp_gtx_rx", "sys")
         self.submodules += decode_err_ps, buffer_err_ps
         self.sync.cxp_gtx_rx += [
-            decode_err_ps.i.eq(bootstrap.decode_err),
+            decode_err_ps.i.eq(reader.decode_err),
-            buffer_err_ps.i.eq(bootstrap.buffer_err),
+            buffer_err_ps.i.eq(reader.buffer_err),
         ]
         self.sync += [
             If(decode_err_ps.o,
-                self.bootstrap_decoder_err.w.eq(1),
+                self.reader_decode_err.w.eq(1),
-            ).Elif(self.bootstrap_decoder_err.re,
+            ).Elif(self.reader_decode_err.re,
-                self.bootstrap_decoder_err.w.eq(0),
+                self.reader_decode_err.w.eq(0),
             ),
             If(buffer_err_ps.o,
-                self.bootstrap_buffer_err.w.eq(1),
+                self.reader_buffer_err.w.eq(1),
-            ).Elif(self.bootstrap_buffer_err.re,
+            ).Elif(self.reader_buffer_err.re,
-                self.bootstrap_buffer_err.w.eq(0),
+                self.reader_buffer_err.w.eq(0),
             ),
         ]
 
-        
+        # TODO: rewrite this to follow crc_error_cnt from frameline
         # test packet error & packet counters
-        self.bootstrap_test_error_counter = CSRStatus(len(bootstrap.test_err_cnt))
+        self.test_error_counter = CSRStatus(len(reader.test_err_cnt))
-        self.bootstrap_test_packet_counter = CSRStatus(len(bootstrap.test_pak_cnt))
+        self.test_packet_counter = CSRStatus(len(reader.test_pak_cnt))
-        self.bootstrap_test_counts_reset = CSR()
+        self.test_counts_reset = CSR()
 
 
         test_reset_ps = PulseSynchronizer("sys", "cxp_gtx_rx")
         self.submodules += test_reset_ps
-        self.sync += test_reset_ps.i.eq(self.bootstrap_test_counts_reset.re),
+        self.sync += test_reset_ps.i.eq(self.test_counts_reset.re),
 
-        self.sync.cxp_gtx_rx += bootstrap.test_cnt_reset.eq(test_reset_ps.o),
+        test_err_cnt_r = Signal.like(reader.test_err_cnt)
+        test_pak_cnt_r = Signal.like(reader.test_pak_cnt)
+        self.sync.cxp_gtx_rx += [ 
+            reader.test_cnt_reset.eq(test_reset_ps.o),
+            test_err_cnt_r.eq(reader.test_err_cnt),
+            test_pak_cnt_r.eq(reader.test_pak_cnt),
+        ]
         self.specials += [
-            MultiReg(bootstrap.test_err_cnt, self.bootstrap_test_error_counter.status),
+            MultiReg(test_err_cnt_r, self.test_error_counter.status),
-            MultiReg(bootstrap.test_pak_cnt, self.bootstrap_test_packet_counter.status),
+            MultiReg(test_pak_cnt_r, self.test_packet_counter.status),
         ]
 
-        # Cicular buffer interface
+        # reader cicular memory control interface
         self.packet_type = CSRStatus(8)
         self.pending_packet = CSR()
         self.read_ptr = CSRStatus(log2_int(buffer_count))
 
         self.specials += [
-            MultiReg(bootstrap.packet_type, self.packet_type.status),
+            MultiReg(reader.packet_type, self.packet_type.status),
-            MultiReg(self.read_ptr.status, bootstrap.read_ptr_rx, odomain="cxp_gtx_rx"),
+            MultiReg(self.read_ptr.status, reader.read_ptr_rx, odomain="cxp_gtx_rx"),
         ]
         self.sync += [
-            self.pending_packet.w.eq(self.read_ptr.status != bootstrap.write_ptr_sys),
+            self.pending_packet.w.eq(self.read_ptr.status != reader.write_ptr_sys),
             If(~gtx.rx_ready,
                 self.read_ptr.status.eq(0),
             ).Elif(self.pending_packet.re & self.pending_packet.w, 
@@ -208,14 +190,12 @@ class DownConn_Interface(Module, AutoCSR):
             )
         ]
         
-        # DEBUG: 
+        self.submodules.fifo = fifo = cdr(stream.SyncFIFO(word_layout_dchar, 32, True))
-        # # add buffer to improve timing & reduce tight setup/hold time
+            
-        # self.submodules.buffer_cdc_fifo = buffer_cdc_fifo = cdr(Buffer(word_layout_dchar))
+        # Drop the K29.7 and mark the EOP for arbiter and crc cheker
-        # cdc_fifo = stream.AsyncFIFO(word_layout_dchar, 512)
+        self.submodules.eop_marker = eop_marker = cdr(EOP_Marker())
-        # self.submodules += ClockDomainsRenamer({"write": "cxp_gtx_rx", "read": "sys"})(cdc_fifo)
-        # self.submodules.debug_out = debug_out = RX_Debug_Buffer(word_layout_dchar)
 
-        rx_pipeline = [phy, dchar_decoder, trig_ack_checker, bootstrap]
+        rx_pipeline = [phy, dchar_decoder, trig_ack_checker, reader, fifo, eop_marker]
         for s, d in zip(rx_pipeline, rx_pipeline[1:]):
             self.comb += s.source.connect(d.sink)
         self.source = rx_pipeline[-1].source
@@ -230,8 +210,8 @@ class DownConn_Interface(Module, AutoCSR):
 
 
         
-class UpConn_Interface(Module, AutoCSR):
+class TX_Pipeline(Module, AutoCSR):
-    def __init__(self, phy, debug_sma, pmod_pads):
+    def __init__(self, phy):
         # Transmission Pipeline
         #
         #              32                                                  32           8
@@ -251,92 +231,254 @@ class UpConn_Interface(Module, AutoCSR):
 
         # Packet FIFOs with transmission priority
         # 0: Trigger packet
-        self.submodules.trig = trig = TX_Trigger()
+        self.submodules.trig = trig = Trigger_Inserter()
         
-        # # DEBUG: INPUT
-        self.trig_stb = CSR()
-        self.trig_delay = CSRStorage(8) 
-        self.linktrigger = CSRStorage()
 
-        # self.sync += [
-        #     trig.stb.eq(self.trig_stb.re),
-        #     trig.delay.eq(self.trig_delay.storage),
-        #     trig.linktrig_mode.eq(self.linktrigger.storage),
-        # ]
-
-        
         # 1: IO acknowledgment for trigger packet
         self.submodules.trig_ack = trig_ack = Trigger_ACK_Inserter()
 
-        # DEBUG: INPUT
-        self.ack = CSR()
-        self.sync += trig_ack.stb.eq(self.ack.re),
-
 
         # 2: All other packets (data & test packet)
         # Control is not timing dependent, all the data packets are handled in firmware
-        self.submodules.bootstrap = bootstrap = TX_Bootstrap()
+        self.submodules.writer = writer = Control_Packet_Writer()
 
+        # writer memory control interface
+        self.writer_word_len = CSRStorage(log2_int(buffer_depth))
+        self.writer_stb = CSR()
+        self.writer_stb_testseq = CSR()
+        self.writer_busy = CSRStatus()
+
+        self.sync += [
+            writer.word_len.eq(self.writer_word_len.storage),
+            writer.stb.eq(self.writer_stb.re),
+            writer.stb_testseq.eq(self.writer_stb_testseq.re),
+            self.writer_busy.status.eq(writer.busy),
+        ]
+
+        # Misc
         self.submodules.pak_wrp = pak_wrp = Packet_Wrapper()
         self.submodules.idle = idle = Idle_Word_Inserter()
-
         self.submodules.converter = converter = stream.StrideConverter(word_layout, char_layout)
 
-        tx_pipeline = [bootstrap, pak_wrp, idle, trig_ack, converter, trig, phy]
+        tx_pipeline = [writer, pak_wrp, idle, trig_ack, converter, trig, phy]
         for s, d in zip(tx_pipeline, tx_pipeline[1:]):
             self.comb += s.source.connect(d.sink)
 
-class CXP_Frame_Buffer(Module, AutoCSR):
+class CXP_Frame_Pipeline(Module, AutoCSR):
     # optimal stream packet size is 2 KiB - Section 9.5.2 (CXP-001-2021)
-    def __init__(self, downconns, pmod_pads, packet_size=16384, n_buffer=2):
+    def __init__(self, pipelines, pmod_pads, roi_engine_count=1, res_width=16, count_width=31, master=0, packet_size=16384):
-        n_downconn = len(downconns)
+        n_channels = len(pipelines)
-        
+        assert n_channels > 0
-        framebuffers = []
+        assert count_width <= 31
-        arr_csr = []
-        cdr = ClockDomainsRenamer("cxp_gtx_rx")
-        for i in range(n_buffer):
-            # TODO: change this to rtio
-            if i > 0:
-                name = "buffer_" + str(i) + "_routingid"
-                csr = CSRStorage(char_width, name=name, reset=i)
-                arr_csr.append(csr)
-                setattr(self, name, csr)
 
+        # Trigger rtio
+        nbit_trigdelay = 8
+        nbit_linktrig = 1
+        self.trigger = rtlink.Interface(
+            rtlink.OInterface(nbit_trigdelay + nbit_linktrig),
+            rtlink.IInterface(word_width, timestamped=False)
+        )
+
+        self.sync.rio += [
+            If(self.trigger.o.stb,
+                pipelines[master].tx.trig.delay.eq(self.trigger.o.data[nbit_linktrig:]),
+                pipelines[master].tx.trig.linktrig_mode.eq(self.trigger.o.data[:nbit_linktrig]),
+            ),
+            pipelines[master].tx.trig.stb.eq(self.trigger.o.stb),
+        ]
+
+
+        # ROI rtio
+         
+        # 4 cfg (x0, y0, x1, y1) per roi_engine
+        self.config = rtlink.Interface(rtlink.OInterface(res_width, bits_for(4*roi_engine_count-1)))
+
+        # select which roi engine can output rtio_input signal
+        self.gate_data = rtlink.Interface(
+            rtlink.OInterface(roi_engine_count),
+            # the 32th bits is for sentinel (gate detection)
+            rtlink.IInterface(count_width+1, timestamped=False)
+        )
+
+        self.crc_error_cnt = CSRStatus(16)
+        self.crc_error_reset = CSR()
+
+        # DEBUG: csr
+        self.arbiter_active_ch = CSRStatus(n_channels)
+
+        self.roi_counter = CSRStatus(count_width)
+        self.roi_update = CSR()
+        self.pix_y = CSRStatus(res_width)
+
+        self.header_l_size = CSRStatus(3*char_width)
+        self.header_x_size = CSRStatus(3*char_width)
+        self.header_y_size = CSRStatus(3*char_width)
+        self.header_new_line = CSRStatus(3*char_width)
+        
+        # # #
+        
+        cdr = ClockDomainsRenamer("cxp_gtx_rx")
+    
+        debug_out = False
+
+        if not debug_out:
+            self.submodules.pixel_pipeline = pixel_pipeline = cdr(Pixel_Pipeline(res_width, count_width, packet_size))
+
+            # CRC error counter
+            self.submodules.crc_reset_ps = crc_reset_ps = PulseSynchronizer("sys", "cxp_gtx_rx")
+            self.sync += crc_reset_ps.i.eq(self.crc_error_reset.re)
+            crc_error_r = Signal()
+            crc_error_cnt_rx = Signal.like(self.crc_error_cnt.status)
+
+            self.sync.cxp_gtx_rx += [
+                # to improve timinig
+                crc_error_r.eq(pixel_pipeline.crc_checker.error),
+                If(crc_error_r,
+                    crc_error_cnt_rx.eq(crc_error_cnt_rx + 1),
+                ).Elif(crc_reset_ps.o,
+                    crc_error_cnt_rx.eq(crc_error_cnt_rx + 1),
+                ),
+            ]
+            self.specials += MultiReg(crc_error_cnt_rx, self.crc_error_cnt.status)
+
+            # RTIO interface
+
+            n = 0
+            cfg = pixel_pipeline.roi.cfg
+            for offset, target in enumerate([cfg.x0, cfg.y0, cfg.x1, cfg.y1]):
+                roi_boundary = Signal.like(target)
+                self.sync.rio += If(self.config.o.stb & (self.config.o.address == 4*n+offset),
+                                roi_boundary.eq(self.config.o.data))
+                self.specials += MultiReg(roi_boundary, target, "cxp_gtx_rx")
+
+
+
+            roi_out = pixel_pipeline.roi.out
+            update = Signal()
+            self.submodules.ps = ps = PulseSynchronizer("cxp_gtx_rx", "sys")
+            self.sync.cxp_gtx_rx += ps.i.eq(roi_out.update)
+            self.sync += update.eq(ps.o)
+
+            sentinel = 2**count_width
+            count_sys = Signal.like(roi_out.count)
+
+            self.specials += MultiReg(roi_out.count, count_sys),
+            self.sync.rio += [
+                # TODO: add gating
+                self.gate_data.i.stb.eq(update),
+                # without the slice, unspecified bits will be 1 for some reason
+                # i.e. data[count_wdith:] = 0b111111... when using data.eq(count_sys)
+                self.gate_data.i.data[:count_width].eq(count_sys),
+            ]
+
+            
+
+            # TODO: fix count_sys seems like end of frame is broken
+            # BUG: it maybe related to the KCode bug that only happens after frameheader decoder COPY (i.e. when sending pixel data)
+            # DEBUG:
+
+            new_line_cnt_rx, new_line_cnt_sys = Signal(3*char_width), Signal(3*char_width)
+            l_size_rx, l_size_sys = Signal(3*char_width), Signal(3*char_width)
+            x_size_rx, x_size_sys = Signal(3*char_width), Signal(3*char_width)
+            y_size_rx, y_size_sys = Signal(3*char_width), Signal(3*char_width)
+            y_pix_rx, y_pix_sys = Signal(res_width), Signal(res_width)
+            self.sync.cxp_gtx_rx += [
+                If(pixel_pipeline.header_decoder.new_line,
+                    new_line_cnt_rx.eq(new_line_cnt_rx + 1),
+                ),
+
+                l_size_rx.eq(pixel_pipeline.header_decoder.metadata.l_size),
+                x_size_rx.eq(pixel_pipeline.header_decoder.metadata.x_size),
+                y_size_rx.eq(pixel_pipeline.header_decoder.metadata.y_size),                
+
+                y_pix_rx.eq(pixel_pipeline.parser.pixel4x[0].y),
+            ]
+            self.specials += [
+                MultiReg(new_line_cnt_rx, new_line_cnt_sys),
+                MultiReg(l_size_rx, l_size_sys),
+                MultiReg(x_size_rx, x_size_sys),
+                MultiReg(y_size_rx, y_size_sys),
+                MultiReg(y_pix_rx, y_pix_sys),
+            ]
+            self.sync += [
+                self.header_new_line.status.eq(new_line_cnt_sys),
+                self.pix_y.status.eq(y_pix_sys),
+                self.header_l_size.status.eq(l_size_sys),
+                self.header_x_size.status.eq(x_size_sys),
+                self.header_y_size.status.eq(y_size_sys),
+                self.roi_counter.status.eq(count_sys),
+                If(update,
+                    self.roi_update.w.eq(1),
+                ).Elif(self.roi_update.re,
+                    self.roi_update.w.eq(0),
+                ),
+            ]
+
+        else:
+            # DEBUG:
             crc_checker = cdr(CXPCRC32_Checker())
 
             # TODO: handle full buffer gracefully
-
             # TODO: investigate why there is a heartbeat message in the middle of the frame with k27.7 code too???
             # NOTE: sometimes there are 0xFBFBFBFB K=0b1111
             # perhaps the buffer is full overflowing and doing strange stuff             
 
             # it should be mem block not "cycle buffer"
             # self.submodules.dropper = dropper = cdr(DChar_Dropper())
+            buffer = cdr(Buffer(word_layout_dchar)) # crcchecker timinig is bad
             buffer_cdc_fifo = cdr(Buffer(word_layout_dchar)) # to improve timing
-            cdc_fifo = stream.AsyncFIFO(word_layout_dchar, 2**log2_int(packet_size//word_dw))
+            cdc_fifo = stream.AsyncFIFO(word_layout_dchar, 2**log2_int(packet_size//word_width))
-            self.submodules += crc_checker, buffer_cdc_fifo
+            self.submodules += buffer, crc_checker, buffer_cdc_fifo
             self.submodules += ClockDomainsRenamer({"write": "cxp_gtx_rx", "read": "sys"})(cdc_fifo)
+            self.submodules.debug_out = debug_out = RX_Debug_Buffer(word_layout_dchar, 2**log2_int(packet_size//word_width))
 
-            pipeline = [crc_checker, buffer_cdc_fifo, cdc_fifo]
+            pipeline = [buffer, crc_checker, buffer_cdc_fifo, cdc_fifo, debug_out]
             for s, d in zip(pipeline, pipeline[1:]):
                 self.comb += s.source.connect(d.sink)
-            framebuffers.append(pipeline[0])
-
-            # DEBUG:
-            if i == 0:
-                self.submodules.debug_out = debug_out = RX_Debug_Buffer(word_layout_dchar, 2**log2_int(packet_size//word_dw))
-                self.comb += pipeline[-1].source.connect(debug_out.sink)
-            else:
-                # remove any backpressure 
-                self.comb += pipeline[-1].source.ack.eq(1)
 
 
-        self.submodules.router = router = cdr(Frame_Packet_Router(downconns, framebuffers, packet_size, pmod_pads))
 
-        for i, csr in enumerate(arr_csr):
+        #
-            self.specials += MultiReg(csr.storage, router.routing_table[i], odomain="cxp_gtx_rx"),
+        #                        +---------+     +-------------+
+        # downconn pipline ----->|         |     |             |------> crc checker ------> raw stream data 
+        #                        | arbiter |---->| broadcaster |
+        # downconn pipline ----->|         |     |             |------> crc checker ------> raw stream data 
+        #                        +---------+     +-------------+
+        #
 
+
+        self.submodules.arbiter = arbiter = cdr(Stream_Arbiter(n_channels))
+        self.submodules.broadcaster = broadcaster = cdr(Stream_Broadcaster())
+
+        # Connect pipeline
+        for i, p in enumerate(pipelines):
+            # Assume downconns pipeline already marks the eop
+            self.comb += p.rx.source.connect(arbiter.sinks[i])
+
+        self.comb += arbiter.source.connect(broadcaster.sink)
+
+        if not debug_out:
+            self.comb += broadcaster.sources[0].connect(pixel_pipeline.sink),
+        else:
+            self.comb += broadcaster.sources[0].connect(pipeline[0].sink),
+
+        # Control interface
         # only the simple topology MASTER:ch0, extension:ch1,2,3 is supported right now
-        active_extensions = Signal(max=n_downconn)
+        active_channels_sys = Signal(n_channels)
-        self.sync += active_extensions.eq(reduce(add, [d.rx_ready.status for d in downconns[1:]]))
+        for i, p in enumerate(pipelines):
-        self.specials += MultiReg(active_extensions, router.n_ext_active, odomain="cxp_gtx_rx"),
+            # TODO: change this to non csr signal?
+            self.sync += active_channels_sys[i].eq(p.rx.ready.status)
+        self.specials += MultiReg(active_channels_sys, arbiter.active_channels, odomain="cxp_gtx_rx"),
+
+        # DEBUG:
+        self.sync += self.arbiter_active_ch.status.eq(active_channels_sys)
+
+        for i, p in enumerate(pipelines):
+            # self.comb += p.rx.source.ack.eq(1)
+
+            rx_stb = Signal()
+            self.sync.cxp_gtx_rx += rx_stb.eq(p.rx.source.stb)
+            self.specials += [
+                Instance("OBUF", i_I=rx_stb, o_O=pmod_pads[i]),
+                # Instance("OBUF", i_I=arbiter.sinks[i].stb, o_O=pmod_pads[i]),
+            ]