From 52afd4ef6b7f5154e4caa0ca0439aa92b8cf0f53 Mon Sep 17 00:00:00 2001 From: Harry Ho Date: Wed, 20 Jan 2021 11:24:20 +0800 Subject: [PATCH] kc705: add GTX multilane support, add multichannel support on master * One DRTIO master channel is enabled by default. * User can set the SMA as the 2nd master channel (by passing --drtio-sma to the argparser). * Multi-channel (i.e. with repeaters) on KC705 satellite is supported but has not been implemented yet. --- .../gateware/drtio/transceiver/gtx_7series.py | 164 ++++++++++++----- .../drtio/transceiver/gtx_7series_init.py | 172 +++++++++++++++--- artiq/gateware/targets/kc705_drtio_master.py | 77 +++++--- .../gateware/targets/kc705_drtio_satellite.py | 21 ++- 4 files changed, 332 insertions(+), 102 deletions(-) diff --git a/artiq/gateware/drtio/transceiver/gtx_7series.py b/artiq/gateware/drtio/transceiver/gtx_7series.py index e2b1378a9..5b8828f41 100644 --- a/artiq/gateware/drtio/transceiver/gtx_7series.py +++ b/artiq/gateware/drtio/transceiver/gtx_7series.py @@ -9,22 +9,23 @@ from artiq.gateware.drtio.transceiver.clock_aligner import BruteforceClockAligne from artiq.gateware.drtio.transceiver.gtx_7series_init import * -class GTX_20X(Module, TransceiverInterface): +class GTX_20X(Module): # Settings: - # * GTX reference clock (at clock_pads) @ 125MHz == coarse RTIO frequency + # * GTX reference clock @ 125MHz == coarse RTIO frequency # * GTX data width = 20 # * GTX PLL frequency @ 2.5GHz # * GTX line rate (TX & RX) @ 2.5Gb/s # * GTX TX/RX USRCLK @ 125MHz == coarse RTIO frequency - def __init__(self, clock_pads, tx_pads, rx_pads, sys_clk_freq): - encoder = ClockDomainsRenamer("rtio")( - Encoder(2, True)) - self.submodules += encoder - decoders = [ClockDomainsRenamer("rtio_rx0")( - (Decoder(True))) for _ in range(2)] - self.submodules += decoders + def __init__(self, refclk, tx_pads, rx_pads, sys_clk_freq, rtio_clk_freq=125e6, tx_mode="single", rx_mode="single"): + assert tx_mode in ["single", "master", "slave"] + assert rx_mode in ["single", "master", "slave"] - TransceiverInterface.__init__(self, [ChannelInterface(encoder, decoders)]) + self.txenable = Signal() + self.submodules.encoder = ClockDomainsRenamer("rtio_tx")( + Encoder(2, True)) + self.submodules.decoders = [ClockDomainsRenamer("rtio_rx")( + (Decoder(True))) for _ in range(2)] + self.rx_ready = Signal() # transceiver direct clock outputs # useful to specify clock constraints in a way palatable to Vivado @@ -33,25 +34,13 @@ class GTX_20X(Module, TransceiverInterface): # # # - refclk = Signal() - stable_clkin_n = Signal() - self.stable_clkin.storage.attr.add("no_retiming") - self.comb += stable_clkin_n.eq(~self.stable_clkin.storage) - self.specials += Instance("IBUFDS_GTE2", - i_CEB=stable_clkin_n, - i_I=clock_pads.p, - i_IB=clock_pads.n, - o_O=refclk - ) - cpllreset = Signal() cplllock = Signal() # TX generates RTIO clock, init must be in system domain - tx_init = GTXInit(sys_clk_freq, False) + self.submodules.tx_init = tx_init = GTXInit(sys_clk_freq, False, mode=tx_mode) # RX receives restart commands from RTIO domain - rx_init = ClockDomainsRenamer("rtio")( - GTXInit(self.rtio_clk_freq, True)) - self.submodules += tx_init, rx_init + self.submodules.rx_init = rx_init = ClockDomainsRenamer("rtio_tx")( + GTXInit(rtio_clk_freq, True, mode=rx_mode)) self.comb += [ cpllreset.eq(tx_init.cpllreset), tx_init.cplllock.eq(cplllock), @@ -60,6 +49,7 @@ class GTX_20X(Module, TransceiverInterface): txdata = Signal(20) rxdata = Signal(20) + # Note: the following parameters were set after consulting AR45360 self.specials += \ Instance("GTXE2_CHANNEL", # PMA Attributes @@ -100,15 +90,22 @@ class GTX_20X(Module, TransceiverInterface): i_TXOUTCLKSEL=0b11, # TX Startup/Reset + i_TXPHDLYRESET=0, + i_TXDLYBYPASS=0, + i_TXPHALIGNEN=1 if tx_mode != "single" else 0, i_GTTXRESET=tx_init.gtXxreset, o_TXRESETDONE=tx_init.Xxresetdone, i_TXDLYSRESET=tx_init.Xxdlysreset, o_TXDLYSRESETDONE=tx_init.Xxdlysresetdone, + i_TXPHINIT=tx_init.txphinit if tx_mode != "single" else 0, + o_TXPHINITDONE=tx_init.txphinitdone if tx_mode != "single" else Signal(), + i_TXPHALIGN=tx_init.Xxphalign if tx_mode != "single" else 0, + i_TXDLYEN=tx_init.Xxdlyen if tx_mode != "single" else 0, o_TXPHALIGNDONE=tx_init.Xxphaligndone, i_TXUSERRDY=tx_init.Xxuserrdy, p_TXPMARESET_TIME=1, p_TXPCSRESET_TIME=1, - i_TXINHIBIT=~self.txenable.storage, + i_TXINHIBIT=~self.txenable, # TX data p_TX_DATA_WIDTH=20, @@ -116,18 +113,23 @@ class GTX_20X(Module, TransceiverInterface): i_TXCHARDISPMODE=Cat(txdata[9], txdata[19]), i_TXCHARDISPVAL=Cat(txdata[8], txdata[18]), i_TXDATA=Cat(txdata[:8], txdata[10:18]), - i_TXUSRCLK=ClockSignal("rtio"), - i_TXUSRCLK2=ClockSignal("rtio"), + i_TXUSRCLK=ClockSignal("rtio_tx"), + i_TXUSRCLK2=ClockSignal("rtio_tx"), # TX electrical i_TXBUFDIFFCTRL=0b100, i_TXDIFFCTRL=0b1000, # RX Startup/Reset + i_RXPHDLYRESET=0, + i_RXDLYBYPASS=0, + i_RXPHALIGNEN=1 if rx_mode != "single" else 0, i_GTRXRESET=rx_init.gtXxreset, o_RXRESETDONE=rx_init.Xxresetdone, i_RXDLYSRESET=rx_init.Xxdlysreset, o_RXDLYSRESETDONE=rx_init.Xxdlysresetdone, + i_RXPHALIGN=rx_init.Xxphalign if rx_mode != "single" else 0, + i_RXDLYEN=rx_init.Xxdlyen if rx_mode != "single" else 0, o_RXPHALIGNDONE=rx_init.Xxphaligndone, i_RXUSERRDY=rx_init.Xxuserrdy, p_RXPMARESET_TIME=1, @@ -158,8 +160,8 @@ class GTX_20X(Module, TransceiverInterface): i_RXSYSCLKSEL=0b00, i_RXOUTCLKSEL=0b010, o_RXOUTCLK=self.rxoutclk, - i_RXUSRCLK=ClockSignal("rtio_rx0"), - i_RXUSRCLK2=ClockSignal("rtio_rx0"), + i_RXUSRCLK=ClockSignal("rtio_rx"), + i_RXUSRCLK2=ClockSignal("rtio_rx"), # RX Clock Correction Attributes p_CLK_CORRECT_USE="FALSE", @@ -240,48 +242,110 @@ class GTX_20X(Module, TransceiverInterface): o_GTXTXN=tx_pads.n, # Other parameters - p_PCS_RSVD_ATTR=0x000, # PCS_RSVD_ATTR[1] = 0: TX Single Lane Auto Mode - # [2] = 0: RX Single Lane Auto Mode - # [8] = 0: OOB is disabled + p_PCS_RSVD_ATTR=( + (tx_mode != "single") << 1 | # PCS_RSVD_ATTR[1] = 0: TX Single Lane Auto Mode + # = 1: TX Manual Mode + (rx_mode != "single") << 2 | # [2] = 0: RX Single Lane Auto Mode + # = 1: RX Manual Mode + 0 << 8 # [8] = 0: OOB is disabled + ), i_RXELECIDLEMODE=0b11, # RXELECIDLEMODE = 0b11: OOB is disabled p_RX_DFE_LPM_HOLD_DURING_EIDLE=0b0, p_ES_EYE_SCAN_EN="TRUE", # Must be TRUE for GTX ) + # TX clocking tx_reset_deglitched = Signal() tx_reset_deglitched.attr.add("no_retiming") self.sync += tx_reset_deglitched.eq(~tx_init.done) - self.specials += [ - Instance("BUFG", i_I=self.txoutclk, o_O=self.cd_rtio.clk), - AsyncResetSynchronizer(self.cd_rtio, tx_reset_deglitched) - ] + self.clock_domains.cd_rtio_tx = ClockDomain() + if tx_mode == "single" or tx_mode == "master": + self.specials += Instance("BUFG", i_I=self.txoutclk, o_O=self.cd_rtio_tx.clk) + self.specials += AsyncResetSynchronizer(self.cd_rtio_tx, tx_reset_deglitched) + + # RX clocking rx_reset_deglitched = Signal() rx_reset_deglitched.attr.add("no_retiming") self.sync.rtio += rx_reset_deglitched.eq(~rx_init.done) - self.specials += [ - Instance("BUFG", i_I=self.rxoutclk, o_O=self.cd_rtio_rx0.clk), - AsyncResetSynchronizer(self.cd_rtio_rx0, rx_reset_deglitched) - ] + self.clock_domains.cd_rtio_rx = ClockDomain() + if rx_mode == "single" or rx_mode == "master": + self.specials += Instance("BUFG", i_I=self.rxoutclk, o_O=self.cd_rtio_rx.clk), + self.specials += AsyncResetSynchronizer(self.cd_rtio_rx, rx_reset_deglitched) - chan = self.channels[0] self.comb += [ - txdata.eq(Cat(chan.encoder.output[0], chan.encoder.output[1])), - chan.decoders[0].input.eq(rxdata[:10]), - chan.decoders[1].input.eq(rxdata[10:]) + txdata.eq(Cat(self.encoder.output[0], self.encoder.output[1])), + self.decoders[0].input.eq(rxdata[:10]), + self.decoders[1].input.eq(rxdata[10:]) ] - clock_aligner = ClockDomainsRenamer({"rtio_rx": "rtio_rx0"})( - BruteforceClockAligner(0b0101111100, self.rtio_clk_freq)) + clock_aligner = BruteforceClockAligner(0b0101111100, rtio_clk_freq) self.submodules += clock_aligner self.comb += [ clock_aligner.rxdata.eq(rxdata), rx_init.restart.eq(clock_aligner.restart), - chan.rx_ready.eq(clock_aligner.ready) + self.rx_ready.eq(clock_aligner.ready) ] -class GTX_1000BASE_BX10(GTX_20X): - rtio_clk_freq = 125e6 + +class GTX(Module, TransceiverInterface): + def __init__(self, clock_pads, tx_pads, rx_pads, sys_clk_freq, rtio_clk_freq=125e6, master=0): + assert len(tx_pads) == len(rx_pads) + self.nchannels = nchannels = len(tx_pads) + self.gtxs = [] + self.rtio_clk_freq = rtio_clk_freq + + # # # + + refclk = Signal() + stable_clkin_n = Signal() + self.specials += Instance("IBUFDS_GTE2", + i_CEB=stable_clkin_n, + i_I=clock_pads.p, + i_IB=clock_pads.n, + o_O=refclk + ) + + rtio_tx_clk = Signal() + channel_interfaces = [] + for i in range(nchannels): + if nchannels == 1: + mode = "single" + else: + mode = "master" if i == master else "slave" + # Note: RX phase alignment is to be done on individual lanes, not multi-lane. + gtx = GTX_20X(refclk, tx_pads[i], rx_pads[i], sys_clk_freq, rtio_clk_freq=rtio_clk_freq, tx_mode=mode, rx_mode="single") + # Fan-out (to slave) / Fan-in (from master) of the TXUSRCLK + if mode == "slave": + self.comb += gtx.cd_rtio_tx.clk.eq(rtio_tx_clk) + else: + self.comb += rtio_tx_clk.eq(gtx.cd_rtio_tx.clk) + self.gtxs.append(gtx) + setattr(self.submodules, "gtx"+str(i), gtx) + channel_interface = ChannelInterface(gtx.encoder, gtx.decoders) + self.comb += channel_interface.rx_ready.eq(gtx.rx_ready) + channel_interfaces.append(channel_interface) + + self.submodules.tx_phase_alignment = GTXInitPhaseAlignment([gtx.tx_init for gtx in self.gtxs]) + + TransceiverInterface.__init__(self, channel_interfaces) + for n, gtx in enumerate(self.gtxs): + self.comb += [ + stable_clkin_n.eq(~self.stable_clkin.storage), + gtx.txenable.eq(self.txenable.storage[n]) + ] + + # Connect master's `rtio_tx` clock to `rtio` clock + self.comb += [ + self.cd_rtio.clk.eq(self.gtxs[master].cd_rtio_tx.clk), + self.cd_rtio.rst.eq(reduce(or_, [gtx.cd_rtio_tx.rst for gtx in self.gtxs])) + ] + # Connect slave i's `rtio_rx` clock to `rtio_rxi` clock + for i in range(nchannels): + self.comb += [ + getattr(self, "cd_rtio_rx" + str(i)).clk.eq(self.gtxs[i].cd_rtio_rx.clk), + getattr(self, "cd_rtio_rx" + str(i)).rst.eq(self.gtxs[i].cd_rtio_rx.rst) + ] class RXSynchronizer(Module, AutoCSR): diff --git a/artiq/gateware/drtio/transceiver/gtx_7series_init.py b/artiq/gateware/drtio/transceiver/gtx_7series_init.py index 6598f2892..9e3f65a15 100644 --- a/artiq/gateware/drtio/transceiver/gtx_7series_init.py +++ b/artiq/gateware/drtio/transceiver/gtx_7series_init.py @@ -10,7 +10,14 @@ from migen.genlib.fsm import FSM class GTXInit(Module): # Based on LiteSATA by Enjoy-Digital - def __init__(self, sys_clk_freq, rx): + # Choose between Auto Mode and Manual Mode for TX/RX phase alignment with buffer bypassed: + # * Auto Mode: When only single lane is involved, as suggested by Xilinx (AR59612) + # * Manual Mode: When only multi-lane is involved, as suggested by Xilinx (AR59612) + def __init__(self, sys_clk_freq, rx, mode="single"): + assert isinstance(rx, bool) + assert mode in ["single", "master", "slave"] + self.mode = mode + self.done = Signal() self.restart = Signal() @@ -23,6 +30,21 @@ class GTXInit(Module): self.Xxdlysresetdone = Signal() self.Xxphaligndone = Signal() self.Xxuserrdy = Signal() + # GTX signals exclusive to multi-lane + if mode != "single": + self.Xxphalign = Signal() + self.Xxdlyen = Signal() + # TX only: + if not rx: + self.txphinit = Signal() + self.txphinitdone = Signal() + + # Strobe from master channel to initialize TX/RX phase alignment on slaves + self.master_phaligndone = Signal() + # Strobe from slave channels to re-enable TX/RX delay alignment on master; + # To be combinatorially AND-ed from all slave's `done` + if mode == "master": + self.slaves_phaligndone = Signal() # # # @@ -37,6 +59,9 @@ class GTXInit(Module): MultiReg(self.Xxdlysresetdone, Xxdlysresetdone), MultiReg(self.Xxphaligndone, Xxphaligndone), ] + if mode != "single": + txphinitdone = Signal() + self.specials += MultiReg(self.txphinitdone, txphinitdone) # Deglitch FSM outputs driving transceiver asynch inputs gtXxreset = Signal() @@ -47,6 +72,16 @@ class GTXInit(Module): self.Xxdlysreset.eq(Xxdlysreset), self.Xxuserrdy.eq(Xxuserrdy) ] + if mode != "single": + Xxphalign = Signal() + Xxdlyen = Signal() + self.sync += [ + self.Xxphalign.eq(Xxphalign), + self.Xxdlyen.eq(Xxdlyen) + ] + if not rx: + txphinit = Signal() + self.sync += self.txphinit.eq(txphinit) # After configuration, transceiver resets have to stay low for # at least 500ns (see AR43482) @@ -67,6 +102,7 @@ class GTXInit(Module): cdr_stable_timer = WaitTimer(1024) self.submodules += cdr_stable_timer + # Rising edge detection for phase alignment "done" Xxphaligndone_r = Signal(reset=1) Xxphaligndone_rising = Signal() self.sync += Xxphaligndone_r.eq(Xxphaligndone) @@ -93,34 +129,103 @@ class GTXInit(Module): startup_fsm.act("RELEASE_GTH_RESET", Xxuserrdy.eq(1), cdr_stable_timer.wait.eq(1), - If(Xxresetdone & cdr_stable_timer.done, NextState("ALIGN")) + If(Xxresetdone & cdr_stable_timer.done, NextState("DELAY_ALIGN")) ) else: startup_fsm.act("RELEASE_GTH_RESET", Xxuserrdy.eq(1), - If(Xxresetdone, NextState("ALIGN")) + If(Xxresetdone, NextState("DELAY_ALIGN")) ) - # Start delay alignment (pulse) - startup_fsm.act("ALIGN", - Xxuserrdy.eq(1), - Xxdlysreset.eq(1), - NextState("WAIT_ALIGN") - ) - # Wait for delay alignment - startup_fsm.act("WAIT_ALIGN", - Xxuserrdy.eq(1), - If(Xxdlysresetdone, NextState("WAIT_FIRST_ALIGN_DONE")) - ) - # Wait 2 rising edges of rxphaligndone - # (from UG476 in buffer bypass config) - startup_fsm.act("WAIT_FIRST_ALIGN_DONE", - Xxuserrdy.eq(1), - If(Xxphaligndone_rising, NextState("WAIT_SECOND_ALIGN_DONE")) - ) - startup_fsm.act("WAIT_SECOND_ALIGN_DONE", - Xxuserrdy.eq(1), - If(Xxphaligndone_rising, NextState("READY")) - ) + + # State(s) exclusive to Auto Mode: + if mode == "single": + # Start delay alignment (pulse) + startup_fsm.act("DELAY_ALIGN", + Xxuserrdy.eq(1), + Xxdlysreset.eq(1), + NextState("WAIT_DELAY_ALIGN") + ) + # Wait for delay alignment + startup_fsm.act("WAIT_DELAY_ALIGN", + Xxuserrdy.eq(1), + If(Xxdlysresetdone, NextState("WAIT_FIRST_PHASE_ALIGN_DONE")) + ) + # Wait 2 rising edges of rxphaligndone + # (from UG476 in buffer bypass config) + startup_fsm.act("WAIT_FIRST_PHASE_ALIGN_DONE", + Xxuserrdy.eq(1), + If(Xxphaligndone_rising, NextState("WAIT_SECOND_PHASE_ALIGN_DONE")) + ) + startup_fsm.act("WAIT_SECOND_PHASE_ALIGN_DONE", + Xxuserrdy.eq(1), + If(Xxphaligndone_rising, NextState("READY")) + ) + + # State(s) exclusive to Manual Mode: + else: + # Start delay alignment (hold) + startup_fsm.act("DELAY_ALIGN", + Xxuserrdy.eq(1), + Xxdlysreset.eq(1), + If(Xxdlysresetdone, + # TX master: proceed to initialize phase alignment manually + (NextState("PHASE_ALIGN_INIT") if not rx else + # RX master: proceed to start phase alignment manually + NextState("PHASE_ALIGN")) if mode == "master" else + # TX/RX slave: wait for phase alignment "done" on master + NextState("WAIT_MASTER") + ) + ) + if mode == "slave": + # TX slave: Wait for phase alignment "done" on master + startup_fsm.act("WAIT_MASTER", + Xxuserrdy.eq(1), + If(self.master_phaligndone, + # TX slave: proceed to initialize phase alignment manually + NextState("PHASE_ALIGN_INIT") if not rx else + # RX slave: proceed to start phase alignment manually + NextState("PHASE_ALIGN") + ) + ) + if not rx: + # TX master/slave: Initialize phase alignment, wait rising edge on "done" + startup_fsm.act("PHASE_ALIGN_INIT", + Xxuserrdy.eq(1), + txphinit.eq(1), + If(txphinitdone, NextState("PHASE_ALIGN")) + ) + # Do phase ealignment, wait rising edge on "done" + startup_fsm.act("PHASE_ALIGN", + Xxuserrdy.eq(1), + Xxphalign.eq(1), + If(Xxphaligndone_rising, + # TX/RX master: proceed to set T/RXDLYEN + NextState("FIRST_DLYEN") if mode == "master" else + # TX/RX slave: proceed to signal master + NextState("READY") + ) + ) + if mode == "master": + # Enable delay alignment in manual mode, wait rising edge on phase alignment "done" + startup_fsm.act("FIRST_DLYEN", + Xxuserrdy.eq(1), + Xxdlyen.eq(1), + If(Xxphaligndone_rising, NextState("WAIT_SLAVES")) + ) + # Wait for phase alignment "done" on all slaves + startup_fsm.act("WAIT_SLAVES", + Xxuserrdy.eq(1), + self.master_phaligndone.eq(1), + If(self.slaves_phaligndone, NextState("SECOND_DLYEN")) + ) + # Re-enable delay alignment in manual mode, wait rising edge on phase alignment "done" + startup_fsm.act("SECOND_DLYEN", + Xxuserrdy.eq(1), + Xxdlyen.eq(1), + If(Xxphaligndone_rising, NextState("READY")) + ) + + # Transceiver is ready, alignment can be restarted startup_fsm.act("READY", Xxuserrdy.eq(1), self.done.eq(1), @@ -128,6 +233,25 @@ class GTXInit(Module): ) +class GTXInitPhaseAlignment(Module): + # Interconnect of phase alignment "done" signals for Manual Mode multi-lane + def __init__(self, gtx_inits): + master_phaligndone = Signal() # Fan-out to `slave.master_phaligndone`s + slaves_phaligndone = Signal(reset=1) # ANDed from `slave.done`s + # Slave channels + for gtx_init in gtx_inits: + if gtx_init.mode == "slave": + self.comb += gtx_init.master_phaligndone.eq(master_phaligndone) + slaves_phaligndone = slaves_phaligndone & gtx_init.done + # Master channels + for gtx_init in gtx_inits: + if gtx_init.mode == "master": + self.comb += [ + master_phaligndone.eq(gtx_init.master_phaligndone), + gtx_init.slaves_phaligndone.eq(slaves_phaligndone) + ] + + # Changes the phase of the transceiver RX clock to align the comma to # the LSBs of RXDATA, fixing the latency. # diff --git a/artiq/gateware/targets/kc705_drtio_master.py b/artiq/gateware/targets/kc705_drtio_master.py index 436d0ada2..7d322e795 100755 --- a/artiq/gateware/targets/kc705_drtio_master.py +++ b/artiq/gateware/targets/kc705_drtio_master.py @@ -30,7 +30,7 @@ class Master(MiniSoC, AMPSoC): } mem_map.update(MiniSoC.mem_map) - def __init__(self, gateware_identifier_str=None, **kwargs): + def __init__(self, gateware_identifier_str=None, drtio_sma=False, **kwargs): MiniSoC.__init__(self, cpu_type="or1k", sdram_controller_type="minicon", @@ -52,11 +52,14 @@ class Master(MiniSoC, AMPSoC): platform = self.platform self.comb += platform.request("sfp_tx_disable_n").eq(1) - tx_pads = platform.request("sfp_tx") - rx_pads = platform.request("sfp_rx") + tx_pads = [platform.request("sfp_tx")] + rx_pads = [platform.request("sfp_rx")] + if drtio_sma: + tx_pads.append(platform.request("user_sma_mgt_tx")) + rx_pads.append(platform.request("user_sma_mgt_rx")) # 1000BASE_BX10 Ethernet compatible, 125MHz RTIO clock - self.submodules.drtio_transceiver = gtx_7series.GTX_1000BASE_BX10( + self.submodules.drtio_transceiver = gtx_7series.GTX( clock_pads=platform.request("si5324_clkout"), tx_pads=tx_pads, rx_pads=rx_pads, @@ -64,24 +67,40 @@ class Master(MiniSoC, AMPSoC): self.csr_devices.append("drtio_transceiver") self.submodules.rtio_tsc = rtio.TSC("async", glbl_fine_ts_width=3) - cdr = ClockDomainsRenamer({"rtio_rx": "rtio_rx0"}) - self.submodules.drtio0 = cdr(DRTIOMaster( - self.rtio_tsc, self.drtio_transceiver.channels[0])) - self.csr_devices.append("drtio0") + drtio_csr_group = [] + drtioaux_csr_group = [] + drtioaux_memory_group = [] + drtio_cri = [] + for i in range(len(self.drtio_transceiver.channels)): + core_name = "drtio" + str(i) + coreaux_name = "drtioaux" + str(i) + memory_name = "drtioaux" + str(i) + "_mem" + drtio_csr_group.append(core_name) + drtioaux_csr_group.append(coreaux_name) + drtioaux_memory_group.append(memory_name) - self.submodules.drtioaux0 = cdr(DRTIOAuxController( - self.drtio0.link_layer)) - self.csr_devices.append("drtioaux0") - self.add_wb_slave(self.mem_map["drtioaux"], 0x800, - self.drtioaux0.bus) - self.add_memory_region("drtioaux0_mem", self.mem_map["drtioaux"] | self.shadow_base, 0x800) + cdr = ClockDomainsRenamer({"rtio_rx": "rtio_rx" + str(i)}) + core = cdr(DRTIOMaster( + self.rtio_tsc, self.drtio_transceiver.channels[i])) + setattr(self.submodules, core_name, core) + drtio_cri.append(core.cri) + self.csr_devices.append(core_name) + + coreaux = cdr(DRTIOAuxController(core.link_layer)) + setattr(self.submodules, coreaux_name, coreaux) + self.csr_devices.append(coreaux_name) + + memory_address = self.mem_map["drtioaux"] + 0x800*i + self.add_wb_slave(memory_address, 0x800, + coreaux.bus) + self.add_memory_region(memory_name, memory_address | self.shadow_base, 0x800) self.config["HAS_DRTIO"] = None self.config["HAS_DRTIO_ROUTING"] = None - self.add_csr_group("drtio", ["drtio0"]) - self.add_csr_group("drtioaux", ["drtioaux0"]) - self.add_memory_group("drtioaux_mem", ["drtioaux0_mem"]) + self.add_csr_group("drtio", drtio_csr_group) + self.add_csr_group("drtioaux", drtioaux_csr_group) + self.add_memory_group("drtioaux_mem", drtioaux_memory_group) self.config["RTIO_FREQUENCY"] = str(self.drtio_transceiver.rtio_clk_freq/1e6) self.submodules.si5324_rst_n = gpio.GPIOOut(platform.request("si5324").rst_n) @@ -99,11 +118,20 @@ class Master(MiniSoC, AMPSoC): ] rtio_clk_period = 1e9/self.drtio_transceiver.rtio_clk_freq - platform.add_period_constraint(self.drtio_transceiver.txoutclk, rtio_clk_period) - platform.add_period_constraint(self.drtio_transceiver.rxoutclk, rtio_clk_period) + # Constrain TX & RX timing for the first transceiver channel + # (First channel acts as master for phase alignment for all channels' TX) + gtx0 = self.drtio_transceiver.gtxs[0] + platform.add_period_constraint(gtx0.txoutclk, rtio_clk_period) + platform.add_period_constraint(gtx0.rxoutclk, rtio_clk_period) platform.add_false_path_constraints( self.crg.cd_sys.clk, - self.drtio_transceiver.txoutclk, self.drtio_transceiver.rxoutclk) + gtx0.txoutclk, gtx0.rxoutclk) + # Constrain RX timing for the each transceiver channel + # (Each channel performs single-lane phase alignment for RX) + for gtx in self.drtio_transceiver.gtxs[1:]: + platform.add_period_constraint(gtx.rxoutclk, rtio_clk_period) + platform.add_false_path_constraints( + self.crg.cd_sys.clk, gtx0.txoutclk, gtx.rxoutclk) rtio_channels = [] for i in range(8): @@ -128,7 +156,7 @@ class Master(MiniSoC, AMPSoC): self.register_kernel_cpu_csrdevice("rtio_dma") self.submodules.cri_con = rtio.CRIInterconnectShared( [self.rtio.cri, self.rtio_dma.cri], - [self.rtio_core.cri, self.drtio0.cri], + [self.rtio_core.cri] + drtio_cri, enable_routing=True) self.register_kernel_cpu_csrdevice("cri_con") self.submodules.routing_table = rtio.RoutingTableAccess(self.cri_con) @@ -141,9 +169,14 @@ def main(): builder_args(parser) soc_kc705_args(parser) parser.set_defaults(output_dir="artiq_kc705/master") + parser.add_argument("--drtio-sma", default=False, action="store_true", + help="use the SMA connectors (RX: J17, J18, TX: J19, J20) as 2nd DRTIO channel") args = parser.parse_args() - soc = Master(**soc_kc705_argdict(args)) + argdict = dict() + argdict["drtio_sma"] = args.drtio_sma + + soc = Master(**soc_kc705_argdict(args), **argdict) build_artiq_soc(soc, builder_argdict(args)) diff --git a/artiq/gateware/targets/kc705_drtio_satellite.py b/artiq/gateware/targets/kc705_drtio_satellite.py index 048b1153e..4cb502fca 100755 --- a/artiq/gateware/targets/kc705_drtio_satellite.py +++ b/artiq/gateware/targets/kc705_drtio_satellite.py @@ -50,11 +50,11 @@ class Satellite(BaseSoC): platform = self.platform self.comb += platform.request("sfp_tx_disable_n").eq(1) - tx_pads = platform.request("sfp_tx") - rx_pads = platform.request("sfp_rx") + tx_pads = [platform.request("sfp_tx")] + rx_pads = [platform.request("sfp_rx")] # 1000BASE_BX10 Ethernet compatible, 125MHz RTIO clock - self.submodules.drtio_transceiver = gtx_7series.GTX_1000BASE_BX10( + self.submodules.drtio_transceiver = gtx_7series.GTX( clock_pads=platform.request("si5324_clkout"), tx_pads=tx_pads, rx_pads=rx_pads, @@ -105,11 +105,20 @@ class Satellite(BaseSoC): ] rtio_clk_period = 1e9/self.drtio_transceiver.rtio_clk_freq - platform.add_period_constraint(self.drtio_transceiver.txoutclk, rtio_clk_period) - platform.add_period_constraint(self.drtio_transceiver.rxoutclk, rtio_clk_period) + # Constrain TX & RX timing for the first transceiver channel + # (First channel acts as master for phase alignment for all channels' TX) + gtx0 = self.drtio_transceiver.gtxs[0] + platform.add_period_constraint(gtx0.txoutclk, rtio_clk_period) + platform.add_period_constraint(gtx0.rxoutclk, rtio_clk_period) platform.add_false_path_constraints( self.crg.cd_sys.clk, - self.drtio_transceiver.txoutclk, self.drtio_transceiver.rxoutclk) + gtx0.txoutclk, gtx0.rxoutclk) + # Constrain RX timing for the each transceiver channel + # (Each channel performs single-lane phase alignment for RX) + for gtx in self.drtio_transceiver.gtxs[1:]: + platform.add_period_constraint(gtx.rxoutclk, rtio_clk_period) + platform.add_false_path_constraints( + self.crg.cd_sys.clk, gtx.rxoutclk) rtio_channels = [] for i in range(8):