drtio: monitor RTIOClockMultiplier PLL (#1155)

Debugging by Tom Harty

artiq/firmware/runtime/rtio_mgt.rs

@@ -380,6 +380,10 @@ fn async_error_thread(io: Io) {
 pub fn startup(io: &Io, aux_mutex: &Mutex,
         routing_table: &Urc<RefCell<drtio_routing::RoutingTable>>,
         up_destinations: &Urc<RefCell<[bool; drtio_routing::DEST_COUNT]>>) {
+    // The RTIO CRG may depend on the DRTIO transceiver clock.
+    // Initialize DRTIO first to bring up transceiver clocking.
+    drtio::startup(io, aux_mutex, routing_table, up_destinations);
+
     #[cfg(has_rtio_crg)]
     {
         #[cfg(has_rtio_clock_switch)]
@@ -422,7 +426,6 @@ pub fn startup(io: &Io, aux_mutex: &Mutex,
         csr::rtio_core::reset_phy_write(1);
     }
 
-    drtio::startup(io, aux_mutex, routing_table, up_destinations);
     io.spawn(4096, async_error_thread);
 }
 

artiq/firmware/satman/main.rs

@@ -351,6 +351,23 @@ fn drtiosat_process_errors() {
     }
 }
 
+
+#[cfg(has_rtio_crg)]
+fn init_rtio_crg() {
+    unsafe {
+        csr::rtio_crg::pll_reset_write(0);
+    }
+    clock::spin_us(150);
+    let locked = unsafe { csr::rtio_crg::pll_locked_read() != 0 };
+    if !locked {
+        error!("RTIO clock failed");
+    }
+}
+
+#[cfg(not(has_rtio_crg))]
+fn init_rtio_crg() { }
+
+
 #[cfg(rtio_frequency = "150.0")]
 const SI5324_SETTINGS: si5324::FrequencySettings
     = si5324::FrequencySettings {
@@ -388,6 +405,7 @@ pub extern fn main() -> i32 {
     unsafe {
         csr::drtio_transceiver::stable_clkin_write(1);
     }
+    init_rtio_crg();
 
     #[cfg(has_allaki_atts)]
     board_artiq::hmc542::program_all(8/*=4dB*/);

artiq/gateware/targets/kasli.py

@@ -592,19 +592,23 @@ class Tester(_StandaloneBase):
         self.add_rtio(self.rtio_channels)
 
 
-class _RTIOClockMultiplier(Module):
+class _RTIOClockMultiplier(Module, AutoCSR):
     def __init__(self, rtio_clk_freq):
+        self.pll_reset = CSRStorage(reset=1)
+        self.pll_locked = CSRStatus()
         self.clock_domains.cd_rtiox4 = ClockDomain(reset_less=True)
 
         # See "Global Clock Network Deskew Using Two BUFGs" in ug472.
         clkfbout = Signal()
         clkfbin = Signal()
         rtiox4_clk = Signal()
+        pll_locked = Signal()
         self.specials += [
             Instance("MMCME2_BASE",
                 p_CLKIN1_PERIOD=1e9/rtio_clk_freq,
                 i_CLKIN1=ClockSignal("rtio"),
-                i_RST=ResetSignal("rtio"),
+                i_RST=self.pll_reset.storage,
+                o_LOCKED=pll_locked,
 
                 p_CLKFBOUT_MULT_F=8.0, p_DIVCLK_DIVIDE=1,
 
@@ -613,7 +617,9 @@ class _RTIOClockMultiplier(Module):
                 p_CLKOUT0_DIVIDE_F=2.0, o_CLKOUT0=rtiox4_clk,
             ),
             Instance("BUFG", i_I=clkfbout, o_O=clkfbin),
-            Instance("BUFG", i_I=rtiox4_clk, o_O=self.cd_rtiox4.clk)
+            Instance("BUFG", i_I=rtiox4_clk, o_O=self.cd_rtiox4.clk),
+
+            MultiReg(pll_locked, self.pll_locked.status)
         ]
 
 
@@ -710,7 +716,8 @@ class _MasterBase(MiniSoC, AMPSoC):
             platform.add_false_path_constraints(
                 self.crg.cd_sys.clk, gtp.rxoutclk)
 
-        self.submodules.rtio_clkmul = _RTIOClockMultiplier(rtio_clk_freq)
+        self.submodules.rtio_crg = _RTIOClockMultiplier(rtio_clk_freq)
+        self.csr_devices.append("rtio_crg")
         fix_serdes_timing_path(platform)
 
     def add_rtio(self, rtio_channels):
@@ -888,7 +895,8 @@ class _SatelliteBase(BaseSoC):
             platform.add_false_path_constraints(
                 self.crg.cd_sys.clk, gtp.rxoutclk)
 
-        self.submodules.rtio_clkmul = _RTIOClockMultiplier(rtio_clk_freq)
+        self.submodules.rtio_crg = _RTIOClockMultiplier(rtio_clk_freq)
+        self.csr_devices.append("rtio_crg")
         fix_serdes_timing_path(platform)
 
     def add_rtio(self, rtio_channels):