From 469a66db61da4c40d72b8627053821df169a8dbe Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Wed, 26 Sep 2018 10:52:08 +0800
Subject: [PATCH] drtio: monitor RTIOClockMultiplier PLL (#1155)

Debugging by Tom Harty
---
 artiq/firmware/runtime/rtio_mgt.rs |  5 ++++-
 artiq/firmware/satman/main.rs      | 18 ++++++++++++++++++
 artiq/gateware/targets/kasli.py    | 18 +++++++++++++-----
 3 files changed, 35 insertions(+), 6 deletions(-)

diff --git a/artiq/firmware/runtime/rtio_mgt.rs b/artiq/firmware/runtime/rtio_mgt.rs
index f2a606d86..a54e8c77a 100644
--- a/artiq/firmware/runtime/rtio_mgt.rs
+++ b/artiq/firmware/runtime/rtio_mgt.rs
@@ -257,6 +257,10 @@ fn async_error_thread(io: Io) {
 }
 
 pub fn startup(io: &Io) {
+    // The RTIO CRG may depend on the DRTIO transceiver clock.
+    // Initialize DRTIO first to bring up transceiver clocking.
+    drtio::startup(io);
+
     #[cfg(has_rtio_crg)]
     {
         #[cfg(has_rtio_clock_switch)]
@@ -296,7 +300,6 @@ pub fn startup(io: &Io) {
         }
     }
 
-    drtio::startup(io);
     init_core(true);
     io.spawn(4096, async_error_thread);
 }
diff --git a/artiq/firmware/satman/main.rs b/artiq/firmware/satman/main.rs
index ad2252955..a734e8355 100644
--- a/artiq/firmware/satman/main.rs
+++ b/artiq/firmware/satman/main.rs
@@ -229,6 +229,23 @@ fn process_errors() {
     }
 }
 
+
+#[cfg(has_rtio_crg)]
+fn init_rtio_crg() {
+    unsafe {
+        csr::rtio_crg::pll_reset_write(0);
+    }
+    clock::spin_us(150);
+    let locked = unsafe { csr::rtio_crg::pll_locked_read() != 0 };
+    if !locked {
+        error!("RTIO clock failed");
+    }
+}
+
+#[cfg(not(has_rtio_crg))]
+fn init_rtio_crg() { }
+
+
 #[cfg(rtio_frequency = "150.0")]
 const SI5324_SETTINGS: si5324::FrequencySettings
     = si5324::FrequencySettings {
@@ -285,6 +302,7 @@ pub extern fn main() -> i32 {
     unsafe {
         csr::drtio_transceiver::stable_clkin_write(1);
     }
+    init_rtio_crg();
 
     #[cfg(has_allaki_atts)]
     board_artiq::hmc542::program_all(8/*=4dB*/);
diff --git a/artiq/gateware/targets/kasli.py b/artiq/gateware/targets/kasli.py
index 34f0241dd..cb51e16ec 100755
--- a/artiq/gateware/targets/kasli.py
+++ b/artiq/gateware/targets/kasli.py
@@ -676,19 +676,23 @@ class Tester(_StandaloneBase):
             self.rtio_crg.cd_rtio.clk, self.grabber0.deserializer.cd_cl.clk)
 
 
-class _RTIOClockMultiplier(Module):
+class _RTIOClockMultiplier(Module, AutoCSR):
     def __init__(self, rtio_clk_freq):
+        self.pll_reset = CSRStorage(reset=1)
+        self.pll_locked = CSRStatus()
         self.clock_domains.cd_rtiox4 = ClockDomain(reset_less=True)
 
         # See "Global Clock Network Deskew Using Two BUFGs" in ug472.
         clkfbout = Signal()
         clkfbin = Signal()
         rtiox4_clk = Signal()
+        pll_locked = Signal()
         self.specials += [
             Instance("MMCME2_BASE",
                 p_CLKIN1_PERIOD=1e9/rtio_clk_freq,
                 i_CLKIN1=ClockSignal("rtio"),
-                i_RST=ResetSignal("rtio"),
+                i_RST=self.pll_reset.storage,
+                o_LOCKED=pll_locked,
 
                 p_CLKFBOUT_MULT_F=8.0, p_DIVCLK_DIVIDE=1,
 
@@ -697,7 +701,9 @@ class _RTIOClockMultiplier(Module):
                 p_CLKOUT0_DIVIDE_F=2.0, o_CLKOUT0=rtiox4_clk,
             ),
             Instance("BUFG", i_I=clkfbout, o_O=clkfbin),
-            Instance("BUFG", i_I=rtiox4_clk, o_O=self.cd_rtiox4.clk)
+            Instance("BUFG", i_I=rtiox4_clk, o_O=self.cd_rtiox4.clk),
+
+            MultiReg(pll_locked, self.pll_locked.status)
         ]
 
 
@@ -781,7 +787,8 @@ class _MasterBase(MiniSoC, AMPSoC):
             platform.add_false_path_constraints(
                 self.crg.cd_sys.clk, gtp.rxoutclk)
 
-        self.submodules.rtio_clkmul = _RTIOClockMultiplier(rtio_clk_freq)
+        self.submodules.rtio_crg = _RTIOClockMultiplier(rtio_clk_freq)
+        self.csr_devices.append("rtio_crg")
         fix_serdes_timing_path(platform)
 
     def add_rtio(self, rtio_channels):
@@ -906,7 +913,8 @@ class _SatelliteBase(BaseSoC):
             self.crg.cd_sys.clk,
             gtp.txoutclk, gtp.rxoutclk)
 
-        self.submodules.rtio_clkmul = _RTIOClockMultiplier(rtio_clk_freq)
+        self.submodules.rtio_crg = _RTIOClockMultiplier(rtio_clk_freq)
+        self.csr_devices.append("rtio_crg")
         fix_serdes_timing_path(platform)
 
     def add_rtio(self, rtio_channels):