drtio: moved satman fw

src/satman/Cargo.toml

@@ -0,0 +1,18 @@
+[package]
+authors = ["M-Labs"]
+name = "satman"
+version = "0.0.0"
+build = "build.rs"
+
+[lib]
+name = "satman"
+crate-type = ["staticlib"]
+path = "main.rs"
+
+[build-dependencies]
+build_misoc = { path = "../libbuild_misoc" }
+
+[dependencies]
+log = { version = "0.4", default-features = false }
+board_misoc = { path = "../libboard_misoc", features = ["uart_console", "log"] }
+board_artiq = { path = "../libboard_artiq" }

src/satman/Makefile

@@ -0,0 +1,21 @@
+include ../include/generated/variables.mak
+include $(MISOC_DIRECTORY)/software/common.mak
+
+LDFLAGS += -L../libbase
+
+RUSTFLAGS += -Cpanic=abort
+
+all:: satman.bin satman.fbi
+
+.PHONY: $(RUSTOUT)/libsatman.a
+$(RUSTOUT)/libsatman.a:
+	$(cargo) --manifest-path $(SATMAN_DIRECTORY)/Cargo.toml
+
+satman.elf: $(RUSTOUT)/libsatman.a
+	$(link) -T $(SATMAN_DIRECTORY)/satman.ld
+
+%.bin: %.elf
+	$(objcopy) -O binary
+
+%.fbi: %.bin
+	$(mscimg) -f

src/satman/build.rs

@@ -0,0 +1,5 @@
+extern crate build_misoc;
+
+fn main() {
+    build_misoc::cfg();
+}

src/satman/jdac_common.rs

@@ -0,0 +1,74 @@
+pub const INIT: u8              = 0x00;
+pub const PRINT_STATUS: u8      = 0x01;
+pub const PRBS: u8              = 0x02;
+pub const STPL: u8              = 0x03;
+
+pub const SYSREF_DELAY_DAC: u8  = 0x10;
+pub const SYSREF_SLIP: u8       = 0x11;
+pub const SYNC: u8              = 0x12;
+
+pub const DDMTD_SYSREF_RAW: u8  = 0x20;
+pub const DDMTD_SYSREF: u8      = 0x21;
+
+
+fn average_2phases(a: i32, b: i32, modulo: i32) -> i32 {
+    let diff = ((a - b + modulo/2 + modulo) % modulo) - modulo/2;
+    return (modulo + b + diff/2) % modulo;
+}
+
+pub fn average_phases(phases: &[i32], modulo: i32) -> i32 {
+    if phases.len() == 1 {
+        panic!("input array length must be a power of 2");
+    } else if phases.len() == 2 {
+        average_2phases(phases[0], phases[1], modulo)
+    } else {
+        let cut = phases.len()/2;
+        average_2phases(
+            average_phases(&phases[..cut], modulo),
+            average_phases(&phases[cut..], modulo),
+            modulo)
+    }
+}
+
+pub const RAW_DDMTD_N_SHIFT: i32 = 6;
+pub const RAW_DDMTD_N: i32 = 1 << RAW_DDMTD_N_SHIFT;
+pub const DDMTD_DITHER_BITS: i32 = 1;
+pub const DDMTD_N_SHIFT: i32 = RAW_DDMTD_N_SHIFT + DDMTD_DITHER_BITS;
+pub const DDMTD_N: i32 = 1 << DDMTD_N_SHIFT;
+
+#[cfg(has_ad9154)]
+use board_misoc::{clock, csr};
+
+#[cfg(has_ad9154)]
+pub fn init_ddmtd() -> Result<(), &'static str> {
+    unsafe {
+        csr::sysref_ddmtd::reset_write(1);
+        clock::spin_us(1);
+        csr::sysref_ddmtd::reset_write(0);
+        clock::spin_us(100);
+        if csr::sysref_ddmtd::locked_read() != 0 {
+            Ok(())
+        } else {
+            Err("DDMTD helper PLL failed to lock")
+        }
+    }
+}
+
+#[cfg(has_ad9154)]
+pub fn measure_ddmdt_phase_raw() -> i32 {
+    unsafe { csr::sysref_ddmtd::dt_read() as i32 }
+}
+
+#[cfg(has_ad9154)]
+pub fn measure_ddmdt_phase() -> i32 {
+    const AVG_PRECISION_SHIFT: i32 = 6;
+    const AVG_PRECISION: i32 = 1 << AVG_PRECISION_SHIFT;
+    const AVG_MOD: i32 = 1 << (RAW_DDMTD_N_SHIFT + AVG_PRECISION_SHIFT + DDMTD_DITHER_BITS);
+
+    let mut measurements = [0; AVG_PRECISION as usize];
+    for i in 0..AVG_PRECISION {
+        measurements[i as usize] = measure_ddmdt_phase_raw() << (AVG_PRECISION_SHIFT + DDMTD_DITHER_BITS);
+        clock::spin_us(10);
+    }
+    average_phases(&measurements, AVG_MOD) >> AVG_PRECISION_SHIFT
+}

src/satman/jdcg.rs

@@ -0,0 +1,589 @@
+pub mod jesd {
+    use board_misoc::{csr, clock};
+
+    pub fn reset(reset: bool) {
+        unsafe {
+            csr::jesd_crg::jreset_write(if reset {1} else {0});
+        }
+    }
+
+    pub fn enable(dacno: u8, en: bool) {
+        unsafe {
+            (csr::JDCG[dacno as usize].jesd_control_enable_write)(if en {1} else {0})
+        }
+    }
+
+    pub fn phy_done(dacno: u8) -> bool {
+       unsafe {
+           (csr::JDCG[dacno as usize].jesd_control_phy_done_read)() != 0
+       }
+    }
+
+    pub fn ready(dacno: u8) -> bool {
+        unsafe {
+            (csr::JDCG[dacno as usize].jesd_control_ready_read)() != 0
+        }
+    }
+
+    pub fn prbs(dacno: u8, en: bool) {
+        unsafe {
+            (csr::JDCG[dacno as usize].jesd_control_prbs_config_write)(if en {0b01} else {0b00})
+        }
+        clock::spin_us(5000);
+    }
+
+    pub fn stpl(dacno: u8, en: bool) {
+        unsafe {
+            (csr::JDCG[dacno as usize].jesd_control_stpl_enable_write)(if en {1} else {0})
+        }
+        clock::spin_us(5000);
+    }
+
+    pub fn jsync(dacno: u8) -> bool {
+        unsafe {
+            (csr::JDCG[dacno as usize].jesd_control_jsync_read)() != 0
+        }
+    }
+}
+
+pub mod jdac {
+    use board_misoc::{csr, clock};
+    use board_artiq::drtioaux;
+
+    use super::jesd;
+    use super::super::jdac_common;
+
+    pub fn basic_request(dacno: u8, reqno: u8, param: u8) -> Result<u8, &'static str> {
+        if let Err(e) = drtioaux::send(1, &drtioaux::Packet::JdacBasicRequest {
+            destination: 0,
+            dacno: dacno,
+            reqno: reqno, 
+            param: param
+        }) {
+            error!("aux packet error ({})", e);
+            return Err("aux packet error while sending for JESD DAC basic request");
+        }
+        match drtioaux::recv_timeout(1, Some(1000)) {
+            Ok(drtioaux::Packet::JdacBasicReply { succeeded, retval }) => {
+                if succeeded {
+                    Ok(retval)
+                } else {
+                    error!("JESD DAC basic request failed (dacno={}, reqno={})", dacno, reqno);
+                    Err("remote error status to JESD DAC basic request")
+                }
+            },
+            Ok(packet) => {
+                error!("received unexpected aux packet: {:?}", packet);
+                Err("unexpected aux packet in reply to JESD DAC basic request")
+            },
+            Err(e) => {
+                error!("aux packet error ({})", e);
+                Err("aux packet error while waiting for JESD DAC basic reply")
+            }
+        }
+    }
+
+    pub fn init() -> Result<(), &'static str> {
+        for dacno in 0..csr::JDCG.len() {
+            let dacno = dacno as u8;
+            info!("DAC-{} initializing...", dacno);
+
+            jesd::enable(dacno, true);
+            clock::spin_us(10_000);
+            if !jesd::phy_done(dacno) {
+                error!("JESD core PHY not done");
+                return Err("JESD core PHY not done");
+            }
+
+            basic_request(dacno, jdac_common::INIT, 0)?;
+
+            // JESD ready depends on JSYNC being valid, so DAC init needs to happen first
+            if !jesd::ready(dacno) {
+                error!("JESD core reported not ready, sending DAC status print request");
+                basic_request(dacno, jdac_common::PRINT_STATUS, 0)?;
+                return Err("JESD core reported not ready");
+            }
+
+            jesd::prbs(dacno, true);
+            basic_request(dacno, jdac_common::PRBS, 0)?;
+            jesd::prbs(dacno, false);
+
+            basic_request(dacno, jdac_common::INIT, 0)?;
+            clock::spin_us(5000);
+
+            if !jesd::jsync(dacno) {
+                error!("JESD core reported bad SYNC");
+                return Err("JESD core reported bad SYNC");
+            }
+
+            info!("  ...done initializing");
+        }
+        Ok(())
+    }
+
+    pub fn stpl() -> Result<(), &'static str> {
+        for dacno in 0..csr::JDCG.len() {
+            let dacno = dacno as u8;
+
+            info!("Running STPL test on DAC-{}...", dacno);
+
+            jesd::stpl(dacno, true);
+            basic_request(dacno, jdac_common::STPL, 0)?;
+            jesd::stpl(dacno, false);
+
+            info!("  ...done STPL test");
+        }
+        Ok(())
+    }
+}
+
+pub mod jesd204sync {
+    use board_misoc::{csr, clock, config};
+
+    use super::jdac;
+    use super::super::jdac_common;
+
+    const HMC7043_ANALOG_DELAY_RANGE: u8 = 24;
+
+    const FPGA_CLK_DIV: u16 = 16;  // Keep in sync with hmc830_7043.rs
+    const SYSREF_DIV: u16 = 256;   // Keep in sync with hmc830_7043.rs
+
+    fn hmc7043_sysref_delay_dac(dacno: u8, phase_offset: u8) -> Result<(), &'static str> {
+        match jdac::basic_request(dacno, jdac_common::SYSREF_DELAY_DAC, phase_offset) {
+            Ok(_) => Ok(()),
+            Err(e) => Err(e)
+        }
+    }
+
+
+    fn hmc7043_sysref_slip() -> Result<(), &'static str> {
+        match jdac::basic_request(0, jdac_common::SYSREF_SLIP, 0) {
+            Ok(_) => Ok(()),
+            Err(e) => Err(e)
+        }
+    }
+
+    fn ad9154_sync(dacno: u8) -> Result<bool, &'static str> {
+        match jdac::basic_request(dacno, jdac_common::SYNC, 0) {
+            Ok(0) => Ok(false),
+            Ok(_) => Ok(true),
+            Err(e) => Err(e)
+        }
+    }
+
+    fn measure_ddmdt_phase_raw() -> Result<i32, &'static str> {
+        Ok(jdac::basic_request(0, jdac_common::DDMTD_SYSREF_RAW, 0)? as i32)
+    }
+
+    fn measure_ddmdt_phase() -> Result<i32, &'static str> {
+        Ok(jdac::basic_request(0, jdac_common::DDMTD_SYSREF, 0)? as i32)
+    }
+
+    fn test_ddmtd_stability(raw: bool, tolerance: i32) -> Result<(), &'static str> {
+        info!("testing DDMTD stability (raw={}, tolerance={})...", raw, tolerance);
+
+        let modulo = if raw { jdac_common::RAW_DDMTD_N } else { jdac_common::DDMTD_N };
+        let measurement = if raw { measure_ddmdt_phase_raw } else { measure_ddmdt_phase };
+        let ntests = if raw { 150 } else { 15 };
+
+        let mut max_pkpk = 0;
+        for _ in 0..32 {
+            // If we are near the edges, wraparound can throw off the simple min/max computation.
+            // In this case, add an offset to get near the center.
+            let quadrant = measure_ddmdt_phase()?;
+            let center_offset =
+                if quadrant < jdac_common::DDMTD_N/4 || quadrant > 3*jdac_common::DDMTD_N/4 {
+                    modulo/2
+                } else {
+                    0
+                };
+
+            let mut min = modulo;
+            let mut max = 0;
+            for _ in 0..ntests {
+                let m = (measurement()? + center_offset) % modulo;
+                if m < min {
+                    min = m;
+                }
+                if m > max {
+                    max = m;
+                }
+            }
+            let pkpk = max - min;
+            if pkpk > max_pkpk {
+                max_pkpk = pkpk;
+            }
+            if pkpk > tolerance {
+                error!("  ...excessive peak-peak jitter: {} (min={} max={} center_offset={})", pkpk,
+                    min, max, center_offset);
+                return Err("excessive DDMTD peak-peak jitter");
+            }
+            hmc7043_sysref_slip();
+        }
+
+        info!("  ...passed, peak-peak jitter: {}", max_pkpk);
+        Ok(())
+    }
+
+    fn test_slip_ddmtd() -> Result<(), &'static str> {
+        // expected_step = (RTIO clock frequency)*(DDMTD N)/(HMC7043 CLKIN frequency)
+        let expected_step = 8;
+        let tolerance = 1;
+
+        info!("testing HMC7043 SYSREF slip against DDMTD...");
+        let mut old_phase = measure_ddmdt_phase()?;
+        for _ in 0..1024 {
+            hmc7043_sysref_slip();
+            let phase = measure_ddmdt_phase()?;
+            let step = (jdac_common::DDMTD_N + old_phase - phase) % jdac_common::DDMTD_N;
+            if (step - expected_step).abs() > tolerance {
+                error!("  ...got unexpected step: {} ({} -> {})", step, old_phase, phase);
+                return Err("HMC7043 SYSREF slip produced unexpected DDMTD step");
+            }
+            old_phase = phase;
+        }
+        info!("  ...passed");
+        Ok(())
+    }
+
+    fn sysref_sh_error() -> bool {
+        unsafe {
+            csr::sysref_sampler::sh_error_reset_write(1);
+            clock::spin_us(1);
+            csr::sysref_sampler::sh_error_reset_write(0);
+            clock::spin_us(10);
+            csr::sysref_sampler::sh_error_read() != 0
+        }
+    }
+
+    const SYSREF_SH_PRECISION_SHIFT: i32 = 5;
+    const SYSREF_SH_PRECISION: i32 = 1 << SYSREF_SH_PRECISION_SHIFT;
+    const SYSREF_SH_MOD: i32 = 1 << (jdac_common::DDMTD_N_SHIFT + SYSREF_SH_PRECISION_SHIFT);
+
+    #[derive(Default)]
+    struct SysrefShLimits {
+        rising_phases: [i32; SYSREF_SH_PRECISION as usize],
+        falling_phases: [i32; SYSREF_SH_PRECISION as usize],
+    }
+
+    fn measure_sysref_sh_limits() -> Result<SysrefShLimits, &'static str> {
+        let mut ret = SysrefShLimits::default();
+        let mut nslips = 0;
+        let mut rising_n = 0;
+        let mut falling_n = 0;
+
+        let mut previous = sysref_sh_error();
+        while rising_n < SYSREF_SH_PRECISION || falling_n < SYSREF_SH_PRECISION {
+            hmc7043_sysref_slip();
+            nslips += 1;
+            if nslips > 1024 {
+                return Err("too many slips and not enough SYSREF S/H error transitions");
+            }
+
+            let current = sysref_sh_error();
+            let phase = measure_ddmdt_phase()?;
+            if current && !previous && rising_n < SYSREF_SH_PRECISION {
+                ret.rising_phases[rising_n as usize] = phase << SYSREF_SH_PRECISION_SHIFT;
+                rising_n += 1;
+            }
+            if !current && previous && falling_n < SYSREF_SH_PRECISION {
+                ret.falling_phases[falling_n as usize] = phase << SYSREF_SH_PRECISION_SHIFT;
+                falling_n += 1;
+            }
+            previous = current;
+        }
+        Ok(ret)
+    }
+
+    fn max_phase_deviation(average: i32, phases: &[i32]) -> i32 {
+        let mut ret = 0;
+        for phase in phases.iter() {
+            let deviation = (phase - average + jdac_common::DDMTD_N) % jdac_common::DDMTD_N;
+            if deviation > ret {
+                ret = deviation;
+            }
+        }
+        return ret;
+    }
+
+    fn reach_sysref_ddmtd_target(target: i32, tolerance: i32) -> Result<i32, &'static str> {
+        for _ in 0..1024 {
+            let delta = (measure_ddmdt_phase()? - target + jdac_common::DDMTD_N) % jdac_common::DDMTD_N;
+            if delta <= tolerance {
+                return Ok(delta)
+            }
+            hmc7043_sysref_slip();
+        }
+        Err("failed to reach SYSREF DDMTD phase target")
+    }
+
+    fn calibrate_sysref_target(rising_average: i32, falling_average: i32) -> Result<i32, &'static str> {
+        info!("calibrating SYSREF DDMTD target phase...");
+        let coarse_target =
+            if rising_average < falling_average {
+                (rising_average + falling_average)/2
+            } else {
+                ((falling_average - (jdac_common::DDMTD_N - rising_average))/2 + jdac_common::DDMTD_N) % jdac_common::DDMTD_N
+            };
+        info!("  SYSREF calibration coarse target: {}", coarse_target);
+        reach_sysref_ddmtd_target(coarse_target, 8)?;
+        let target = measure_ddmdt_phase()?;
+        info!("  ...done, target={}", target);
+        Ok(target)
+    }
+
+    fn sysref_get_tsc_phase_raw() -> Result<u8, &'static str> {
+        if sysref_sh_error() {
+            return Err("SYSREF failed S/H timing");
+        }
+        let ret = unsafe { csr::sysref_sampler::sysref_phase_read() };
+        Ok(ret)
+    }
+
+    // Note: the code below assumes RTIO/SYSREF frequency ratio is a power of 2
+
+    fn sysref_get_tsc_phase() -> Result<i32, &'static str> {
+        let mask = (SYSREF_DIV/FPGA_CLK_DIV - 1) as u8;
+        Ok((sysref_get_tsc_phase_raw()? & mask) as i32)
+    }
+
+    pub fn test_sysref_frequency() -> Result<(), &'static str> {
+        info!("testing SYSREF frequency against raw TSC phase bit toggles...");
+
+        let mut all_toggles = 0;
+        let initial_phase = sysref_get_tsc_phase_raw()?;
+        for _ in 0..20000 {
+            clock::spin_us(1);
+            all_toggles |= sysref_get_tsc_phase_raw()? ^ initial_phase;
+        }
+
+        let ratio = (SYSREF_DIV/FPGA_CLK_DIV) as u8;
+        let expected_toggles = 0xff ^ (ratio - 1);
+        if all_toggles == expected_toggles {
+            info!("  ...done (0x{:02x})", all_toggles);
+            Ok(())
+        } else {
+            error!("  ...unexpected toggles: got 0x{:02x}, expected 0x{:02x}",
+                all_toggles, expected_toggles);
+            Err("unexpected toggles")
+        }
+    }
+
+    fn sysref_slip_rtio_cycle()  {
+        for _ in 0..FPGA_CLK_DIV {
+            hmc7043_sysref_slip();
+        }
+    }
+
+    pub fn test_slip_tsc() -> Result<(), &'static str> {
+        info!("testing HMC7043 SYSREF slip against TSC phase...");
+        let initial_phase = sysref_get_tsc_phase()?;
+        let modulo = (SYSREF_DIV/FPGA_CLK_DIV) as i32;
+        for i in 0..128 {
+            sysref_slip_rtio_cycle();
+            let expected_phase = (initial_phase + i + 1) % modulo;
+            let phase = sysref_get_tsc_phase()?;
+            if phase != expected_phase {
+                error!("  ...unexpected TSC phase: got {}, expected {} ", phase, expected_phase);
+                return Err("HMC7043 SYSREF slip produced unexpected TSC phase");
+            }
+        }
+        info!("  ...done");
+        Ok(())
+    }
+
+    pub fn sysref_rtio_align() -> Result<(), &'static str> {
+        info!("aligning SYSREF with RTIO TSC...");
+        let mut nslips = 0;
+        loop {
+            sysref_slip_rtio_cycle();
+            if sysref_get_tsc_phase()? == 0 {
+                info!("  ...done");
+                return Ok(())
+            }
+
+            nslips += 1;
+            if nslips > SYSREF_DIV/FPGA_CLK_DIV {
+                return Err("failed to find SYSREF transition aligned with RTIO TSC");
+            }
+        }
+    }
+
+    pub fn sysref_auto_rtio_align() -> Result<(), &'static str> {
+        test_ddmtd_stability(true, 4)?;
+        test_ddmtd_stability(false, 1)?;
+        test_slip_ddmtd()?;
+
+        info!("determining SYSREF S/H limits...");
+        let sysref_sh_limits = measure_sysref_sh_limits()?;
+        let rising_average = jdac_common::average_phases(&sysref_sh_limits.rising_phases, SYSREF_SH_MOD);
+        let falling_average = jdac_common::average_phases(&sysref_sh_limits.falling_phases, SYSREF_SH_MOD);
+        let rising_max_deviation = max_phase_deviation(rising_average, &sysref_sh_limits.rising_phases);
+        let falling_max_deviation = max_phase_deviation(falling_average, &sysref_sh_limits.falling_phases);
+
+        let rising_average = rising_average >> SYSREF_SH_PRECISION_SHIFT;
+        let falling_average = falling_average >> SYSREF_SH_PRECISION_SHIFT;
+        let rising_max_deviation = rising_max_deviation >> SYSREF_SH_PRECISION_SHIFT;
+        let falling_max_deviation = falling_max_deviation >> SYSREF_SH_PRECISION_SHIFT;
+
+        info!("  SYSREF S/H average limits (DDMTD phases): {} {}", rising_average, falling_average);
+        info!("  SYSREF S/H maximum limit deviation: {} {}", rising_max_deviation, falling_max_deviation);
+        if rising_max_deviation > 8 || falling_max_deviation > 8 {
+            return Err("excessive SYSREF S/H limit deviation");
+        }
+        info!("  ...done");
+
+        let entry = config::read_str("sysref_ddmtd_phase_fpga", |r| r.map(|s| s.parse()));
+        let target_phase = match entry {
+            Ok(Ok(phase)) => {
+                info!("using FPGA SYSREF DDMTD phase target from config: {}", phase);
+                phase
+            }
+            _ => {
+                let phase = calibrate_sysref_target(rising_average, falling_average)?;
+                if let Err(e) = config::write_int("sysref_ddmtd_phase_fpga", phase as u32) {
+                    error!("failed to update FPGA SYSREF DDMTD phase target in config: {}", e);
+                }
+                phase
+            }
+        };
+
+        info!("aligning SYSREF with RTIO clock...");
+        let delta = reach_sysref_ddmtd_target(target_phase, 3)?;
+        if sysref_sh_error() {
+            return Err("SYSREF does not meet S/H timing at DDMTD phase target");
+        }
+        info!("  ...done, delta={}", delta);
+
+        test_sysref_frequency()?;
+        test_slip_tsc()?;
+        sysref_rtio_align()?;
+
+        Ok(())
+    }
+
+    fn sysref_cal_dac(dacno: u8) -> Result<u8, &'static str> {
+        info!("calibrating SYSREF delay at DAC-{}...", dacno);
+
+        // Allocate for more than expected as jitter may create spurious entries.
+        let mut limits_buf = [0; 8];
+        let mut n_limits = 0;
+
+        limits_buf[n_limits] = -1;
+        n_limits += 1;
+
+        // avoid spurious rotation at delay=0
+        hmc7043_sysref_delay_dac(dacno, 0);
+        ad9154_sync(dacno)?;
+
+        for scan_delay in 0..HMC7043_ANALOG_DELAY_RANGE {
+            hmc7043_sysref_delay_dac(dacno, scan_delay);
+            if ad9154_sync(dacno)? {
+                limits_buf[n_limits] = scan_delay as i16;
+                n_limits += 1;
+                if n_limits >= limits_buf.len() - 1  {
+                    break;
+                }
+            }
+        }
+
+        limits_buf[n_limits] = HMC7043_ANALOG_DELAY_RANGE as i16;
+        n_limits += 1;
+
+        info!("  using limits: {:?}", &limits_buf[..n_limits]);
+
+        let mut delay = 0;
+        let mut best_margin = 0;
+
+        for i in 0..(n_limits-1) {
+            let margin = limits_buf[i+1] - limits_buf[i];
+            if margin > best_margin {
+                best_margin = margin;
+                delay = ((limits_buf[i+1] + limits_buf[i])/2) as u8;
+            }
+        }
+
+        info!("  ...done, delay={}", delay);
+        Ok(delay)
+    }
+
+    fn sysref_dac_align(dacno: u8, delay: u8) -> Result<(), &'static str> {
+        let tolerance = 5;
+
+        info!("verifying SYSREF margins at DAC-{}...", dacno);
+
+        // avoid spurious rotation at delay=0
+        hmc7043_sysref_delay_dac(dacno, 0);
+        ad9154_sync(dacno)?;
+
+        let mut rotation_seen = false;
+        for scan_delay in 0..HMC7043_ANALOG_DELAY_RANGE {
+            hmc7043_sysref_delay_dac(dacno, scan_delay);
+            if ad9154_sync(dacno)? {
+                rotation_seen = true;
+                let distance = (scan_delay as i16 - delay as i16).abs();
+                if distance < tolerance {
+                    error!("  rotation at delay={} is {} delay steps from target (FAIL)", scan_delay, distance);
+                    return Err("insufficient SYSREF margin at DAC");
+                } else {
+                    info!("  rotation at delay={} is {} delay steps from target (PASS)", scan_delay, distance);
+                }
+            }
+        }
+
+        if !rotation_seen {
+            return Err("no rotation seen when scanning DAC SYSREF delay");
+        }
+
+        info!("  ...done");
+
+        // We tested that the value is correct - now use it
+        info!("synchronizing DAC-{}", dacno);
+        hmc7043_sysref_delay_dac(dacno, delay);
+        ad9154_sync(dacno)?;
+
+        Ok(())
+    }
+
+    pub fn sysref_auto_dac_align() -> Result<(), &'static str> {
+        // We assume that DAC SYSREF traces are length-matched so only one delay
+        // value is needed, and we use DAC-0 as calibration reference.
+
+        let entry = config::read_str("sysref_7043_delay_dac", |r| r.map(|s| s.parse()));
+        let delay = match entry {
+            Ok(Ok(delay)) => {
+                info!("using DAC SYSREF delay from config: {}", delay);
+                delay
+            },
+            _ => {
+                let delay = sysref_cal_dac(0)?;
+                if let Err(e) = config::write_int("sysref_7043_delay_dac", delay as u32) {
+                    error!("failed to update DAC SYSREF delay in config: {}", e);
+                }
+                delay
+            }
+        };
+
+        for dacno in 0..csr::JDCG.len() {
+            sysref_dac_align(dacno as u8, delay)?;
+        }
+        Ok(())
+    }
+
+    pub fn sysref_auto_align() {
+        if let Err(e) = sysref_auto_rtio_align() {
+            error!("failed to align SYSREF at FPGA: {}", e);
+        }
+        if let Err(e) = sysref_auto_dac_align() {
+            error!("failed to align SYSREF at DAC: {}", e);
+        }
+    }
+
+    pub fn resync_dacs() -> Result<(), &'static str> {
+        for dacno in 0..csr::JDCG.len() {
+            info!("resynchronizing DAC-{}", dacno);
+            ad9154_sync(dacno as u8)?;
+        }
+        Ok(())
+    }
+}

src/satman/main.rs

@@ -0,0 +1,691 @@
+#![feature(never_type, panic_implementation, panic_info_message, const_slice_len, try_from)]
+#![no_std]
+
+#[macro_use]
+extern crate log;
+#[macro_use]
+extern crate board_misoc;
+extern crate board_artiq;
+
+use core::convert::TryFrom;
+use board_misoc::{csr, irq, ident, clock, uart_logger, i2c};
+#[cfg(has_si5324)]
+use board_artiq::si5324;
+#[cfg(has_wrpll)]
+use board_artiq::wrpll;
+use board_artiq::{spi, drtioaux};
+use board_artiq::drtio_routing;
+#[cfg(has_hmc830_7043)]
+use board_artiq::hmc830_7043;
+
+mod repeater;
+#[cfg(has_jdcg)]
+mod jdcg;
+#[cfg(any(has_ad9154, has_jdcg))]
+pub mod jdac_common;
+
+fn drtiosat_reset(reset: bool) {
+    unsafe {
+        csr::drtiosat::reset_write(if reset { 1 } else { 0 });
+    }
+}
+
+fn drtiosat_reset_phy(reset: bool) {
+    unsafe {
+        csr::drtiosat::reset_phy_write(if reset { 1 } else { 0 });
+    }
+}
+
+fn drtiosat_link_rx_up() -> bool {
+    unsafe {
+        csr::drtiosat::rx_up_read() == 1
+    }
+}
+
+fn drtiosat_tsc_loaded() -> bool {
+    unsafe {
+        let tsc_loaded = csr::drtiosat::tsc_loaded_read() == 1;
+        if tsc_loaded {
+            csr::drtiosat::tsc_loaded_write(1);
+        }
+        tsc_loaded
+    }
+}
+
+
+#[cfg(has_drtio_routing)]
+macro_rules! forward {
+    ($routing_table:expr, $destination:expr, $rank:expr, $repeaters:expr, $packet:expr) => {{
+        let hop = $routing_table.0[$destination as usize][$rank as usize];
+        if hop != 0 {
+            let repno = (hop - 1) as usize;
+            if repno < $repeaters.len() {
+                return $repeaters[repno].aux_forward($packet);
+            } else {
+                return Err(drtioaux::Error::RoutingError);
+            }
+        }
+    }}
+}
+
+#[cfg(not(has_drtio_routing))]
+macro_rules! forward {
+    ($routing_table:expr, $destination:expr, $rank:expr, $repeaters:expr, $packet:expr) => {}
+}
+
+fn process_aux_packet(_repeaters: &mut [repeater::Repeater],
+        _routing_table: &mut drtio_routing::RoutingTable, _rank: &mut u8,
+        packet: drtioaux::Packet) -> Result<(), drtioaux::Error<!>> {
+    // In the code below, *_chan_sel_write takes an u8 if there are fewer than 256 channels,
+    // and u16 otherwise; hence the `as _` conversion.
+    match packet {
+        drtioaux::Packet::EchoRequest =>
+            drtioaux::send(0, &drtioaux::Packet::EchoReply),
+        drtioaux::Packet::ResetRequest => {
+            info!("resetting RTIO");
+            drtiosat_reset(true);
+            clock::spin_us(100);
+            drtiosat_reset(false);
+            for rep in _repeaters.iter() {
+                if let Err(e) = rep.rtio_reset() {
+                    error!("failed to issue RTIO reset ({})", e);
+                }
+            }
+            drtioaux::send(0, &drtioaux::Packet::ResetAck)
+        },
+
+        drtioaux::Packet::DestinationStatusRequest { destination: _destination } => {
+            #[cfg(has_drtio_routing)]
+            let hop = _routing_table.0[_destination as usize][*_rank as usize];
+            #[cfg(not(has_drtio_routing))]
+            let hop = 0;
+
+            if hop == 0 {
+                let errors;
+                unsafe {
+                    errors = csr::drtiosat::rtio_error_read();
+                }
+                if errors & 1 != 0 {
+                    let channel;
+                    unsafe {
+                        channel = csr::drtiosat::sequence_error_channel_read();
+                        csr::drtiosat::rtio_error_write(1);
+                    }
+                    drtioaux::send(0,
+                        &drtioaux::Packet::DestinationSequenceErrorReply { channel })?;
+                } else if errors & 2 != 0 {
+                    let channel;
+                    unsafe {
+                        channel = csr::drtiosat::collision_channel_read();
+                        csr::drtiosat::rtio_error_write(2);
+                    }
+                    drtioaux::send(0,
+                        &drtioaux::Packet::DestinationCollisionReply { channel })?;
+                } else if errors & 4 != 0 {
+                    let channel;
+                    unsafe {
+                        channel = csr::drtiosat::busy_channel_read();
+                        csr::drtiosat::rtio_error_write(4);
+                    }
+                    drtioaux::send(0,
+                        &drtioaux::Packet::DestinationBusyReply { channel })?;
+                }
+                else {
+                    drtioaux::send(0, &drtioaux::Packet::DestinationOkReply)?;
+                }
+            }
+
+            #[cfg(has_drtio_routing)]
+            {
+                if hop != 0 {
+                    let hop = hop as usize;
+                    if hop <= csr::DRTIOREP.len() {
+                        let repno = hop - 1;
+                        match _repeaters[repno].aux_forward(&drtioaux::Packet::DestinationStatusRequest {
+                            destination: _destination
+                        }) {
+                            Ok(()) => (),
+                            Err(drtioaux::Error::LinkDown) => drtioaux::send(0, &drtioaux::Packet::DestinationDownReply)?,
+                            Err(e) => {
+                                drtioaux::send(0, &drtioaux::Packet::DestinationDownReply)?;
+                                error!("aux error when handling destination status request: {}", e);
+                            },
+                        }
+                    } else {
+                        drtioaux::send(0, &drtioaux::Packet::DestinationDownReply)?;
+                    }
+                }
+            }
+
+            Ok(())
+        }
+
+        #[cfg(has_drtio_routing)]
+        drtioaux::Packet::RoutingSetPath { destination, hops } => {
+            _routing_table.0[destination as usize] = hops;
+            for rep in _repeaters.iter() {
+                if let Err(e) = rep.set_path(destination, &hops) {
+                    error!("failed to set path ({})", e);
+                }
+            }
+            drtioaux::send(0, &drtioaux::Packet::RoutingAck)
+        }
+        #[cfg(has_drtio_routing)]
+        drtioaux::Packet::RoutingSetRank { rank } => {
+            *_rank = rank;
+            drtio_routing::interconnect_enable_all(_routing_table, rank);
+
+            let rep_rank = rank + 1;
+            for rep in _repeaters.iter() {
+                if let Err(e) = rep.set_rank(rep_rank) {
+                    error!("failed to set rank ({})", e);
+                }
+            }
+
+            info!("rank: {}", rank);
+            info!("routing table: {}", _routing_table);
+
+            drtioaux::send(0, &drtioaux::Packet::RoutingAck)
+        }
+
+        #[cfg(not(has_drtio_routing))]
+        drtioaux::Packet::RoutingSetPath { destination: _, hops: _ } => {
+            drtioaux::send(0, &drtioaux::Packet::RoutingAck)
+        }
+        #[cfg(not(has_drtio_routing))]
+        drtioaux::Packet::RoutingSetRank { rank: _ } => {
+            drtioaux::send(0, &drtioaux::Packet::RoutingAck)
+        }
+
+        drtioaux::Packet::MonitorRequest { destination: _destination, channel, probe } => {
+            forward!(_routing_table, _destination, *_rank, _repeaters, &packet);
+            let value;
+            #[cfg(has_rtio_moninj)]
+            unsafe {
+                csr::rtio_moninj::mon_chan_sel_write(channel as _);
+                csr::rtio_moninj::mon_probe_sel_write(probe);
+                csr::rtio_moninj::mon_value_update_write(1);
+                value = csr::rtio_moninj::mon_value_read();
+            }
+            #[cfg(not(has_rtio_moninj))]
+            {
+                value = 0;
+            }
+            let reply = drtioaux::Packet::MonitorReply { value: value as u32 };
+            drtioaux::send(0, &reply)
+        },
+        drtioaux::Packet::InjectionRequest { destination: _destination, channel, overrd, value } => {
+            forward!(_routing_table, _destination, *_rank, _repeaters, &packet);
+            #[cfg(has_rtio_moninj)]
+            unsafe {
+                csr::rtio_moninj::inj_chan_sel_write(channel as _);
+                csr::rtio_moninj::inj_override_sel_write(overrd);
+                csr::rtio_moninj::inj_value_write(value);
+            }
+            Ok(())
+        },
+        drtioaux::Packet::InjectionStatusRequest { destination: _destination, channel, overrd } => {
+            forward!(_routing_table, _destination, *_rank, _repeaters, &packet);
+            let value;
+            #[cfg(has_rtio_moninj)]
+            unsafe {
+                csr::rtio_moninj::inj_chan_sel_write(channel as _);
+                csr::rtio_moninj::inj_override_sel_write(overrd);
+                value = csr::rtio_moninj::inj_value_read();
+            }
+            #[cfg(not(has_rtio_moninj))]
+            {
+                value = 0;
+            }
+            drtioaux::send(0, &drtioaux::Packet::InjectionStatusReply { value: value })
+        },
+
+        drtioaux::Packet::I2cStartRequest { destination: _destination, busno } => {
+            forward!(_routing_table, _destination, *_rank, _repeaters, &packet);
+            let succeeded = i2c::start(busno).is_ok();
+            drtioaux::send(0, &drtioaux::Packet::I2cBasicReply { succeeded: succeeded })
+        }
+        drtioaux::Packet::I2cRestartRequest { destination: _destination, busno } => {
+            forward!(_routing_table, _destination, *_rank, _repeaters, &packet);
+            let succeeded = i2c::restart(busno).is_ok();
+            drtioaux::send(0, &drtioaux::Packet::I2cBasicReply { succeeded: succeeded })
+        }
+        drtioaux::Packet::I2cStopRequest { destination: _destination, busno } => {
+            forward!(_routing_table, _destination, *_rank, _repeaters, &packet);
+            let succeeded = i2c::stop(busno).is_ok();
+            drtioaux::send(0, &drtioaux::Packet::I2cBasicReply { succeeded: succeeded })
+        }
+        drtioaux::Packet::I2cWriteRequest { destination: _destination, busno, data } => {
+            forward!(_routing_table, _destination, *_rank, _repeaters, &packet);
+            match i2c::write(busno, data) {
+                Ok(ack) => drtioaux::send(0,
+                    &drtioaux::Packet::I2cWriteReply { succeeded: true, ack: ack }),
+                Err(_) => drtioaux::send(0,
+                    &drtioaux::Packet::I2cWriteReply { succeeded: false, ack: false })
+            }
+        }
+        drtioaux::Packet::I2cReadRequest { destination: _destination, busno, ack } => {
+            forward!(_routing_table, _destination, *_rank, _repeaters, &packet);
+            match i2c::read(busno, ack) {
+                Ok(data) => drtioaux::send(0,
+                    &drtioaux::Packet::I2cReadReply { succeeded: true, data: data }),
+                Err(_) => drtioaux::send(0,
+                    &drtioaux::Packet::I2cReadReply { succeeded: false, data: 0xff })
+            }
+        }
+
+        drtioaux::Packet::SpiSetConfigRequest { destination: _destination, busno, flags, length, div, cs } => {
+            forward!(_routing_table, _destination, *_rank, _repeaters, &packet);
+            let succeeded = spi::set_config(busno, flags, length, div, cs).is_ok();
+            drtioaux::send(0,
+                &drtioaux::Packet::SpiBasicReply { succeeded: succeeded })
+        },
+        drtioaux::Packet::SpiWriteRequest { destination: _destination, busno, data } => {
+            forward!(_routing_table, _destination, *_rank, _repeaters, &packet);
+            let succeeded = spi::write(busno, data).is_ok();
+            drtioaux::send(0,
+                &drtioaux::Packet::SpiBasicReply { succeeded: succeeded })
+        }
+        drtioaux::Packet::SpiReadRequest { destination: _destination, busno } => {
+            forward!(_routing_table, _destination, *_rank, _repeaters, &packet);
+            match spi::read(busno) {
+                Ok(data) => drtioaux::send(0,
+                    &drtioaux::Packet::SpiReadReply { succeeded: true, data: data }),
+                Err(_) => drtioaux::send(0,
+                    &drtioaux::Packet::SpiReadReply { succeeded: false, data: 0 })
+            }
+        }
+
+        drtioaux::Packet::JdacBasicRequest { destination: _destination, dacno: _dacno,
+                                             reqno: _reqno, param: _param } => {
+            forward!(_routing_table, _destination, *_rank, _repeaters, &packet);
+            #[cfg(has_ad9154)]
+            let (succeeded, retval) = {
+                #[cfg(rtio_frequency = "125.0")]
+                const LINERATE: u64 = 5_000_000_000;
+                #[cfg(rtio_frequency = "150.0")]
+                const LINERATE: u64 = 6_000_000_000;
+                match _reqno {
+                    jdac_common::INIT => (board_artiq::ad9154::setup(_dacno, LINERATE).is_ok(), 0),
+                    jdac_common::PRINT_STATUS => { board_artiq::ad9154::status(_dacno); (true, 0) },
+                    jdac_common::PRBS => (board_artiq::ad9154::prbs(_dacno).is_ok(), 0),
+                    jdac_common::STPL => (board_artiq::ad9154::stpl(_dacno, 4, 2).is_ok(), 0),
+                    jdac_common::SYSREF_DELAY_DAC => { board_artiq::hmc830_7043::hmc7043::sysref_delay_dac(_dacno, _param); (true, 0) },
+                    jdac_common::SYSREF_SLIP => { board_artiq::hmc830_7043::hmc7043::sysref_slip(); (true, 0) },
+                    jdac_common::SYNC => {
+                        match board_artiq::ad9154::sync(_dacno) {
+                            Ok(false) => (true, 0),
+                            Ok(true) => (true, 1),
+                            Err(e) => {
+                                error!("DAC sync failed: {}", e);
+                                (false, 0)
+                            }
+                        }
+                    },
+                    jdac_common::DDMTD_SYSREF_RAW => (true, jdac_common::measure_ddmdt_phase_raw() as u8),
+                    jdac_common::DDMTD_SYSREF => (true, jdac_common::measure_ddmdt_phase() as u8),
+                    _ => (false, 0)
+                }
+            };
+            #[cfg(not(has_ad9154))]
+            let (succeeded, retval) = (false, 0);
+            drtioaux::send(0,
+                &drtioaux::Packet::JdacBasicReply { succeeded: succeeded, retval: retval })
+        }
+
+        _ => {
+            warn!("received unexpected aux packet");
+            Ok(())
+        }
+    }
+}
+
+fn process_aux_packets(repeaters: &mut [repeater::Repeater],
+        routing_table: &mut drtio_routing::RoutingTable, rank: &mut u8) {
+    let result =
+        drtioaux::recv(0).and_then(|packet| {
+            if let Some(packet) = packet {
+                process_aux_packet(repeaters, routing_table, rank, packet)
+            } else {
+                Ok(())
+            }
+        });
+    match result {
+        Ok(()) => (),
+        Err(e) => warn!("aux packet error ({})", e)
+    }
+}
+
+fn drtiosat_process_errors() {
+    let errors;
+    unsafe {
+        errors = csr::drtiosat::protocol_error_read();
+    }
+    if errors & 1 != 0 {
+        error!("received packet of an unknown type");
+    }
+    if errors & 2 != 0 {
+        error!("received truncated packet");
+    }
+    if errors & 4 != 0 {
+        let destination;
+        unsafe {
+            destination = csr::drtiosat::buffer_space_timeout_dest_read();
+        }
+        error!("timeout attempting to get buffer space from CRI, destination=0x{:02x}", destination)
+    }
+    if errors & 8 != 0 {
+        let channel;
+        let timestamp_event;
+        let timestamp_counter;
+        unsafe {
+            channel = csr::drtiosat::underflow_channel_read();
+            timestamp_event = csr::drtiosat::underflow_timestamp_event_read() as i64;
+            timestamp_counter = csr::drtiosat::underflow_timestamp_counter_read() as i64;
+        }
+        error!("write underflow, channel={}, timestamp={}, counter={}, slack={}",
+               channel, timestamp_event, timestamp_counter, timestamp_event-timestamp_counter);
+    }
+    if errors & 16 != 0 {
+        error!("write overflow");
+    }
+    unsafe {
+        csr::drtiosat::protocol_error_write(errors);
+    }
+}
+
+
+#[cfg(has_rtio_crg)]
+fn init_rtio_crg() {
+    unsafe {
+        csr::rtio_crg::pll_reset_write(0);
+    }
+    clock::spin_us(150);
+    let locked = unsafe { csr::rtio_crg::pll_locked_read() != 0 };
+    if !locked {
+        error!("RTIO clock failed");
+    }
+}
+
+#[cfg(not(has_rtio_crg))]
+fn init_rtio_crg() { }
+
+fn hardware_tick(ts: &mut u64) {
+    let now = clock::get_ms();
+    if now > *ts {
+        #[cfg(has_grabber)]
+        board_artiq::grabber::tick();
+        *ts = now + 200;
+    }
+}
+
+#[cfg(all(has_si5324, rtio_frequency = "150.0"))]
+const SI5324_SETTINGS: si5324::FrequencySettings
+    = si5324::FrequencySettings {
+    n1_hs  : 6,
+    nc1_ls : 6,
+    n2_hs  : 10,
+    n2_ls  : 270,
+    n31    : 75,
+    n32    : 75,
+    bwsel  : 4,
+    crystal_ref: true
+};
+
+#[cfg(all(has_si5324, rtio_frequency = "125.0"))]
+const SI5324_SETTINGS: si5324::FrequencySettings
+    = si5324::FrequencySettings {
+    n1_hs  : 5,
+    nc1_ls : 8,
+    n2_hs  : 7,
+    n2_ls  : 360,
+    n31    : 63,
+    n32    : 63,
+    bwsel  : 4,
+    crystal_ref: true
+};
+
+#[no_mangle]
+pub extern fn main() -> i32 {
+    clock::init();
+    uart_logger::ConsoleLogger::register();
+
+    info!("ARTIQ satellite manager starting...");
+    info!("software ident {}", csr::CONFIG_IDENTIFIER_STR);
+    info!("gateware ident {}", ident::read(&mut [0; 64]));
+
+    #[cfg(has_i2c)]
+    i2c::init().expect("I2C initialization failed");
+    #[cfg(all(soc_platform = "kasli", hw_rev = "v2.0"))]
+    let (mut io_expander0, mut io_expander1);
+    #[cfg(all(soc_platform = "kasli", hw_rev = "v2.0"))]
+    {
+        io_expander0 = board_misoc::io_expander::IoExpander::new(0);
+        io_expander1 = board_misoc::io_expander::IoExpander::new(1);
+        io_expander0.init().expect("I2C I/O expander #0 initialization failed");
+        io_expander1.init().expect("I2C I/O expander #1 initialization failed");
+        #[cfg(has_wrpll)]
+        {
+            io_expander0.set_oe(1, 1 << 7).unwrap();
+            io_expander0.set(1, 7, true);
+            io_expander0.service().unwrap();
+            io_expander1.set_oe(0, 1 << 7).unwrap();
+            io_expander1.set_oe(1, 1 << 7).unwrap();
+            io_expander1.set(0, 7, true);
+            io_expander1.set(1, 7, true);
+            io_expander1.service().unwrap();
+        }
+
+        // Actively drive TX_DISABLE to false on SFP0..3
+        io_expander0.set_oe(0, 1 << 1).unwrap();
+        io_expander0.set_oe(1, 1 << 1).unwrap();
+        io_expander1.set_oe(0, 1 << 1).unwrap();
+        io_expander1.set_oe(1, 1 << 1).unwrap();
+        io_expander0.set(0, 1, false);
+        io_expander0.set(1, 1, false);
+        io_expander1.set(0, 1, false);
+        io_expander1.set(1, 1, false);
+        io_expander0.service().unwrap();
+        io_expander1.service().unwrap();
+    }
+
+    #[cfg(has_si5324)]
+    si5324::setup(&SI5324_SETTINGS, si5324::Input::Ckin1).expect("cannot initialize Si5324");
+    #[cfg(has_wrpll)]
+    wrpll::init();
+
+    unsafe {
+        csr::drtio_transceiver::stable_clkin_write(1);
+    }
+    clock::spin_us(1500); // wait for CPLL/QPLL lock
+    #[cfg(not(has_jdcg))]
+    unsafe {
+        csr::drtio_transceiver::txenable_write(0xffffffffu32 as _);
+    }
+    #[cfg(has_wrpll)]
+    wrpll::diagnostics();
+    init_rtio_crg();
+
+    #[cfg(has_hmc830_7043)]
+    /* must be the first SPI init because of HMC830 SPI mode selection */
+    hmc830_7043::init().expect("cannot initialize HMC830/7043");
+    #[cfg(has_ad9154)]
+    {
+        jdac_common::init_ddmtd().expect("failed to initialize SYSREF DDMTD core");
+        for dacno in 0..csr::CONFIG_AD9154_COUNT {
+            board_artiq::ad9154::reset_and_detect(dacno as u8).expect("AD9154 DAC not detected");
+        }
+    }
+
+    #[cfg(has_drtio_routing)]
+    let mut repeaters = [repeater::Repeater::default(); csr::DRTIOREP.len()];
+    #[cfg(not(has_drtio_routing))]
+    let mut repeaters = [repeater::Repeater::default(); 0];
+    for i in 0..repeaters.len() {
+        repeaters[i] = repeater::Repeater::new(i as u8);
+    } 
+    let mut routing_table = drtio_routing::RoutingTable::default_empty();
+    let mut rank = 1;
+
+    let mut hardware_tick_ts = 0;
+
+    loop {
+        #[cfg(has_jdcg)]
+        unsafe {
+            // Hide from uplink until RTM is ready
+            csr::drtio_transceiver::txenable_write(0xfffffffeu32 as _);
+        }
+        while !drtiosat_link_rx_up() {
+            drtiosat_process_errors();
+            for mut rep in repeaters.iter_mut() {
+                rep.service(&routing_table, rank);
+            }
+            #[cfg(all(soc_platform = "kasli", hw_rev = "v2.0"))]
+            {
+                io_expander0.service().expect("I2C I/O expander #0 service failed");
+                io_expander1.service().expect("I2C I/O expander #1 service failed");
+            }
+            hardware_tick(&mut hardware_tick_ts);
+        }
+
+        info!("uplink is up, switching to recovered clock");
+        #[cfg(has_si5324)]
+        {
+            si5324::siphaser::select_recovered_clock(true).expect("failed to switch clocks");
+            si5324::siphaser::calibrate_skew().expect("failed to calibrate skew");
+        }
+        #[cfg(has_wrpll)]
+        wrpll::select_recovered_clock(true);
+
+        drtioaux::reset(0);
+        drtiosat_reset(false);
+        drtiosat_reset_phy(false);
+
+        #[cfg(has_jdcg)]
+        let mut was_up = false;
+        while drtiosat_link_rx_up() {
+            drtiosat_process_errors();
+            process_aux_packets(&mut repeaters, &mut routing_table, &mut rank);
+            for mut rep in repeaters.iter_mut() {
+                rep.service(&routing_table, rank);
+            }
+            #[cfg(all(soc_platform = "kasli", hw_rev = "v2.0"))]
+            {
+                io_expander0.service().expect("I2C I/O expander #0 service failed");
+                io_expander1.service().expect("I2C I/O expander #1 service failed");
+            }
+            hardware_tick(&mut hardware_tick_ts);
+            if drtiosat_tsc_loaded() {
+                info!("TSC loaded from uplink");
+                #[cfg(has_jdcg)]
+                {
+                    // We assume that the RTM on repeater0 is up.
+                    // Uplink should not send a TSC load command unless the link is
+                    // up, and we are hiding when the RTM is down.
+                    if let Err(e) = jdcg::jesd204sync::sysref_rtio_align() {
+                        error!("failed to align SYSREF with TSC ({})", e);
+                    }
+                    if let Err(e) = jdcg::jesd204sync::resync_dacs() {
+                        error!("DAC resync failed after SYSREF/TSC realignment ({})", e);
+                    }
+                }
+                for rep in repeaters.iter() {
+                    if let Err(e) = rep.sync_tsc() {
+                        error!("failed to sync TSC ({})", e);
+                    }
+                }
+                if let Err(e) = drtioaux::send(0, &drtioaux::Packet::TSCAck) {
+                    error!("aux packet error: {}", e);
+                }
+            }
+            #[cfg(has_jdcg)]
+            {
+                let is_up = repeaters[0].is_up();
+                if is_up && !was_up {
+                    /*
+                     * One side of the JESD204 elastic buffer is clocked by the jitter filter
+                     * (Si5324 or WRPLL), the other by the RTM.
+                     * The elastic buffer can operate only when those two clocks are derived from
+                     * the same oscillator.
+                     * This is the case when either of those conditions is true:
+                     * (1) The DRTIO master and the RTM are clocked directly from a common external
+                     *     source, *and* the jitter filter has locked to the recovered clock.
+                     *     This clocking scheme may provide less noise and phase drift at the DACs.
+                     * (2) The RTM clock is connected to the jitter filter output.
+                     * To handle those cases, we simply keep the JESD204 core in reset unless the
+                     * jitter filter is locked to the recovered clock.
+                     */
+                    jdcg::jesd::reset(false);
+                    let _ = jdcg::jdac::init();
+                    jdcg::jesd204sync::sysref_auto_align();
+                    jdcg::jdac::stpl();
+                    unsafe {
+                        csr::drtio_transceiver::txenable_write(0xffffffffu32 as _);  // unhide
+                    }
+                }
+                was_up = is_up;
+            }
+        }
+
+        #[cfg(has_jdcg)]
+        jdcg::jesd::reset(true);
+
+        drtiosat_reset_phy(true);
+        drtiosat_reset(true);
+        drtiosat_tsc_loaded();
+        info!("uplink is down, switching to local oscillator clock");
+        #[cfg(has_si5324)]
+        si5324::siphaser::select_recovered_clock(false).expect("failed to switch clocks");
+        #[cfg(has_wrpll)]
+        wrpll::select_recovered_clock(false);
+    }
+}
+
+#[no_mangle]
+pub extern fn exception(vect: u32, _regs: *const u32, pc: u32, ea: u32) {
+    let vect = irq::Exception::try_from(vect).expect("unknown exception");
+
+    fn hexdump(addr: u32) {
+        let addr = (addr - addr % 4) as *const u32;
+        let mut ptr  = addr;
+        println!("@ {:08p}", ptr);
+        for _ in 0..4 {
+            print!("+{:04x}: ", ptr as usize - addr as usize);
+            print!("{:08x} ",   unsafe { *ptr }); ptr = ptr.wrapping_offset(1);
+            print!("{:08x} ",   unsafe { *ptr }); ptr = ptr.wrapping_offset(1);
+            print!("{:08x} ",   unsafe { *ptr }); ptr = ptr.wrapping_offset(1);
+            print!("{:08x}\n",  unsafe { *ptr }); ptr = ptr.wrapping_offset(1);
+        }
+    }
+
+    hexdump(pc);
+    hexdump(ea);
+    panic!("exception {:?} at PC 0x{:x}, EA 0x{:x}", vect, pc, ea)
+}
+
+#[no_mangle]
+pub extern fn abort() {
+    println!("aborted");
+    loop {}
+}
+
+#[no_mangle] // https://github.com/rust-lang/rust/issues/{38281,51647}
+#[panic_implementation]
+pub fn panic_fmt(info: &core::panic::PanicInfo) -> ! {
+    #[cfg(has_error_led)]
+    unsafe {
+        csr::error_led::out_write(1);
+    }
+
+    if let Some(location) = info.location() {
+        print!("panic at {}:{}:{}", location.file(), location.line(), location.column());
+    } else {
+        print!("panic at unknown location");
+    }
+    if let Some(message) = info.message() {
+        println!(": {}", message);
+    } else {
+        println!("");
+    }
+    loop {}
+}

src/satman/repeater.rs

@@ -0,0 +1,283 @@
+use board_artiq::{drtioaux, drtio_routing};
+#[cfg(has_drtio_routing)]
+use board_misoc::{csr, clock};
+
+#[cfg(has_drtio_routing)]
+fn rep_link_rx_up(repno: u8) -> bool {
+    let repno = repno as usize;
+    unsafe {
+        (csr::DRTIOREP[repno].rx_up_read)() == 1
+    }
+}
+
+#[cfg(has_drtio_routing)]
+#[derive(Clone, Copy, PartialEq)]
+enum RepeaterState {
+    Down,
+    SendPing { ping_count: u16 },
+    WaitPingReply { ping_count: u16, timeout: u64 },
+    Up,
+    Failed
+}
+
+#[cfg(has_drtio_routing)]
+impl Default for RepeaterState {
+    fn default() -> RepeaterState { RepeaterState::Down }
+}
+
+#[cfg(has_drtio_routing)]
+#[derive(Clone, Copy, Default)]
+pub struct Repeater {
+    repno: u8,
+    auxno: u8,
+    state: RepeaterState
+}
+
+#[cfg(has_drtio_routing)]
+impl Repeater {
+    pub fn new(repno: u8) -> Repeater {
+        Repeater {
+            repno: repno,
+            auxno: repno + 1,
+            state: RepeaterState::Down
+        }
+    }
+
+    #[allow(dead_code)]
+    pub fn is_up(&self) -> bool {
+        self.state == RepeaterState::Up
+    }
+
+    pub fn service(&mut self, routing_table: &drtio_routing::RoutingTable, rank: u8) {
+        self.process_local_errors();
+
+        match self.state {
+            RepeaterState::Down => {
+                if rep_link_rx_up(self.repno) {
+                    info!("[REP#{}] link RX became up, pinging", self.repno);
+                    self.state = RepeaterState::SendPing { ping_count: 0 };
+                }
+            }
+            RepeaterState::SendPing { ping_count } => {
+                if rep_link_rx_up(self.repno) {
+                    drtioaux::send(self.auxno, &drtioaux::Packet::EchoRequest).unwrap();
+                    self.state = RepeaterState::WaitPingReply {
+                        ping_count: ping_count + 1,
+                        timeout: clock::get_ms() + 100
+                    }
+                } else {
+                    error!("[REP#{}] link RX went down during ping", self.repno);
+                    self.state = RepeaterState::Down;
+                }
+            }
+            RepeaterState::WaitPingReply { ping_count, timeout } => {
+                if rep_link_rx_up(self.repno) {
+                    if let Ok(Some(drtioaux::Packet::EchoReply)) = drtioaux::recv(self.auxno) {
+                        info!("[REP#{}] remote replied after {} packets", self.repno, ping_count);
+                        self.state = RepeaterState::Up;
+                        if let Err(e) = self.sync_tsc() {
+                            error!("[REP#{}] failed to sync TSC ({})", self.repno, e);
+                            self.state = RepeaterState::Failed;
+                            return;
+                        }
+                        if let Err(e) = self.load_routing_table(routing_table) {
+                            error!("[REP#{}] failed to load routing table ({})", self.repno, e);
+                            self.state = RepeaterState::Failed;
+                            return;
+                        }
+                        if let Err(e) = self.set_rank(rank + 1) {
+                            error!("[REP#{}] failed to set rank ({})", self.repno, e);
+                            self.state = RepeaterState::Failed;
+                            return;
+                        }
+                    } else {
+                        if clock::get_ms() > timeout {
+                            if ping_count > 200 {
+                                error!("[REP#{}] ping failed", self.repno);
+                                self.state = RepeaterState::Failed;
+                            } else {
+                                self.state = RepeaterState::SendPing { ping_count: ping_count };
+                            }
+                        }
+                    }
+                } else {
+                    error!("[REP#{}] link RX went down during ping", self.repno);
+                    self.state = RepeaterState::Down;
+                }
+            }
+            RepeaterState::Up => {
+                self.process_unsolicited_aux();
+                if !rep_link_rx_up(self.repno) {
+                    info!("[REP#{}] link is down", self.repno);
+                    self.state = RepeaterState::Down;
+                }
+            }
+            RepeaterState::Failed => {
+                if !rep_link_rx_up(self.repno) {
+                    info!("[REP#{}] link is down", self.repno);
+                    self.state = RepeaterState::Down;
+                }
+            }
+        }
+    }
+
+    fn process_unsolicited_aux(&self) {
+        match drtioaux::recv(self.auxno) {
+            Ok(Some(packet)) => warn!("[REP#{}] unsolicited aux packet: {:?}", self.repno, packet),
+            Ok(None) => (),
+            Err(_) => warn!("[REP#{}] aux packet error", self.repno)
+        }
+    }
+
+    fn process_local_errors(&self) {
+        let repno = self.repno as usize;
+        let errors;
+        unsafe {
+            errors = (csr::DRTIOREP[repno].protocol_error_read)();
+        }
+        if errors & 1 != 0 {
+            error!("[REP#{}] received packet of an unknown type", repno);
+        }
+        if errors & 2 != 0 {
+            error!("[REP#{}] received truncated packet", repno);
+        }
+        if errors & 4 != 0 {
+            let cmd;
+            let chan_sel;
+            unsafe {
+                cmd = (csr::DRTIOREP[repno].command_missed_cmd_read)();
+                chan_sel = (csr::DRTIOREP[repno].command_missed_chan_sel_read)();
+            }
+            error!("[REP#{}] CRI command missed, cmd={}, chan_sel=0x{:06x}", repno, cmd, chan_sel)
+        }
+        if errors & 8 != 0 {
+            let destination;
+            unsafe {
+                destination = (csr::DRTIOREP[repno].buffer_space_timeout_dest_read)();
+            }
+            error!("[REP#{}] timeout attempting to get remote buffer space, destination=0x{:02x}", repno, destination);
+        }
+        unsafe {
+            (csr::DRTIOREP[repno].protocol_error_write)(errors);
+        }
+    }
+
+    fn recv_aux_timeout(&self, timeout: u32) -> Result<drtioaux::Packet, drtioaux::Error<!>> {
+        let max_time = clock::get_ms() + timeout as u64;
+        loop {
+            if !rep_link_rx_up(self.repno) {
+                return Err(drtioaux::Error::LinkDown);
+            }
+            if clock::get_ms() > max_time {
+                return Err(drtioaux::Error::TimedOut);
+            }
+            match drtioaux::recv(self.auxno) {
+                Ok(Some(packet)) => return Ok(packet),
+                Ok(None) => (),
+                Err(e) => return Err(e)
+            }
+        }
+    }
+
+    pub fn aux_forward(&self, request: &drtioaux::Packet) -> Result<(), drtioaux::Error<!>> {
+        if self.state != RepeaterState::Up {
+            return Err(drtioaux::Error::LinkDown);
+        }
+        drtioaux::send(self.auxno, request).unwrap();
+        let reply = self.recv_aux_timeout(200)?;
+        drtioaux::send(0, &reply).unwrap();
+        Ok(())
+    }
+
+    pub fn sync_tsc(&self) -> Result<(), drtioaux::Error<!>> {
+        if self.state != RepeaterState::Up {
+            return Ok(());
+        }
+
+        let repno = self.repno as usize;
+        unsafe {
+            (csr::DRTIOREP[repno].set_time_write)(1);
+            while (csr::DRTIOREP[repno].set_time_read)() == 1 {}
+        }
+
+        // TSCAck is the only aux packet that is sent spontaneously
+        // by the satellite, in response to a TSC set on the RT link.
+        let reply = self.recv_aux_timeout(10000)?;
+        if reply == drtioaux::Packet::TSCAck {
+            return Ok(());
+        } else {
+            return Err(drtioaux::Error::UnexpectedReply);
+        }
+    }
+
+    pub fn set_path(&self, destination: u8, hops: &[u8; drtio_routing::MAX_HOPS]) -> Result<(), drtioaux::Error<!>> {
+        if self.state != RepeaterState::Up {
+            return Ok(());
+        }
+
+        drtioaux::send(self.auxno, &drtioaux::Packet::RoutingSetPath {
+            destination: destination,
+            hops: *hops
+        }).unwrap();
+        let reply = self.recv_aux_timeout(200)?;
+        if reply != drtioaux::Packet::RoutingAck {
+            return Err(drtioaux::Error::UnexpectedReply);
+        }
+        Ok(())
+    }
+
+    pub fn load_routing_table(&self, routing_table: &drtio_routing::RoutingTable) -> Result<(), drtioaux::Error<!>> {
+        for i in 0..drtio_routing::DEST_COUNT {
+            self.set_path(i as u8, &routing_table.0[i])?;
+        }
+        Ok(())
+    }
+
+    pub fn set_rank(&self, rank: u8) -> Result<(), drtioaux::Error<!>> {
+        if self.state != RepeaterState::Up {
+            return Ok(());
+        }
+        drtioaux::send(self.auxno, &drtioaux::Packet::RoutingSetRank {
+            rank: rank
+        }).unwrap();
+        let reply = self.recv_aux_timeout(200)?;
+        if reply != drtioaux::Packet::RoutingAck {
+            return Err(drtioaux::Error::UnexpectedReply);
+        }
+        Ok(())
+    }
+
+    pub fn rtio_reset(&self) -> Result<(), drtioaux::Error<!>> {
+        let repno = self.repno as usize;
+        unsafe { (csr::DRTIOREP[repno].reset_write)(1); }
+        clock::spin_us(100);
+        unsafe { (csr::DRTIOREP[repno].reset_write)(0); }
+
+        if self.state != RepeaterState::Up {
+            return Ok(());
+        }
+
+        drtioaux::send(self.auxno, &drtioaux::Packet::ResetRequest).unwrap();
+        let reply = self.recv_aux_timeout(200)?;
+        if reply != drtioaux::Packet::ResetAck {
+            return Err(drtioaux::Error::UnexpectedReply);
+        }
+        Ok(())
+    }
+}
+
+#[cfg(not(has_drtio_routing))]
+#[derive(Clone, Copy, Default)]
+pub struct Repeater {
+}
+
+#[cfg(not(has_drtio_routing))]
+impl Repeater {
+    pub fn new(_repno: u8) -> Repeater { Repeater::default() }
+
+    pub fn service(&self, _routing_table: &drtio_routing::RoutingTable, _rank: u8) { }
+
+    pub fn sync_tsc(&self) -> Result<(), drtioaux::Error<!>> { Ok(()) }
+
+    pub fn rtio_reset(&self) -> Result<(), drtioaux::Error<!>> { Ok(()) }
+}

src/satman/satman.ld

@@ -0,0 +1,55 @@
+INCLUDE generated/output_format.ld
+INCLUDE generated/regions.ld
+ENTRY(_reset_handler)
+
+SECTIONS
+{
+    .vectors :
+    {
+        *(.vectors)
+    } > main_ram
+
+    .text :
+    {
+        *(.text .text.*)
+    } > main_ram
+
+    /* https://sourceware.org/bugzilla/show_bug.cgi?id=20475 */
+    .got :
+    {
+        PROVIDE(_GLOBAL_OFFSET_TABLE_ = .);
+        *(.got)
+    } > main_ram
+
+    .got.plt :
+    {
+        *(.got.plt)
+    } > main_ram
+
+    .rodata :
+    {
+        _frodata = .;
+        *(.rodata .rodata.*)
+        _erodata = .;
+    } > main_ram
+
+    .data :
+    {
+        *(.data .data.*)
+    } > main_ram
+
+    .bss ALIGN(4) :
+    {
+        _fbss = .;
+        *(.bss .bss.*)
+        . = ALIGN(4);
+        _ebss = .;
+    } > main_ram
+
+    .stack :
+    {
+        _estack = .;
+        . += 0x10000;
+        _fstack = . - 4;
+    } > main_ram
+}