From e4d48a78ebac60b14ec6a6f3a538e533d6ccbccb Mon Sep 17 00:00:00 2001
From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Thu, 26 Jul 2018 20:28:17 +0800
Subject: [PATCH] drtio: wait for remote to ack TSC synchronization

Sayma takes a long time after TSC sync to align SYSREF, and this caused two issues:
1. Aux packets getting lost and causing error reports
2. DRTIO links reported up and kernels proceeding despite the DACs not being properly synced.
---
 .../firmware/libproto_artiq/drtioaux_proto.rs |  4 +++
 artiq/firmware/runtime/rtio_mgt.rs            | 27 ++++++++++++++++++-
 artiq/firmware/satman/main.rs                 |  9 ++++---
 3 files changed, 36 insertions(+), 4 deletions(-)

diff --git a/artiq/firmware/libproto_artiq/drtioaux_proto.rs b/artiq/firmware/libproto_artiq/drtioaux_proto.rs
index ae103f44b..2d0a24e87 100644
--- a/artiq/firmware/libproto_artiq/drtioaux_proto.rs
+++ b/artiq/firmware/libproto_artiq/drtioaux_proto.rs
@@ -20,6 +20,7 @@ pub enum Packet {
     EchoReply,
     ResetRequest { phy: bool },
     ResetAck,
+    TSCAck,
 
     RtioErrorRequest,
     RtioNoErrorReply,
@@ -60,6 +61,7 @@ impl Packet {
                 phy: reader.read_bool()?
             },
             0x03 => Packet::ResetAck,
+            0x04 => Packet::TSCAck,
 
             0x20 => Packet::RtioErrorRequest,
             0x21 => Packet::RtioNoErrorReply,
@@ -163,6 +165,8 @@ impl Packet {
             },
             Packet::ResetAck =>
                 writer.write_u8(0x03)?,
+            Packet::TSCAck =>
+                writer.write_u8(0x04)?,
 
             Packet::RtioErrorRequest =>
                 writer.write_u8(0x20)?,
diff --git a/artiq/firmware/runtime/rtio_mgt.rs b/artiq/firmware/runtime/rtio_mgt.rs
index effa1db5e..f2a606d86 100644
--- a/artiq/firmware/runtime/rtio_mgt.rs
+++ b/artiq/firmware/runtime/rtio_mgt.rs
@@ -114,6 +114,27 @@ pub mod drtio {
         }
     }
 
+    fn wait_tsc_ack(linkno: u8, io: &Io) -> bool {
+        loop {
+            let mut count = 0;
+            if !link_rx_up(linkno) {
+                return false;
+            }
+            count += 1;
+            if count > 200 {
+                return false;
+            }
+            io.sleep(100).unwrap();
+            // TSCAck is the only aux packet that is sent spontaneously
+            // by the satellite, in response to a TSC set on the RT link.
+            let pr = drtioaux::recv_link(linkno);
+            match pr {
+                Ok(Some(drtioaux::Packet::TSCAck)) => return true,
+                _ => {}
+            }
+        }
+    }
+
     fn process_local_errors(linkno: u8) {
         let errors;
         let linkidx = linkno as usize;
@@ -173,7 +194,11 @@ pub mod drtio {
                             set_link_up(linkno, true);
                             init_buffer_space(linkno);
                             sync_tsc(linkno);
-                            info!("[LINK#{}] link initialization completed", linkno);
+                            if !wait_tsc_ack(linkno, &io) {
+                                info!("[LINK#{}] remote failed to ack TSC", linkno);
+                            } else {
+                                info!("[LINK#{}] link initialization completed", linkno);
+                            }
                         } else {
                             info!("[LINK#{}] ping failed", linkno);
                         }
diff --git a/artiq/firmware/satman/main.rs b/artiq/firmware/satman/main.rs
index d91637dc9..fb0546cac 100644
--- a/artiq/firmware/satman/main.rs
+++ b/artiq/firmware/satman/main.rs
@@ -294,9 +294,9 @@ pub extern fn main() -> i32 {
         while drtio_link_rx_up() {
             process_errors();
             process_aux_packets();
-            #[cfg(has_ad9154)]
-            {
-                if drtio_tsc_loaded() {
+            if drtio_tsc_loaded() {
+                #[cfg(has_ad9154)]
+                {
                     if let Err(e) = board_artiq::jesd204sync::sysref_auto_rtio_align() {
                         error!("failed to align SYSREF at FPGA: {}", e);
                     }
@@ -304,6 +304,9 @@ pub extern fn main() -> i32 {
                         error!("failed to align SYSREF at DAC: {}", e);
                     }
                 }
+                if let Err(e) = drtioaux::send_link(0, &drtioaux::Packet::TSCAck) {
+                    error!("aux packet error: {}", e);
+                }
             }
         }
         drtio_reset_phy(true);