drtio: wait for remote to ack TSC synchronization

Sayma takes a long time after TSC sync to align SYSREF, and this caused two issues:
1. Aux packets getting lost and causing error reports
2. DRTIO links reported up and kernels proceeding despite the DACs not being properly synced.

artiq/firmware/libproto_artiq/drtioaux_proto.rs

@@ -20,6 +20,7 @@ pub enum Packet {
     EchoReply,
     ResetRequest { phy: bool },
     ResetAck,
+    TSCAck,
 
     RtioErrorRequest,
     RtioNoErrorReply,
@@ -60,6 +61,7 @@ impl Packet {
                 phy: reader.read_bool()?
             },
             0x03 => Packet::ResetAck,
+            0x04 => Packet::TSCAck,
 
             0x20 => Packet::RtioErrorRequest,
             0x21 => Packet::RtioNoErrorReply,
@@ -163,6 +165,8 @@ impl Packet {
             },
             Packet::ResetAck =>
                 writer.write_u8(0x03)?,
+            Packet::TSCAck =>
+                writer.write_u8(0x04)?,
 
             Packet::RtioErrorRequest =>
                 writer.write_u8(0x20)?,

artiq/firmware/runtime/rtio_mgt.rs

@@ -114,6 +114,27 @@ pub mod drtio {
         }
     }
 
+    fn wait_tsc_ack(linkno: u8, io: &Io) -> bool {
+        loop {
+            let mut count = 0;
+            if !link_rx_up(linkno) {
+                return false;
+            }
+            count += 1;
+            if count > 200 {
+                return false;
+            }
+            io.sleep(100).unwrap();
+            // TSCAck is the only aux packet that is sent spontaneously
+            // by the satellite, in response to a TSC set on the RT link.
+            let pr = drtioaux::recv_link(linkno);
+            match pr {
+                Ok(Some(drtioaux::Packet::TSCAck)) => return true,
+                _ => {}
+            }
+        }
+    }
+
     fn process_local_errors(linkno: u8) {
         let errors;
         let linkidx = linkno as usize;
@@ -173,7 +194,11 @@ pub mod drtio {
                             set_link_up(linkno, true);
                             init_buffer_space(linkno);
                             sync_tsc(linkno);
+                            if !wait_tsc_ack(linkno, &io) {
+                                info!("[LINK#{}] remote failed to ack TSC", linkno);
+                            } else {
                                 info!("[LINK#{}] link initialization completed", linkno);
+                            }
                         } else {
                             info!("[LINK#{}] ping failed", linkno);
                         }

artiq/firmware/satman/main.rs

@@ -294,9 +294,9 @@ pub extern fn main() -> i32 {
         while drtio_link_rx_up() {
             process_errors();
             process_aux_packets();
+            if drtio_tsc_loaded() {
                 #[cfg(has_ad9154)]
                 {
-                if drtio_tsc_loaded() {
                     if let Err(e) = board_artiq::jesd204sync::sysref_auto_rtio_align() {
                         error!("failed to align SYSREF at FPGA: {}", e);
                     }
@@ -304,6 +304,9 @@ pub extern fn main() -> i32 {
                         error!("failed to align SYSREF at DAC: {}", e);
                     }
                 }
+                if let Err(e) = drtioaux::send_link(0, &drtioaux::Packet::TSCAck) {
+                    error!("aux packet error: {}", e);
+                }
             }
         }
         drtio_reset_phy(true);