diff --git a/artiq/firmware/libboard_artiq/drtioaux.rs b/artiq/firmware/libboard_artiq/drtioaux.rs index 9c189e148..d176e67da 100644 --- a/artiq/firmware/libboard_artiq/drtioaux.rs +++ b/artiq/firmware/libboard_artiq/drtioaux.rs @@ -10,14 +10,21 @@ pub use proto_artiq::drtioaux_proto::Packet; // this is parametric over T because there's no impl Fail for !. #[derive(Fail, Debug)] pub enum Error { - #[fail(display = "packet CRC failed")] - CorruptedPacket, - #[fail(display = "timed out waiting for data")] - TimedOut, #[fail(display = "invalid node number")] NoRoute, + #[fail(display = "gateware reported error")] GatewareError, + #[fail(display = "packet CRC failed")] + CorruptedPacket, + + #[fail(display = "link is down")] + LinkDown, + #[fail(display = "timed out waiting for data")] + TimedOut, + #[fail(display = "unexpected reply")] + UnexpectedReply, + #[fail(display = "protocol error: {}", _0)] Protocol(#[cause] ProtocolError) } diff --git a/artiq/firmware/libproto_artiq/drtioaux_proto.rs b/artiq/firmware/libproto_artiq/drtioaux_proto.rs index e20194442..044f9b80e 100644 --- a/artiq/firmware/libproto_artiq/drtioaux_proto.rs +++ b/artiq/firmware/libproto_artiq/drtioaux_proto.rs @@ -22,11 +22,12 @@ pub enum Packet { ResetAck, TSCAck, - RtioErrorRequest, - RtioNoErrorReply, - RtioErrorSequenceErrorReply { channel: u16 }, - RtioErrorCollisionReply { channel: u16 }, - RtioErrorBusyReply { channel: u16 }, + DestinationStatusRequest { destination: u8 }, + DestinationDownReply, + DestinationOkReply, + DestinationSequenceErrorReply { channel: u16 }, + DestinationCollisionReply { channel: u16 }, + DestinationBusyReply { channel: u16 }, RoutingSetPath { destination: u8, hops: [u8; 32] }, RoutingSetRank { rank: u8 }, @@ -67,15 +68,18 @@ impl Packet { 0x03 => Packet::ResetAck, 0x04 => Packet::TSCAck, - 0x20 => Packet::RtioErrorRequest, - 0x21 => Packet::RtioNoErrorReply, - 0x22 => Packet::RtioErrorSequenceErrorReply { + 0x20 => Packet::DestinationStatusRequest { + destination: reader.read_u8()? + }, + 0x21 => Packet::DestinationDownReply, + 0x22 => Packet::DestinationOkReply, + 0x23 => Packet::DestinationSequenceErrorReply { channel: reader.read_u16()? }, - 0x23 => Packet::RtioErrorCollisionReply { + 0x24 => Packet::DestinationCollisionReply { channel: reader.read_u16()? }, - 0x24 => Packet::RtioErrorBusyReply { + 0x25 => Packet::DestinationBusyReply { channel: reader.read_u16()? }, @@ -186,22 +190,26 @@ impl Packet { Packet::TSCAck => writer.write_u8(0x04)?, - Packet::RtioErrorRequest => - writer.write_u8(0x20)?, - Packet::RtioNoErrorReply => - writer.write_u8(0x21)?, - Packet::RtioErrorSequenceErrorReply { channel } => { - writer.write_u8(0x22)?; - writer.write_u16(channel)?; + Packet::DestinationStatusRequest {destination } => { + writer.write_u8(0x20)?; + writer.write_u8(destination)?; }, - Packet::RtioErrorCollisionReply { channel } => { + Packet::DestinationDownReply => + writer.write_u8(0x21)?, + Packet::DestinationOkReply => + writer.write_u8(0x22)?, + Packet::DestinationSequenceErrorReply { channel } => { writer.write_u8(0x23)?; writer.write_u16(channel)?; }, - Packet::RtioErrorBusyReply { channel } => { + Packet::DestinationCollisionReply { channel } => { writer.write_u8(0x24)?; writer.write_u16(channel)?; }, + Packet::DestinationBusyReply { channel } => { + writer.write_u8(0x25)?; + writer.write_u16(channel)?; + }, Packet::RoutingSetPath { destination, hops } => { writer.write_u8(0x30)?; diff --git a/artiq/firmware/runtime/rtio_mgt.rs b/artiq/firmware/runtime/rtio_mgt.rs index ea34792c4..c6ba76d43 100644 --- a/artiq/firmware/runtime/rtio_mgt.rs +++ b/artiq/firmware/runtime/rtio_mgt.rs @@ -201,22 +201,63 @@ pub mod drtio { } } - fn process_aux_errors(io: &Io, linkno: u8) { - drtioaux::send_link(linkno, &drtioaux::Packet::RtioErrorRequest).unwrap(); - match recv_aux_timeout(io, linkno, 200) { - Ok(drtioaux::Packet::RtioNoErrorReply) => (), - Ok(drtioaux::Packet::RtioErrorSequenceErrorReply { channel }) => - error!("[LINK#{}] RTIO sequence error involving channel {}", linkno, channel), - Ok(drtioaux::Packet::RtioErrorCollisionReply { channel }) => - error!("[LINK#{}] RTIO collision involving channel {}", linkno, channel), - Ok(drtioaux::Packet::RtioErrorBusyReply { channel }) => - error!("[LINK#{}] RTIO busy error involving channel {}", linkno, channel), - Ok(_) => error!("[LINK#{}] received unexpected aux packet", linkno), - Err(e) => error!("[LINK#{}] communication failed ({})", linkno, e) + fn destination_survey(io: &Io, routing_table: &drtio_routing::RoutingTable, + up_destinations: &mut [bool; drtio_routing::DEST_COUNT]) { + for destination in 0..drtio_routing::DEST_COUNT { + let hop = routing_table.0[destination][0]; + + if hop == 0 { + /* local RTIO */ + up_destinations[destination] = true; + } else if hop as usize <= csr::DRTIO.len() { + let linkno = hop - 1; + if up_destinations[destination] { + if link_up(linkno) { + drtioaux::send_link(linkno, &drtioaux::Packet::DestinationStatusRequest { + destination: destination as u8 + }).unwrap(); + match recv_aux_timeout(io, linkno, 200) { + Ok(drtioaux::Packet::DestinationDownReply) => { + info!("[DEST#{}] destination is down", destination); + up_destinations[destination] = false; + }, + Ok(drtioaux::Packet::DestinationOkReply) => (), + Ok(drtioaux::Packet::DestinationSequenceErrorReply { channel }) => + error!("[DEST#{}] RTIO sequence error involving channel 0x{:04x}", destination, channel), + Ok(drtioaux::Packet::DestinationCollisionReply { channel }) => + error!("[DEST#{}] RTIO collision involving channel 0x{:04x}", destination, channel), + Ok(drtioaux::Packet::DestinationBusyReply { channel }) => + error!("[DEST#{}] RTIO busy error involving channel 0x{:04x}", destination, channel), + Ok(packet) => error!("[DEST#{}] received unexpected aux packet: {:?}", destination, packet), + Err(e) => error!("[DEST#{}] communication failed ({})", destination, e) + } + } else { + info!("[DEST#{}] destination is down", destination); + up_destinations[destination] = false; + } + } else { + if link_up(linkno) { + drtioaux::send_link(linkno, &drtioaux::Packet::DestinationStatusRequest { + destination: destination as u8 + }).unwrap(); + match recv_aux_timeout(io, linkno, 200) { + Ok(drtioaux::Packet::DestinationDownReply) => (), + Ok(drtioaux::Packet::DestinationOkReply) => { + info!("[DEST#{}] destination is up", destination); + up_destinations[destination] = true; + /* TODO: get buffer space */ + }, + Ok(packet) => error!("[DEST#{}] received unexpected aux packet: {:?}", destination, packet), + Err(e) => error!("[DEST#{}] communication failed ({})", destination, e) + } + } + } + } } } pub fn link_thread(io: Io, routing_table: &drtio_routing::RoutingTable) { + let mut up_destinations = [false; drtio_routing::DEST_COUNT]; loop { for linkno in 0..csr::DRTIO.len() { let linkno = linkno as u8; @@ -225,7 +266,6 @@ pub mod drtio { if link_rx_up(linkno) { process_unsolicited_aux(linkno); process_local_errors(linkno); - process_aux_errors(&io, linkno); } else { info!("[LINK#{}] link is down", linkno); set_link_up(linkno, false); @@ -255,6 +295,7 @@ pub mod drtio { } } } + destination_survey(&io, routing_table, &mut up_destinations); io.sleep(200).unwrap(); } } diff --git a/artiq/firmware/satman/main.rs b/artiq/firmware/satman/main.rs index e8c0b60f4..8e3b22e7d 100644 --- a/artiq/firmware/satman/main.rs +++ b/artiq/firmware/satman/main.rs @@ -71,39 +71,70 @@ fn process_aux_packet(_repeaters: &mut [repeater::Repeater], drtioaux::send_link(0, &drtioaux::Packet::ResetAck) }, - drtioaux::Packet::RtioErrorRequest => { - let errors; - unsafe { - errors = csr::drtiosat::rtio_error_read(); - } - if errors & 1 != 0 { - let channel; + drtioaux::Packet::DestinationStatusRequest { destination } => { + #[cfg(has_drtio_routing)] + let hop = _routing_table.0[destination as usize][*_rank as usize]; + #[cfg(not(has_drtio_routing))] + let hop = 0; + + if hop == 0 { + let errors; unsafe { - channel = csr::drtiosat::sequence_error_channel_read(); - csr::drtiosat::rtio_error_write(1); + errors = csr::drtiosat::rtio_error_read(); } - drtioaux::send_link(0, - &drtioaux::Packet::RtioErrorSequenceErrorReply { channel }) - } else if errors & 2 != 0 { - let channel; - unsafe { - channel = csr::drtiosat::collision_channel_read(); - csr::drtiosat::rtio_error_write(2); + if errors & 1 != 0 { + let channel; + unsafe { + channel = csr::drtiosat::sequence_error_channel_read(); + csr::drtiosat::rtio_error_write(1); + } + drtioaux::send_link(0, + &drtioaux::Packet::DestinationSequenceErrorReply { channel })?; + } else if errors & 2 != 0 { + let channel; + unsafe { + channel = csr::drtiosat::collision_channel_read(); + csr::drtiosat::rtio_error_write(2); + } + drtioaux::send_link(0, + &drtioaux::Packet::DestinationCollisionReply { channel })?; + } else if errors & 4 != 0 { + let channel; + unsafe { + channel = csr::drtiosat::busy_channel_read(); + csr::drtiosat::rtio_error_write(4); + } + drtioaux::send_link(0, + &drtioaux::Packet::DestinationBusyReply { channel })?; } - drtioaux::send_link(0, - &drtioaux::Packet::RtioErrorCollisionReply { channel }) - } else if errors & 4 != 0 { - let channel; - unsafe { - channel = csr::drtiosat::busy_channel_read(); - csr::drtiosat::rtio_error_write(4); + else { + drtioaux::send_link(0, &drtioaux::Packet::DestinationOkReply)?; } - drtioaux::send_link(0, - &drtioaux::Packet::RtioErrorBusyReply { channel }) } - else { - drtioaux::send_link(0, &drtioaux::Packet::RtioNoErrorReply) + + #[cfg(has_drtio_routing)] + { + if hop != 0 { + let hop = hop as usize; + if hop <= csr::DRTIOREP.len() { + let repno = hop - 1; + match _repeaters[repno].aux_forward(&drtioaux::Packet::DestinationStatusRequest { + destination: destination + }) { + Ok(()) => (), + Err(drtioaux::Error::LinkDown) => drtioaux::send_link(0, &drtioaux::Packet::DestinationDownReply)?, + Err(e) => { + drtioaux::send_link(0, &drtioaux::Packet::DestinationDownReply)?; + error!("aux error when handling destination status request: {}", e); + }, + } + } else { + drtioaux::send_link(0, &drtioaux::Packet::DestinationDownReply)?; + } + } } + + Ok(()) } #[cfg(has_drtio_routing)] @@ -135,11 +166,11 @@ fn process_aux_packet(_repeaters: &mut [repeater::Repeater], } #[cfg(not(has_drtio_routing))] - drtioaux::Packet::RoutingSetPath { _destination, _hops } => { + drtioaux::Packet::RoutingSetPath { destination, hops } => { drtioaux::send_link(0, &drtioaux::Packet::RoutingAck) } #[cfg(not(has_drtio_routing))] - drtioaux::Packet::RoutingSetRank { _rank } => { + drtioaux::Packet::RoutingSetRank { rank } => { drtioaux::send_link(0, &drtioaux::Packet::RoutingAck) } diff --git a/artiq/firmware/satman/repeater.rs b/artiq/firmware/satman/repeater.rs index bd2fe8999..c97ece3e5 100644 --- a/artiq/firmware/satman/repeater.rs +++ b/artiq/firmware/satman/repeater.rs @@ -153,24 +153,34 @@ impl Repeater { } } - fn recv_aux_timeout(&self, timeout: u32) -> Result { + fn recv_aux_timeout(&self, timeout: u32) -> Result> { let max_time = clock::get_ms() + timeout as u64; loop { if !rep_link_rx_up(self.repno) { - return Err("link went down"); + return Err(drtioaux::Error::LinkDown); } if clock::get_ms() > max_time { - return Err("timeout"); + return Err(drtioaux::Error::TimedOut); } match drtioaux::recv_link(self.auxno) { Ok(Some(packet)) => return Ok(packet), Ok(None) => (), - Err(_) => return Err("aux packet error") + Err(e) => return Err(e) } } } - pub fn sync_tsc(&self) -> Result<(), &'static str> { + pub fn aux_forward(&self, request: &drtioaux::Packet) -> Result<(), drtioaux::Error> { + if self.state != RepeaterState::Up { + return Err(drtioaux::Error::LinkDown); + } + drtioaux::send_link(self.auxno, request).unwrap(); + let reply = self.recv_aux_timeout(200)?; + drtioaux::send_link(0, &reply).unwrap(); + Ok(()) + } + + pub fn sync_tsc(&self) -> Result<(), drtioaux::Error> { if self.state != RepeaterState::Up { return Ok(()); } @@ -187,11 +197,11 @@ impl Repeater { if reply == drtioaux::Packet::TSCAck { return Ok(()); } else { - return Err("unexpected reply"); + return Err(drtioaux::Error::UnexpectedReply); } } - pub fn set_path(&self, destination: u8, hops: &[u8; drtio_routing::MAX_HOPS]) -> Result<(), &'static str> { + pub fn set_path(&self, destination: u8, hops: &[u8; drtio_routing::MAX_HOPS]) -> Result<(), drtioaux::Error> { if self.state != RepeaterState::Up { return Ok(()); } @@ -202,19 +212,19 @@ impl Repeater { }).unwrap(); let reply = self.recv_aux_timeout(200)?; if reply != drtioaux::Packet::RoutingAck { - return Err("unexpected reply"); + return Err(drtioaux::Error::UnexpectedReply); } Ok(()) } - pub fn load_routing_table(&self, routing_table: &drtio_routing::RoutingTable) -> Result<(), &'static str> { + pub fn load_routing_table(&self, routing_table: &drtio_routing::RoutingTable) -> Result<(), drtioaux::Error> { for i in 0..drtio_routing::DEST_COUNT { self.set_path(i as u8, &routing_table.0[i])?; } Ok(()) } - pub fn set_rank(&self, rank: u8) -> Result<(), &'static str> { + pub fn set_rank(&self, rank: u8) -> Result<(), drtioaux::Error> { if self.state != RepeaterState::Up { return Ok(()); } @@ -223,12 +233,12 @@ impl Repeater { }).unwrap(); let reply = self.recv_aux_timeout(200)?; if reply != drtioaux::Packet::RoutingAck { - return Err("unexpected reply"); + return Err(drtioaux::Error::UnexpectedReply); } Ok(()) } - pub fn rtio_reset(&self, phy: bool) -> Result<(), &'static str> { + pub fn rtio_reset(&self, phy: bool) -> Result<(), drtioaux::Error> { if self.state != RepeaterState::Up { return Ok(()); } @@ -237,7 +247,7 @@ impl Repeater { }).unwrap(); let reply = self.recv_aux_timeout(200)?; if reply != drtioaux::Packet::ResetAck { - return Err("unexpected reply"); + return Err(drtioaux::Error::UnexpectedReply); } Ok(()) }