From 7204feae1f504e4a1dcd54b95c59e8f36c62b701 Mon Sep 17 00:00:00 2001 From: mwojcik Date: Mon, 22 Apr 2024 17:27:04 +0800 Subject: [PATCH] master: aux_transact support for async messages --- artiq/firmware/runtime/analyzer.rs | 13 +-- artiq/firmware/runtime/kern_hwreq.rs | 77 +++++++++------- artiq/firmware/runtime/kernel.rs | 24 ++--- artiq/firmware/runtime/main.rs | 8 +- artiq/firmware/runtime/moninj.rs | 41 +++++---- artiq/firmware/runtime/rtio_dma.rs | 16 ++-- artiq/firmware/runtime/rtio_mgt.rs | 129 +++++++++++++++------------ artiq/firmware/runtime/session.rs | 34 +++---- 8 files changed, 193 insertions(+), 149 deletions(-) diff --git a/artiq/firmware/runtime/analyzer.rs b/artiq/firmware/runtime/analyzer.rs index 41cca6e46..355da1977 100644 --- a/artiq/firmware/runtime/analyzer.rs +++ b/artiq/firmware/runtime/analyzer.rs @@ -52,7 +52,7 @@ pub mod remote_analyzer { pub data: Vec } - pub fn get_data(io: &Io, aux_mutex: &Mutex, routing_table: &drtio_routing::RoutingTable, + pub fn get_data(io: &Io, aux_mutex: &Mutex, ddma_mutex: &Mutex, subkernel_mutex: &Mutex, routing_table: &drtio_routing::RoutingTable, up_destinations: &Urc> ) -> Result { // gets data from satellites and returns consolidated data @@ -62,7 +62,7 @@ pub mod remote_analyzer { let mut remote_total_bytes = 0; let data_vec = drtio::analyzer_query( - io, aux_mutex, routing_table, up_destinations + io, aux_mutex, ddma_mutex, subkernel_mutex, routing_table, up_destinations )?; for data in data_vec { remote_total_bytes += data.total_byte_count; @@ -82,7 +82,8 @@ pub mod remote_analyzer { -fn worker(stream: &mut TcpStream, _io: &Io, _aux_mutex: &Mutex, +fn worker(stream: &mut TcpStream, _io: &Io, _aux_mutex: &Mutex, + _ddma_mutex: &Mutex, _subkernel_mutex: &Mutex, _routing_table: &drtio_routing::RoutingTable, _up_destinations: &Urc> ) -> Result<(), IoError> { @@ -96,7 +97,7 @@ fn worker(stream: &mut TcpStream, _io: &Io, _aux_mutex: &Mutex, #[cfg(has_drtio)] let remote = remote_analyzer::get_data( - _io, _aux_mutex, _routing_table, _up_destinations); + _io, _aux_mutex, _ddma_mutex, _subkernel_mutex, _routing_table, _up_destinations); #[cfg(has_drtio)] let (header, remote_data) = match remote { Ok(remote) => (Header { @@ -143,7 +144,7 @@ fn worker(stream: &mut TcpStream, _io: &Io, _aux_mutex: &Mutex, Ok(()) } -pub fn thread(io: Io, aux_mutex: &Mutex, +pub fn thread(io: Io, aux_mutex: &Mutex, ddma_mutex: &Mutex, subkernel_mutex: &Mutex, routing_table: &Urc>, up_destinations: &Urc>) { let listener = TcpListener::new(&io, 65535); @@ -158,7 +159,7 @@ pub fn thread(io: Io, aux_mutex: &Mutex, disarm(); let routing_table = routing_table.borrow(); - match worker(&mut stream, &io, aux_mutex, &routing_table, up_destinations) { + match worker(&mut stream, &io, aux_mutex, ddma_mutex, subkernel_mutex, &routing_table, up_destinations) { Ok(()) => (), Err(err) => error!("analyzer aborted: {}", err) } diff --git a/artiq/firmware/runtime/kern_hwreq.rs b/artiq/firmware/runtime/kern_hwreq.rs index 49aa2d8af..9dea387d5 100644 --- a/artiq/firmware/runtime/kern_hwreq.rs +++ b/artiq/firmware/runtime/kern_hwreq.rs @@ -11,13 +11,15 @@ use board_artiq::spi as local_spi; #[cfg(has_drtio)] mod remote_i2c { use drtioaux; + use drtio_routing; use rtio_mgt::drtio; use sched::{Io, Mutex}; - pub fn start(io: &Io, aux_mutex: &Mutex, + pub fn start(io: &Io, aux_mutex: &Mutex, ddma_mutex: &Mutex, subkernel_mutex: &Mutex, + routing_table: &drtio_routing::RoutingTable, linkno: u8, destination: u8, busno: u8 ) -> Result<(), &'static str> { - let reply = drtio::aux_transact(io, aux_mutex, linkno, + let reply = drtio::aux_transact(io, aux_mutex, ddma_mutex, subkernel_mutex, routing_table, linkno, &drtioaux::Packet::I2cStartRequest { destination: destination, busno: busno @@ -37,10 +39,11 @@ mod remote_i2c { } } - pub fn restart(io: &Io, aux_mutex: &Mutex, + pub fn restart(io: &Io, aux_mutex: &Mutex, ddma_mutex: &Mutex, subkernel_mutex: &Mutex, + routing_table: &drtio_routing::RoutingTable, linkno: u8, destination: u8, busno: u8 ) -> Result<(), &'static str> { - let reply = drtio::aux_transact(io, aux_mutex, linkno, + let reply = drtio::aux_transact(io, aux_mutex, ddma_mutex, subkernel_mutex, routing_table, linkno, &drtioaux::Packet::I2cRestartRequest { destination: destination, busno: busno @@ -60,10 +63,11 @@ mod remote_i2c { } } - pub fn stop(io: &Io, aux_mutex: &Mutex, + pub fn stop(io: &Io, aux_mutex: &Mutex, ddma_mutex: &Mutex, subkernel_mutex: &Mutex, + routing_table: &drtio_routing::RoutingTable, linkno: u8, destination: u8, busno: u8 ) -> Result<(), &'static str> { - let reply = drtio::aux_transact(io, aux_mutex, linkno, + let reply = drtio::aux_transact(io, aux_mutex, ddma_mutex, subkernel_mutex, routing_table, linkno, &drtioaux::Packet::I2cStopRequest { destination: destination, busno: busno @@ -83,10 +87,11 @@ mod remote_i2c { } } - pub fn write(io: &Io, aux_mutex: &Mutex, + pub fn write(io: &Io, aux_mutex: &Mutex, ddma_mutex: &Mutex, subkernel_mutex: &Mutex, + routing_table: &drtio_routing::RoutingTable, linkno: u8, destination: u8, busno: u8, data: u8 ) -> Result { - let reply = drtio::aux_transact(io, aux_mutex, linkno, + let reply = drtio::aux_transact(io, aux_mutex, ddma_mutex, subkernel_mutex, routing_table, linkno, &drtioaux::Packet::I2cWriteRequest { destination: destination, busno: busno, @@ -107,10 +112,11 @@ mod remote_i2c { } } - pub fn read(io: &Io, aux_mutex: &Mutex, + pub fn read(io: &Io, aux_mutex: &Mutex, ddma_mutex: &Mutex, subkernel_mutex: &Mutex, + routing_table: &drtio_routing::RoutingTable, linkno: u8, destination: u8, busno: u8, ack: bool ) -> Result { - let reply = drtio::aux_transact(io, aux_mutex, linkno, + let reply = drtio::aux_transact(io, aux_mutex, ddma_mutex, subkernel_mutex, routing_table, linkno, &drtioaux::Packet::I2cReadRequest { destination: destination, busno: busno, @@ -131,10 +137,11 @@ mod remote_i2c { } } - pub fn switch_select(io: &Io, aux_mutex: &Mutex, + pub fn switch_select(io: &Io, aux_mutex: &Mutex, ddma_mutex: &Mutex, subkernel_mutex: &Mutex, + routing_table: &drtio_routing::RoutingTable, linkno: u8, destination: u8, busno: u8, address: u8, mask: u8 ) -> Result<(), &'static str> { - let reply = drtio::aux_transact(io, aux_mutex, linkno, + let reply = drtio::aux_transact(io, aux_mutex, ddma_mutex, subkernel_mutex, routing_table, linkno, &drtioaux::Packet::I2cSwitchSelectRequest { destination: destination, busno: busno, @@ -160,13 +167,15 @@ mod remote_i2c { #[cfg(has_drtio)] mod remote_spi { use drtioaux; + use drtio_routing; use rtio_mgt::drtio; use sched::{Io, Mutex}; - pub fn set_config(io: &Io, aux_mutex: &Mutex, + pub fn set_config(io: &Io, aux_mutex: &Mutex, ddma_mutex: &Mutex, subkernel_mutex: &Mutex, + routing_table: &drtio_routing::RoutingTable, linkno: u8, destination: u8, busno: u8, flags: u8, length: u8, div: u8, cs: u8 ) -> Result<(), ()> { - let reply = drtio::aux_transact(io, aux_mutex, linkno, &drtioaux::Packet::SpiSetConfigRequest { + let reply = drtio::aux_transact(io, aux_mutex, ddma_mutex, subkernel_mutex, routing_table, linkno, &drtioaux::Packet::SpiSetConfigRequest { destination: destination, busno: busno, flags: flags, @@ -189,10 +198,11 @@ mod remote_spi { } } - pub fn write(io: &Io, aux_mutex: &Mutex, + pub fn write(io: &Io, aux_mutex: &Mutex, ddma_mutex: &Mutex, subkernel_mutex: &Mutex, + routing_table: &drtio_routing::RoutingTable, linkno: u8, destination: u8, busno: u8, data: u32 ) -> Result<(), ()> { - let reply = drtio::aux_transact(io, aux_mutex, linkno, &drtioaux::Packet::SpiWriteRequest { + let reply = drtio::aux_transact(io, aux_mutex, ddma_mutex, subkernel_mutex, routing_table, linkno, &drtioaux::Packet::SpiWriteRequest { destination: destination, busno: busno, data: data @@ -212,9 +222,10 @@ mod remote_spi { } } - pub fn read(io: &Io, aux_mutex: &Mutex, linkno: u8, destination: u8, busno: u8 + pub fn read(io: &Io, aux_mutex: &Mutex, ddma_mutex: &Mutex, subkernel_mutex: &Mutex, + routing_table: &drtio_routing::RoutingTable, linkno: u8, destination: u8, busno: u8 ) -> Result { - let reply = drtio::aux_transact(io, aux_mutex, linkno, + let reply = drtio::aux_transact(io, aux_mutex, ddma_mutex, subkernel_mutex, routing_table, linkno, &drtioaux::Packet::SpiReadRequest { destination: destination, busno: busno @@ -238,7 +249,7 @@ mod remote_spi { #[cfg(has_drtio)] macro_rules! dispatch { - ($io:ident, $aux_mutex:ident, $mod_local:ident, $mod_remote:ident, $routing_table:ident, $busno:expr, $func:ident $(, $param:expr)*) => {{ + ($io:ident, $aux_mutex:ident, $ddma_mutex:ident, $subkernel_mutex:ident, $mod_local:ident, $mod_remote:ident, $routing_table:ident, $busno:expr, $func:ident $(, $param:expr)*) => {{ let destination = ($busno >> 16) as u8; let busno = $busno as u8; let hop = $routing_table.0[destination as usize][0]; @@ -246,27 +257,27 @@ macro_rules! dispatch { $mod_local::$func(busno, $($param, )*) } else { let linkno = hop - 1; - $mod_remote::$func($io, $aux_mutex, linkno, destination, busno, $($param, )*) + $mod_remote::$func($io, $aux_mutex, $ddma_mutex, $subkernel_mutex, $routing_table, linkno, destination, busno, $($param, )*) } }} } #[cfg(not(has_drtio))] macro_rules! dispatch { - ($io:ident, $aux_mutex:ident, $mod_local:ident, $mod_remote:ident, $routing_table:ident, $busno:expr, $func:ident $(, $param:expr)*) => {{ + ($io:ident, $aux_mutex:ident, $ddma_mutex:ident, $subkernel_mutex:ident, $mod_local:ident, $mod_remote:ident, $routing_table:ident, $busno:expr, $func:ident $(, $param:expr)*) => {{ let busno = $busno as u8; $mod_local::$func(busno, $($param, )*) }} } -pub fn process_kern_hwreq(io: &Io, aux_mutex: &Mutex, - _routing_table: &drtio_routing::RoutingTable, +pub fn process_kern_hwreq(io: &Io, aux_mutex: &Mutex, ddma_mutex: &Mutex, subkernel_mutex: &Mutex, + routing_table: &drtio_routing::RoutingTable, _up_destinations: &Urc>, request: &kern::Message) -> Result> { match request { &kern::RtioInitRequest => { info!("resetting RTIO"); - rtio_mgt::reset(io, aux_mutex); + rtio_mgt::reset(io, aux_mutex, ddma_mutex, subkernel_mutex, routing_table); kern_acknowledge() } @@ -282,47 +293,47 @@ pub fn process_kern_hwreq(io: &Io, aux_mutex: &Mutex, } &kern::I2cStartRequest { busno } => { - let succeeded = dispatch!(io, aux_mutex, local_i2c, remote_i2c, _routing_table, busno, start).is_ok(); + let succeeded = dispatch!(io, aux_mutex, ddma_mutex, subkernel_mutex, local_i2c, remote_i2c, routing_table, busno, start).is_ok(); kern_send(io, &kern::I2cBasicReply { succeeded: succeeded }) } &kern::I2cRestartRequest { busno } => { - let succeeded = dispatch!(io, aux_mutex, local_i2c, remote_i2c, _routing_table, busno, restart).is_ok(); + let succeeded = dispatch!(io, aux_mutex, ddma_mutex, subkernel_mutex, local_i2c, remote_i2c, routing_table, busno, restart).is_ok(); kern_send(io, &kern::I2cBasicReply { succeeded: succeeded }) } &kern::I2cStopRequest { busno } => { - let succeeded = dispatch!(io, aux_mutex, local_i2c, remote_i2c, _routing_table, busno, stop).is_ok(); + let succeeded = dispatch!(io, aux_mutex, ddma_mutex, subkernel_mutex, local_i2c, remote_i2c, routing_table, busno, stop).is_ok(); kern_send(io, &kern::I2cBasicReply { succeeded: succeeded }) } &kern::I2cWriteRequest { busno, data } => { - match dispatch!(io, aux_mutex, local_i2c, remote_i2c, _routing_table, busno, write, data) { + match dispatch!(io, aux_mutex, ddma_mutex, subkernel_mutex, local_i2c, remote_i2c, routing_table, busno, write, data) { Ok(ack) => kern_send(io, &kern::I2cWriteReply { succeeded: true, ack: ack }), Err(_) => kern_send(io, &kern::I2cWriteReply { succeeded: false, ack: false }) } } &kern::I2cReadRequest { busno, ack } => { - match dispatch!(io, aux_mutex, local_i2c, remote_i2c, _routing_table, busno, read, ack) { + match dispatch!(io, aux_mutex, ddma_mutex, subkernel_mutex, local_i2c, remote_i2c, routing_table, busno, read, ack) { Ok(data) => kern_send(io, &kern::I2cReadReply { succeeded: true, data: data }), Err(_) => kern_send(io, &kern::I2cReadReply { succeeded: false, data: 0xff }) } } &kern::I2cSwitchSelectRequest { busno, address, mask } => { - let succeeded = dispatch!(io, aux_mutex, local_i2c, remote_i2c, _routing_table, busno, + let succeeded = dispatch!(io, aux_mutex, ddma_mutex, subkernel_mutex, local_i2c, remote_i2c, routing_table, busno, switch_select, address, mask).is_ok(); kern_send(io, &kern::I2cBasicReply { succeeded: succeeded }) } &kern::SpiSetConfigRequest { busno, flags, length, div, cs } => { - let succeeded = dispatch!(io, aux_mutex, local_spi, remote_spi, _routing_table, busno, + let succeeded = dispatch!(io, aux_mutex, ddma_mutex, subkernel_mutex, local_spi, remote_spi, routing_table, busno, set_config, flags, length, div, cs).is_ok(); kern_send(io, &kern::SpiBasicReply { succeeded: succeeded }) }, &kern::SpiWriteRequest { busno, data } => { - let succeeded = dispatch!(io, aux_mutex, local_spi, remote_spi, _routing_table, busno, + let succeeded = dispatch!(io, aux_mutex, ddma_mutex, subkernel_mutex, local_spi, remote_spi, routing_table, busno, write, data).is_ok(); kern_send(io, &kern::SpiBasicReply { succeeded: succeeded }) } &kern::SpiReadRequest { busno } => { - match dispatch!(io, aux_mutex, local_spi, remote_spi, _routing_table, busno, read) { + match dispatch!(io, aux_mutex, ddma_mutex, subkernel_mutex, local_spi, remote_spi, routing_table, busno, read) { Ok(data) => kern_send(io, &kern::SpiReadReply { succeeded: true, data: data }), Err(_) => kern_send(io, &kern::SpiReadReply { succeeded: false, data: 0 }) } diff --git a/artiq/firmware/runtime/kernel.rs b/artiq/firmware/runtime/kernel.rs index acbce47d7..dda190cd9 100644 --- a/artiq/firmware/runtime/kernel.rs +++ b/artiq/firmware/runtime/kernel.rs @@ -181,17 +181,17 @@ pub mod subkernel { Ok(()) } - pub fn upload(io: &Io, aux_mutex: &Mutex, subkernel_mutex: &Mutex, + pub fn upload(io: &Io, aux_mutex: &Mutex, ddma_mutex: &Mutex, subkernel_mutex: &Mutex, routing_table: &RoutingTable, id: u32) -> Result<(), Error> { let _lock = subkernel_mutex.lock(io)?; let subkernel = unsafe { SUBKERNELS.get_mut(&id).unwrap() }; - drtio::subkernel_upload(io, aux_mutex, routing_table, id, + drtio::subkernel_upload(io, aux_mutex, ddma_mutex, subkernel_mutex, routing_table, id, subkernel.destination, &subkernel.data)?; subkernel.state = SubkernelState::Uploaded; Ok(()) } - pub fn load(io: &Io, aux_mutex: &Mutex, subkernel_mutex: &Mutex, routing_table: &RoutingTable, + pub fn load(io: &Io, aux_mutex: &Mutex, ddma_mutex: &Mutex, subkernel_mutex: &Mutex, routing_table: &RoutingTable, id: u32, run: bool) -> Result<(), Error> { let _lock = subkernel_mutex.lock(io)?; let subkernel = unsafe { SUBKERNELS.get_mut(&id).unwrap() }; @@ -199,7 +199,7 @@ pub mod subkernel { error!("for id: {} expected Uploaded, got: {:?}", id, subkernel.state); return Err(Error::IncorrectState); } - drtio::subkernel_load(io, aux_mutex, routing_table, id, subkernel.destination, run)?; + drtio::subkernel_load(io, aux_mutex, ddma_mutex, subkernel_mutex, routing_table, id, subkernel.destination, run)?; if run { subkernel.state = SubkernelState::Running; } @@ -234,14 +234,14 @@ pub mod subkernel { } } - pub fn destination_changed(io: &Io, aux_mutex: &Mutex, subkernel_mutex: &Mutex, + pub fn destination_changed(io: &Io, aux_mutex: &Mutex, ddma_mutex: &Mutex, subkernel_mutex: &Mutex, routing_table: &RoutingTable, destination: u8, up: bool) { let _lock = subkernel_mutex.lock(io).unwrap(); let subkernels_iter = unsafe { SUBKERNELS.iter_mut() }; for (id, subkernel) in subkernels_iter { if subkernel.destination == destination { if up { - match drtio::subkernel_upload(io, aux_mutex, routing_table, *id, destination, &subkernel.data) + match drtio::subkernel_upload(io, aux_mutex, ddma_mutex, subkernel_mutex, routing_table, *id, destination, &subkernel.data) { Ok(_) => subkernel.state = SubkernelState::Uploaded, Err(e) => error!("Error adding subkernel on destination {}: {}", destination, e) @@ -256,7 +256,7 @@ pub mod subkernel { } } - pub fn retrieve_finish_status(io: &Io, aux_mutex: &Mutex, subkernel_mutex: &Mutex, + pub fn retrieve_finish_status(io: &Io, aux_mutex: &Mutex, ddma_mutex: &Mutex, subkernel_mutex: &Mutex, routing_table: &RoutingTable, id: u32) -> Result { let _lock = subkernel_mutex.lock(io)?; let mut subkernel = unsafe { SUBKERNELS.get_mut(&id).unwrap() }; @@ -267,7 +267,7 @@ pub mod subkernel { id: id, comm_lost: status == FinishStatus::CommLost, exception: if let FinishStatus::Exception(dest) = status { - Some(drtio::subkernel_retrieve_exception(io, aux_mutex, + Some(drtio::subkernel_retrieve_exception(io, aux_mutex, ddma_mutex, subkernel_mutex, routing_table, dest)?) } else { None } }) @@ -278,7 +278,7 @@ pub mod subkernel { } } - pub fn await_finish(io: &Io, aux_mutex: &Mutex, subkernel_mutex: &Mutex, + pub fn await_finish(io: &Io, aux_mutex: &Mutex, ddma_mutex: &Mutex, subkernel_mutex: &Mutex, routing_table: &RoutingTable, id: u32, timeout: i64) -> Result { { let _lock = subkernel_mutex.lock(io)?; @@ -309,7 +309,7 @@ pub mod subkernel { error!("Remote subkernel finish await timed out"); return Err(Error::Timeout); } - retrieve_finish_status(io, aux_mutex, subkernel_mutex, routing_table, id) + retrieve_finish_status(io, aux_mutex, ddma_mutex, subkernel_mutex, routing_table, id) } pub struct Message { @@ -418,7 +418,7 @@ pub mod subkernel { } } - pub fn message_send<'a>(io: &Io, aux_mutex: &Mutex, subkernel_mutex: &Mutex, + pub fn message_send<'a>(io: &Io, aux_mutex: &Mutex, ddma_mutex: &Mutex, subkernel_mutex: &Mutex, routing_table: &RoutingTable, id: u32, destination: Option, count: u8, tag: &'a [u8], message: *const *const () ) -> Result<(), Error> { let mut writer = Cursor::new(Vec::new()); @@ -433,7 +433,7 @@ pub mod subkernel { let data = &mut writer.into_inner()[3..]; data[0] = count; Ok(drtio::subkernel_send_message( - io, aux_mutex, routing_table, id, destination, data + io, aux_mutex, ddma_mutex, subkernel_mutex, routing_table, id, destination, data )?) } } \ No newline at end of file diff --git a/artiq/firmware/runtime/main.rs b/artiq/firmware/runtime/main.rs index a27c337ee..6d26432a7 100644 --- a/artiq/firmware/runtime/main.rs +++ b/artiq/firmware/runtime/main.rs @@ -214,15 +214,19 @@ fn startup() { #[cfg(any(has_rtio_moninj, has_drtio))] { let aux_mutex = aux_mutex.clone(); + let ddma_mutex = ddma_mutex.clone(); + let subkernel_mutex = subkernel_mutex.clone(); let drtio_routing_table = drtio_routing_table.clone(); - io.spawn(4096, move |io| { moninj::thread(io, &aux_mutex, &drtio_routing_table) }); + io.spawn(4096, move |io| { moninj::thread(io, &aux_mutex, &ddma_mutex, &subkernel_mutex, &drtio_routing_table) }); } #[cfg(has_rtio_analyzer)] { let aux_mutex = aux_mutex.clone(); + let ddma_mutex = ddma_mutex.clone(); + let subkernel_mutex = subkernel_mutex.clone(); let drtio_routing_table = drtio_routing_table.clone(); let up_destinations = up_destinations.clone(); - io.spawn(8192, move |io| { analyzer::thread(io, &aux_mutex, &drtio_routing_table, &up_destinations) }); + io.spawn(8192, move |io| { analyzer::thread(io, &aux_mutex, &ddma_mutex, &subkernel_mutex, &drtio_routing_table, &up_destinations) }); } #[cfg(has_grabber)] diff --git a/artiq/firmware/runtime/moninj.rs b/artiq/firmware/runtime/moninj.rs index 1ef6b4215..b66b65d95 100644 --- a/artiq/firmware/runtime/moninj.rs +++ b/artiq/firmware/runtime/moninj.rs @@ -50,12 +50,15 @@ mod local_moninj { #[cfg(has_drtio)] mod remote_moninj { use drtioaux; + use drtio_routing; use rtio_mgt::drtio; use sched::{Io, Mutex}; - pub fn read_probe(io: &Io, aux_mutex: &Mutex, linkno: u8, + pub fn read_probe(io: &Io, aux_mutex: &Mutex, + ddma_mutex: &Mutex, subkernel_mutex: &Mutex, + routing_table: &drtio_routing::RoutingTable, linkno: u8, destination: u8, channel: u16, probe: u8) -> u64 { - let reply = drtio::aux_transact(io, aux_mutex, linkno, + let reply = drtio::aux_transact(io, aux_mutex, ddma_mutex, subkernel_mutex, routing_table, linkno, &drtioaux::Packet::MonitorRequest { destination: destination, channel: channel, @@ -69,7 +72,9 @@ mod remote_moninj { 0 } - pub fn inject(io: &Io, aux_mutex: &Mutex, linkno: u8, + pub fn inject(io: &Io, aux_mutex: &Mutex, + _ddma_mutex: &Mutex, _subkernel_mutex: &Mutex, + _routing_table: &drtio_routing::RoutingTable, linkno: u8, destination: u8, channel: u16, overrd: u8, value: u8) { let _lock = aux_mutex.lock(io).unwrap(); drtioaux::send(linkno, &drtioaux::Packet::InjectionRequest { @@ -80,9 +85,11 @@ mod remote_moninj { }).unwrap(); } - pub fn read_injection_status(io: &Io, aux_mutex: &Mutex, linkno: u8, + pub fn read_injection_status(io: &Io, aux_mutex: &Mutex, + ddma_mutex: &Mutex, subkernel_mutex: &Mutex, + routing_table: &drtio_routing::RoutingTable, linkno: u8, destination: u8, channel: u16, overrd: u8) -> u8 { - let reply = drtio::aux_transact(io, aux_mutex, linkno, + let reply = drtio::aux_transact(io, aux_mutex, ddma_mutex, subkernel_mutex, routing_table, linkno, &drtioaux::Packet::InjectionStatusRequest { destination: destination, channel: channel, @@ -99,7 +106,7 @@ mod remote_moninj { #[cfg(has_drtio)] macro_rules! dispatch { - ($io:ident, $aux_mutex:ident, $routing_table:ident, $channel:expr, $func:ident $(, $param:expr)*) => {{ + ($io:ident, $aux_mutex:ident, $ddma_mutex:ident, $subkernel_mutex:ident, $routing_table:ident, $channel:expr, $func:ident $(, $param:expr)*) => {{ let destination = ($channel >> 16) as u8; let channel = $channel as u16; let hop = $routing_table.0[destination as usize][0]; @@ -107,21 +114,21 @@ macro_rules! dispatch { local_moninj::$func(channel, $($param, )*) } else { let linkno = hop - 1; - remote_moninj::$func($io, $aux_mutex, linkno, destination, channel, $($param, )*) + remote_moninj::$func($io, $aux_mutex, $ddma_mutex, $subkernel_mutex, $routing_table, linkno, destination, channel, $($param, )*) } }} } #[cfg(not(has_drtio))] macro_rules! dispatch { - ($io:ident, $aux_mutex:ident, $routing_table:ident, $channel:expr, $func:ident $(, $param:expr)*) => {{ + ($io:ident, $aux_mutex:ident, $ddma_mutex:ident, $subkernel_mutex:ident, $routing_table:ident, $channel:expr, $func:ident $(, $param:expr)*) => {{ let channel = $channel as u16; local_moninj::$func(channel, $($param, )*) }} } -fn connection_worker(io: &Io, _aux_mutex: &Mutex, _routing_table: &drtio_routing::RoutingTable, - mut stream: &mut TcpStream) -> Result<(), Error> { +fn connection_worker(io: &Io, _aux_mutex: &Mutex, _ddma_mutex: &Mutex, _subkernel_mutex: &Mutex, + _routing_table: &drtio_routing::RoutingTable, mut stream: &mut TcpStream) -> Result<(), Error> { let mut probe_watch_list = BTreeMap::new(); let mut inject_watch_list = BTreeMap::new(); let mut next_check = 0; @@ -150,9 +157,9 @@ fn connection_worker(io: &Io, _aux_mutex: &Mutex, _routing_table: &drtio_routing } }, HostMessage::Inject { channel, overrd, value } => dispatch!( - io, _aux_mutex, _routing_table, channel, inject, overrd, value), + io, _aux_mutex, _ddma_mutex, _subkernel_mutex, _routing_table, channel, inject, overrd, value), HostMessage::GetInjectionStatus { channel, overrd } => { - let value = dispatch!(io, _aux_mutex, _routing_table, channel, read_injection_status, overrd); + let value = dispatch!(io, _aux_mutex, _ddma_mutex, _subkernel_mutex, _routing_table, channel, read_injection_status, overrd); let reply = DeviceMessage::InjectionStatus { channel: channel, overrd: overrd, @@ -169,7 +176,7 @@ fn connection_worker(io: &Io, _aux_mutex: &Mutex, _routing_table: &drtio_routing if clock::get_ms() > next_check { for (&(channel, probe), previous) in probe_watch_list.iter_mut() { - let current = dispatch!(io, _aux_mutex, _routing_table, channel, read_probe, probe); + let current = dispatch!(io, _aux_mutex, _ddma_mutex, _subkernel_mutex, _routing_table, channel, read_probe, probe); if previous.is_none() || previous.unwrap() != current { let message = DeviceMessage::MonitorStatus { channel: channel, @@ -184,7 +191,7 @@ fn connection_worker(io: &Io, _aux_mutex: &Mutex, _routing_table: &drtio_routing } } for (&(channel, overrd), previous) in inject_watch_list.iter_mut() { - let current = dispatch!(io, _aux_mutex, _routing_table, channel, read_injection_status, overrd); + let current = dispatch!(io, _aux_mutex, _ddma_mutex, _subkernel_mutex, _routing_table, channel, read_injection_status, overrd); if previous.is_none() || previous.unwrap() != current { let message = DeviceMessage::InjectionStatus { channel: channel, @@ -205,18 +212,20 @@ fn connection_worker(io: &Io, _aux_mutex: &Mutex, _routing_table: &drtio_routing } } -pub fn thread(io: Io, aux_mutex: &Mutex, routing_table: &Urc>) { +pub fn thread(io: Io, aux_mutex: &Mutex, ddma_mutex: &Mutex, subkernel_mutex: &Mutex, routing_table: &Urc>) { let listener = TcpListener::new(&io, 2047); listener.listen(1383).expect("moninj: cannot listen"); loop { let aux_mutex = aux_mutex.clone(); + let ddma_mutex = ddma_mutex.clone(); + let subkernel_mutex = subkernel_mutex.clone(); let routing_table = routing_table.clone(); let stream = listener.accept().expect("moninj: cannot accept").into_handle(); io.spawn(16384, move |io| { let routing_table = routing_table.borrow(); let mut stream = TcpStream::from_handle(&io, stream); - match connection_worker(&io, &aux_mutex, &routing_table, &mut stream) { + match connection_worker(&io, &aux_mutex, &ddma_mutex, &subkernel_mutex, &routing_table, &mut stream) { Ok(()) => {}, Err(err) => error!("moninj aborted: {}", err) } diff --git a/artiq/firmware/runtime/rtio_dma.rs b/artiq/firmware/runtime/rtio_dma.rs index 666986919..e1efa5567 100644 --- a/artiq/firmware/runtime/rtio_dma.rs +++ b/artiq/firmware/runtime/rtio_dma.rs @@ -120,12 +120,12 @@ pub mod remote_dma { Ok(playback_state) } - pub fn erase(io: &Io, aux_mutex: &Mutex, ddma_mutex: &Mutex, + pub fn erase(io: &Io, aux_mutex: &Mutex, ddma_mutex: &Mutex, subkernel_mutex: &Mutex, routing_table: &RoutingTable, id: u32) -> Result<(), Error> { let _lock = ddma_mutex.lock(io)?; let destinations = unsafe { TRACES.get(&id).unwrap() }; for destination in destinations.keys() { - match drtio::ddma_send_erase(io, aux_mutex, routing_table, id, *destination) { + match drtio::ddma_send_erase(io, aux_mutex, ddma_mutex, subkernel_mutex, routing_table, id, *destination) { Ok(_) => (), Err(e) => error!("Error erasing trace on DMA: {}", e) } @@ -134,18 +134,18 @@ pub mod remote_dma { Ok(()) } - pub fn upload_traces(io: &Io, aux_mutex: &Mutex, ddma_mutex: &Mutex, + pub fn upload_traces(io: &Io, aux_mutex: &Mutex, ddma_mutex: &Mutex, subkernel_mutex: &Mutex, routing_table: &RoutingTable, id: u32) -> Result<(), Error> { let _lock = ddma_mutex.lock(io)?; let traces = unsafe { TRACES.get_mut(&id).unwrap() }; for (destination, mut trace) in traces { - drtio::ddma_upload_trace(io, aux_mutex, routing_table, id, *destination, trace.get_trace())?; + drtio::ddma_upload_trace(io, aux_mutex, ddma_mutex, subkernel_mutex, routing_table, id, *destination, trace.get_trace())?; trace.state = RemoteState::Loaded; } Ok(()) } - pub fn playback(io: &Io, aux_mutex: &Mutex, ddma_mutex: &Mutex, + pub fn playback(io: &Io, aux_mutex: &Mutex, ddma_mutex: &Mutex, subkernel_mutex: &Mutex, routing_table: &RoutingTable, id: u32, timestamp: u64) -> Result<(), Error>{ // triggers playback on satellites let destinations = unsafe { @@ -161,7 +161,7 @@ pub mod remote_dma { return Err(Error::IncorrectState); } } - drtio::ddma_send_playback(io, aux_mutex, routing_table, id, *destination, timestamp)?; + drtio::ddma_send_playback(io, aux_mutex, ddma_mutex, subkernel_mutex, routing_table, id, *destination, timestamp)?; } Ok(()) } @@ -178,7 +178,7 @@ pub mod remote_dma { }; } - pub fn destination_changed(io: &Io, aux_mutex: &Mutex, ddma_mutex: &Mutex, + pub fn destination_changed(io: &Io, aux_mutex: &Mutex, ddma_mutex: &Mutex, subkernel_mutex: &Mutex, routing_table: &RoutingTable, destination: u8, up: bool) { // update state of the destination, resend traces if it's up let _lock = ddma_mutex.lock(io).unwrap(); @@ -186,7 +186,7 @@ pub mod remote_dma { for (id, dest_traces) in traces_iter { if let Some(trace) = dest_traces.get_mut(&destination) { if up { - match drtio::ddma_upload_trace(io, aux_mutex, routing_table, *id, destination, trace.get_trace()) + match drtio::ddma_upload_trace(io, aux_mutex, ddma_mutex, subkernel_mutex, routing_table, *id, destination, trace.get_trace()) { Ok(_) => trace.state = RemoteState::Loaded, Err(e) => error!("Error adding DMA trace on destination {}: {}", destination, e) diff --git a/artiq/firmware/runtime/rtio_mgt.rs b/artiq/firmware/runtime/rtio_mgt.rs index 3f2c9e7f3..f08737691 100644 --- a/artiq/firmware/runtime/rtio_mgt.rs +++ b/artiq/firmware/runtime/rtio_mgt.rs @@ -87,7 +87,6 @@ pub mod drtio { if clock::get_ms() > max_time { return Err(Error::Timeout); } - // todo: reinsert handling of async messages match drtioaux::recv(linkno) { Ok(Some(packet)) => return Ok(packet), Ok(None) => (), @@ -118,7 +117,7 @@ pub mod drtio { ).unwrap(); None }, - // routable packets + // (potentially) routable packets drtioaux::Packet::DmaAddTraceRequest { destination, .. } | drtioaux::Packet::DmaAddTraceReply { destination, .. } | drtioaux::Packet::DmaRemoveTraceRequest { destination, .. } | @@ -131,26 +130,42 @@ pub mod drtio { drtioaux::Packet::SubkernelMessageAck { destination, .. } | drtioaux::Packet::DmaPlaybackStatus { destination, .. } | drtioaux::Packet::SubkernelFinished { destination, .. } => { - let dest_link = routing_table.0[destination as usize][0] - 1; - if dest_link == linkno { - warn!("[LINK#{}] Re-routed packet would return to the same link, dropping: {:?}", linkno, packet); - } else if destination == 0 { - warn!("[LINK#{}] Received invalid routable packet: {:?}", linkno, packet) + if destination == 0 { + Some(packet) } else { - drtioaux::send(dest_link, &packet).unwrap(); + let dest_link = routing_table.0[destination as usize][0] - 1; + if dest_link == linkno { + warn!("[LINK#{}] Re-routed packet would return to the same link, dropping: {:?}", linkno, packet); + } else { + drtioaux::send(dest_link, &packet).unwrap(); + } + None } - None } other => Some(other) } } - pub fn aux_transact(io: &Io, aux_mutex: &Mutex, linkno: u8, request: &drtioaux::Packet + pub fn aux_transact(io: &Io, aux_mutex: &Mutex, ddma_mutex: &Mutex, subkernel_mutex: &Mutex, + routing_table: &drtio_routing::RoutingTable, linkno: u8, request: &drtioaux::Packet ) -> Result { let _lock = aux_mutex.lock(io)?; drtioaux::send(linkno, request).unwrap(); - let reply = recv_aux_timeout(io, linkno, 200)?; - Ok(reply) + loop { + let reply = recv_aux_timeout(io, linkno, 200)?; + if let Some(reply) = process_async_packets(io, ddma_mutex, subkernel_mutex, routing_table, linkno, reply) { + // returns none if it was an async packet + return Ok(reply); + } + } + } + + fn setup_transact(io: &Io, aux_mutex: &Mutex, linkno: u8, request: &drtioaux::Packet) -> Result { + /* shorter aux_transact for setup purposes, no checking for async packets, + as they should not be generated yet */ + let _lock = aux_mutex.lock(io)?; + drtioaux::send(linkno, request).unwrap(); + recv_aux_timeout(io, linkno, 200) } pub fn clear_buffers(io: &Io, aux_mutex: &Mutex) { @@ -173,7 +188,7 @@ pub mod drtio { if count > 100 { return 0; } - let reply = aux_transact(io, aux_mutex, linkno, &drtioaux::Packet::EchoRequest); + let reply = setup_transact(io, aux_mutex, linkno, &drtioaux::Packet::EchoRequest); match reply { Ok(drtioaux::Packet::EchoReply) => { // make sure receive buffer is drained @@ -212,7 +227,7 @@ pub mod drtio { fn load_routing_table(io: &Io, aux_mutex: &Mutex, linkno: u8, routing_table: &drtio_routing::RoutingTable) -> Result<(), Error> { for i in 0..drtio_routing::DEST_COUNT { - let reply = aux_transact(io, aux_mutex, linkno, &drtioaux::Packet::RoutingSetPath { + let reply = setup_transact(io, aux_mutex, linkno, &drtioaux::Packet::RoutingSetPath { destination: i as u8, hops: routing_table.0[i] })?; @@ -225,7 +240,7 @@ pub mod drtio { fn set_rank(io: &Io, aux_mutex: &Mutex, linkno: u8, rank: u8) -> Result<(), Error> { - let reply = aux_transact(io, aux_mutex, linkno, + let reply = setup_transact(io, aux_mutex, linkno, &drtioaux::Packet::RoutingSetRank { rank: rank })?; @@ -321,7 +336,8 @@ pub mod drtio { let linkno = hop - 1; if destination_up(up_destinations, destination) { if up_links[linkno as usize] { - let reply = aux_transact(io, aux_mutex, linkno, + let reply = aux_transact(io, aux_mutex, + ddma_mutex, subkernel_mutex, routing_table, linkno, &drtioaux::Packet::DestinationStatusRequest { destination: destination }); @@ -329,8 +345,8 @@ pub mod drtio { match reply { drtioaux::Packet::DestinationDownReply => { destination_set_up(routing_table, up_destinations, destination, false); - remote_dma::destination_changed(io, aux_mutex, ddma_mutex, routing_table, destination, false); - subkernel::destination_changed(io, aux_mutex, subkernel_mutex, routing_table, destination, false); + remote_dma::destination_changed(io, aux_mutex, ddma_mutex, subkernel_mutex, routing_table, destination, false); + subkernel::destination_changed(io, aux_mutex, ddma_mutex, subkernel_mutex, routing_table, destination, false); } drtioaux::Packet::DestinationOkReply => (), drtioaux::Packet::DestinationSequenceErrorReply { channel } => { @@ -353,12 +369,13 @@ pub mod drtio { } } else { destination_set_up(routing_table, up_destinations, destination, false); - remote_dma::destination_changed(io, aux_mutex, ddma_mutex, routing_table, destination, false); - subkernel::destination_changed(io, aux_mutex, subkernel_mutex, routing_table, destination, false); + remote_dma::destination_changed(io, aux_mutex, ddma_mutex, subkernel_mutex, routing_table, destination, false); + subkernel::destination_changed(io, aux_mutex, ddma_mutex, subkernel_mutex, routing_table, destination, false); } } else { if up_links[linkno as usize] { - let reply = aux_transact(io, aux_mutex, linkno, + let reply = aux_transact(io, aux_mutex, ddma_mutex, + subkernel_mutex, routing_table, linkno, &drtioaux::Packet::DestinationStatusRequest { destination: destination }); @@ -367,8 +384,8 @@ pub mod drtio { Ok(drtioaux::Packet::DestinationOkReply) => { destination_set_up(routing_table, up_destinations, destination, true); init_buffer_space(destination as u8, linkno); - remote_dma::destination_changed(io, aux_mutex, ddma_mutex, routing_table, destination, true); - subkernel::destination_changed(io, aux_mutex, subkernel_mutex, routing_table, destination, true); + remote_dma::destination_changed(io, aux_mutex, ddma_mutex, subkernel_mutex, routing_table, destination, true); + subkernel::destination_changed(io, aux_mutex, ddma_mutex, subkernel_mutex, routing_table, destination, true); }, Ok(packet) => error!("[DEST#{}] received unexpected aux packet: {:?}", destination, packet), Err(e) => error!("[DEST#{}] communication failed ({:?})", destination, e) @@ -425,7 +442,7 @@ pub mod drtio { } } - pub fn reset(io: &Io, aux_mutex: &Mutex) { + pub fn reset(io: &Io, aux_mutex: &Mutex, ddma_mutex: &Mutex, subkernel_mutex: &Mutex, routing_table: &drtio_routing::RoutingTable) { for linkno in 0..csr::DRTIO.len() { unsafe { (csr::DRTIO[linkno].reset_write)(1); @@ -441,7 +458,7 @@ pub mod drtio { for linkno in 0..csr::DRTIO.len() { let linkno = linkno as u8; if link_rx_up(linkno) { - let reply = aux_transact(io, aux_mutex, linkno, + let reply = aux_transact(io, aux_mutex, ddma_mutex, subkernel_mutex, routing_table, linkno, &drtioaux::Packet::ResetRequest); match reply { Ok(drtioaux::Packet::ResetAck) => (), @@ -468,12 +485,12 @@ pub mod drtio { Ok(()) } - pub fn ddma_upload_trace(io: &Io, aux_mutex: &Mutex, - routing_table: &drtio_routing::RoutingTable, - id: u32, destination: u8, trace: &[u8]) -> Result<(), Error> { + pub fn ddma_upload_trace(io: &Io, aux_mutex: &Mutex, ddma_mutex: &Mutex, subkernel_mutex: &Mutex, + routing_table: &drtio_routing::RoutingTable, id: u32, destination: u8, trace: &[u8] + ) -> Result<(), Error> { let linkno = routing_table.0[destination as usize][0] - 1; partition_data(trace, |slice, status, len: usize| { - let reply = aux_transact(io, aux_mutex, linkno, + let reply = aux_transact(io, aux_mutex, ddma_mutex, subkernel_mutex, routing_table, linkno, &drtioaux::Packet::DmaAddTraceRequest { id: id, source: 0, destination: destination, status: status, length: len as u16, trace: *slice})?; match reply { @@ -484,10 +501,10 @@ pub mod drtio { }) } - pub fn ddma_send_erase(io: &Io, aux_mutex: &Mutex, routing_table: &drtio_routing::RoutingTable, - id: u32, destination: u8) -> Result<(), Error> { + pub fn ddma_send_erase(io: &Io, aux_mutex: &Mutex, ddma_mutex: &Mutex, subkernel_mutex: &Mutex, + routing_table: &drtio_routing::RoutingTable, id: u32, destination: u8) -> Result<(), Error> { let linkno = routing_table.0[destination as usize][0] - 1; - let reply = aux_transact(io, aux_mutex, linkno, + let reply = aux_transact(io, aux_mutex, ddma_mutex, subkernel_mutex, routing_table, linkno, &drtioaux::Packet::DmaRemoveTraceRequest { id: id, source: 0, destination: destination })?; match reply { drtioaux::Packet::DmaRemoveTraceReply { destination: 0, succeeded: true } => Ok(()), @@ -496,10 +513,10 @@ pub mod drtio { } } - pub fn ddma_send_playback(io: &Io, aux_mutex: &Mutex, routing_table: &drtio_routing::RoutingTable, - id: u32, destination: u8, timestamp: u64) -> Result<(), Error> { + pub fn ddma_send_playback(io: &Io, aux_mutex: &Mutex, ddma_mutex: &Mutex, subkernel_mutex: &Mutex, + routing_table: &drtio_routing::RoutingTable, id: u32, destination: u8, timestamp: u64) -> Result<(), Error> { let linkno = routing_table.0[destination as usize][0] - 1; - let reply = aux_transact(io, aux_mutex, linkno, + let reply = aux_transact(io, aux_mutex, ddma_mutex, subkernel_mutex, routing_table, linkno, &drtioaux::Packet::DmaPlaybackRequest{ id: id, source: 0, destination: destination, timestamp: timestamp })?; match reply { drtioaux::Packet::DmaPlaybackReply { destination: 0, succeeded: true } => Ok(()), @@ -510,10 +527,10 @@ pub mod drtio { } #[cfg(has_rtio_analyzer)] - fn analyzer_get_data(io: &Io, aux_mutex: &Mutex, routing_table: &drtio_routing::RoutingTable, - destination: u8) -> Result { + fn analyzer_get_data(io: &Io, aux_mutex: &Mutex, ddma_mutex: &Mutex, subkernel_mutex: &Mutex, + routing_table: &drtio_routing::RoutingTable, destination: u8) -> Result { let linkno = routing_table.0[destination as usize][0] - 1; - let reply = aux_transact(io, aux_mutex, linkno, + let reply = aux_transact(io, aux_mutex, ddma_mutex, subkernel_mutex, routing_table, linkno, &drtioaux::Packet::AnalyzerHeaderRequest { destination: destination })?; let (sent, total, overflow) = match reply { drtioaux::Packet::AnalyzerHeader { sent_bytes, total_byte_count, overflow_occurred } => @@ -525,7 +542,7 @@ pub mod drtio { if sent > 0 { let mut last_packet = false; while !last_packet { - let reply = aux_transact(io, aux_mutex, linkno, + let reply = aux_transact(io, aux_mutex, ddma_mutex, subkernel_mutex, routing_table, linkno, &drtioaux::Packet::AnalyzerDataRequest { destination: destination })?; match reply { drtioaux::Packet::AnalyzerData { last, length, data } => { @@ -546,23 +563,23 @@ pub mod drtio { } #[cfg(has_rtio_analyzer)] - pub fn analyzer_query(io: &Io, aux_mutex: &Mutex, routing_table: &drtio_routing::RoutingTable, - up_destinations: &Urc> + pub fn analyzer_query(io: &Io, aux_mutex: &Mutex, ddma_mutex: &Mutex, subkernel_mutex: &Mutex, + routing_table: &drtio_routing::RoutingTable, up_destinations: &Urc> ) -> Result, Error> { let mut remote_buffers: Vec = Vec::new(); for i in 1..drtio_routing::DEST_COUNT { if destination_up(up_destinations, i as u8) { - remote_buffers.push(analyzer_get_data(io, aux_mutex, routing_table, i as u8)?); + remote_buffers.push(analyzer_get_data(io, aux_mutex, ddma_mutex, subkernel_mutex, routing_table, i as u8)?); } } Ok(remote_buffers) } - pub fn subkernel_upload(io: &Io, aux_mutex: &Mutex, routing_table: &drtio_routing::RoutingTable, - id: u32, destination: u8, data: &Vec) -> Result<(), Error> { + pub fn subkernel_upload(io: &Io, aux_mutex: &Mutex, ddma_mutex: &Mutex, subkernel_mutex: &Mutex, + routing_table: &drtio_routing::RoutingTable, id: u32, destination: u8, data: &Vec) -> Result<(), Error> { let linkno = routing_table.0[destination as usize][0] - 1; partition_data(data, |slice, status, len: usize| { - let reply = aux_transact(io, aux_mutex, linkno, + let reply = aux_transact(io, aux_mutex, ddma_mutex, subkernel_mutex, routing_table, linkno, &drtioaux::Packet::SubkernelAddDataRequest { id: id, destination: destination, status: status, length: len as u16, data: *slice})?; match reply { @@ -574,10 +591,11 @@ pub mod drtio { }) } - pub fn subkernel_load(io: &Io, aux_mutex: &Mutex, routing_table: &drtio_routing::RoutingTable, - id: u32, destination: u8, run: bool) -> Result<(), Error> { + pub fn subkernel_load(io: &Io, aux_mutex: &Mutex, ddma_mutex: &Mutex, subkernel_mutex: &Mutex, + routing_table: &drtio_routing::RoutingTable, id: u32, destination: u8, run: bool + ) -> Result<(), Error> { let linkno = routing_table.0[destination as usize][0] - 1; - let reply = aux_transact(io, aux_mutex, linkno, + let reply = aux_transact(io, aux_mutex, ddma_mutex, subkernel_mutex, routing_table, linkno, &drtioaux::Packet::SubkernelLoadRunRequest{ id: id, source: 0, destination: destination, run: run })?; match reply { drtioaux::Packet::SubkernelLoadRunReply { destination: 0, succeeded: true } => Ok(()), @@ -587,13 +605,13 @@ pub mod drtio { } } - pub fn subkernel_retrieve_exception(io: &Io, aux_mutex: &Mutex, + pub fn subkernel_retrieve_exception(io: &Io, aux_mutex: &Mutex, ddma_mutex: &Mutex, subkernel_mutex: &Mutex, routing_table: &drtio_routing::RoutingTable, destination: u8 ) -> Result, Error> { let linkno = routing_table.0[destination as usize][0] - 1; let mut remote_data: Vec = Vec::new(); loop { - let reply = aux_transact(io, aux_mutex, linkno, + let reply = aux_transact(io, aux_mutex, ddma_mutex, subkernel_mutex, routing_table, linkno, &drtioaux::Packet::SubkernelExceptionRequest { destination: destination })?; match reply { drtioaux::Packet::SubkernelException { last, length, data } => { @@ -607,12 +625,12 @@ pub mod drtio { } } - pub fn subkernel_send_message(io: &Io, aux_mutex: &Mutex, + pub fn subkernel_send_message(io: &Io, aux_mutex: &Mutex, ddma_mutex: &Mutex, subkernel_mutex: &Mutex, routing_table: &drtio_routing::RoutingTable, id: u32, destination: u8, message: &[u8] ) -> Result<(), Error> { let linkno = routing_table.0[destination as usize][0] - 1; partition_data(message, |slice, status, len: usize| { - let reply = aux_transact(io, aux_mutex, linkno, + let reply = aux_transact(io, aux_mutex, ddma_mutex, subkernel_mutex, routing_table, linkno, &drtioaux::Packet::SubkernelMessage { source: 0, destination: destination, id: id, status: status, length: len as u16, data: *slice})?; @@ -632,7 +650,7 @@ pub mod drtio { _routing_table: &Urc>, _up_destinations: &Urc>, _ddma_mutex: &Mutex, _subkernel_mutex: &Mutex) {} - pub fn reset(_io: &Io, _aux_mutex: &Mutex) {} + pub fn reset(_io: &Io, _aux_mutex: &Mutex, _ddma_mutex: &Mutex, _subkernel_mutex: &Mutex, _routing_table: &drtio_routing::RoutingTable) {} } static mut SEEN_ASYNC_ERRORS: u8 = 0; @@ -704,9 +722,10 @@ pub fn startup(io: &Io, aux_mutex: &Mutex, io.spawn(4096, async_error_thread); } -pub fn reset(io: &Io, aux_mutex: &Mutex) { +pub fn reset(io: &Io, aux_mutex: &Mutex, ddma_mutex: &Mutex, subkernel_mutex: &Mutex, + routing_table: &drtio_routing::RoutingTable) { unsafe { csr::rtio_core::reset_write(1); } - drtio::reset(io, aux_mutex) + drtio::reset(io, aux_mutex, ddma_mutex, subkernel_mutex, routing_table) } diff --git a/artiq/firmware/runtime/session.rs b/artiq/firmware/runtime/session.rs index 52daaa2d0..388a74337 100644 --- a/artiq/firmware/runtime/session.rs +++ b/artiq/firmware/runtime/session.rs @@ -316,7 +316,7 @@ fn kern_run(session: &mut Session) -> Result<(), Error> { } -fn process_flash_kernel(io: &Io, _aux_mutex: &Mutex, _subkernel_mutex: &Mutex, +fn process_flash_kernel(io: &Io, _aux_mutex: &Mutex, _subkernel_mutex: &Mutex, _ddma_mutex: &Mutex, _routing_table: &drtio_routing::RoutingTable, _up_destinations: &Urc>, session: &mut Session, kernel: &[u8] @@ -355,7 +355,7 @@ fn process_flash_kernel(io: &Io, _aux_mutex: &Mutex, _subkernel_mutex: &Mutex, if up { let subkernel_lib = entry.data().to_vec(); subkernel::add_subkernel(io, _subkernel_mutex, sid, dest, subkernel_lib)?; - subkernel::upload(io, _aux_mutex, _subkernel_mutex, _routing_table, sid)?; + subkernel::upload(io, _aux_mutex, _ddma_mutex, _subkernel_mutex, _routing_table, sid)?; } else { return Err(Error::DestinationDown); } @@ -468,7 +468,7 @@ fn process_host_message(io: &Io, _aux_mutex: &Mutex, _ddma_mutex: &Mutex, _subke #[cfg(has_drtio)] { subkernel::add_subkernel(io, _subkernel_mutex, _id, _dest, _kernel)?; - match subkernel::upload(io, _aux_mutex, _subkernel_mutex, _routing_table, _id) { + match subkernel::upload(io, _aux_mutex, _ddma_mutex, _subkernel_mutex, _routing_table, _id) { Ok(_) => host_write(stream, host::Reply::LoadCompleted)?, Err(error) => { subkernel::clear_subkernels(io, _subkernel_mutex)?; @@ -489,7 +489,7 @@ fn process_host_message(io: &Io, _aux_mutex: &Mutex, _ddma_mutex: &Mutex, _subke fn process_kern_message(io: &Io, aux_mutex: &Mutex, routing_table: &drtio_routing::RoutingTable, up_destinations: &Urc>, - ddma_mutex: &Mutex, _subkernel_mutex: &Mutex, mut stream: Option<&mut TcpStream>, + ddma_mutex: &Mutex, subkernel_mutex: &Mutex, mut stream: Option<&mut TcpStream>, session: &mut Session) -> Result> { kern_recv_notrace(io, |request| { match (request, session.kernel_state) { @@ -507,7 +507,7 @@ fn process_kern_message(io: &Io, aux_mutex: &Mutex, kern_recv_dotrace(request); - if kern_hwreq::process_kern_hwreq(io, aux_mutex, routing_table, up_destinations, request)? { + if kern_hwreq::process_kern_hwreq(io, aux_mutex, ddma_mutex, subkernel_mutex, routing_table, up_destinations, request)? { return Ok(false) } @@ -531,7 +531,7 @@ fn process_kern_message(io: &Io, aux_mutex: &Mutex, if let Some(_id) = session.congress.dma_manager.record_start(name) { // replace the record #[cfg(has_drtio)] - remote_dma::erase(io, aux_mutex, ddma_mutex, routing_table, _id)?; + remote_dma::erase(io, aux_mutex, ddma_mutex, subkernel_mutex, routing_table, _id)?; } kern_acknowledge() } @@ -543,7 +543,7 @@ fn process_kern_message(io: &Io, aux_mutex: &Mutex, let _id = session.congress.dma_manager.record_stop(duration, enable_ddma, io, ddma_mutex)?; #[cfg(has_drtio)] if enable_ddma { - remote_dma::upload_traces(io, aux_mutex, ddma_mutex, routing_table, _id)?; + remote_dma::upload_traces(io, aux_mutex, ddma_mutex, subkernel_mutex, routing_table, _id)?; } cache::flush_l2_cache(); kern_acknowledge() @@ -551,7 +551,7 @@ fn process_kern_message(io: &Io, aux_mutex: &Mutex, &kern::DmaEraseRequest { name } => { #[cfg(has_drtio)] if let Some(id) = session.congress.dma_manager.get_id(name) { - remote_dma::erase(io, aux_mutex, ddma_mutex, routing_table, *id)?; + remote_dma::erase(io, aux_mutex, ddma_mutex, subkernel_mutex, routing_table, *id)?; } session.congress.dma_manager.erase(name); kern_acknowledge() @@ -574,7 +574,7 @@ fn process_kern_message(io: &Io, aux_mutex: &Mutex, } &kern::DmaStartRemoteRequest { id: _id, timestamp: _timestamp } => { #[cfg(has_drtio)] - remote_dma::playback(io, aux_mutex, ddma_mutex, routing_table, _id as u32, _timestamp as u64)?; + remote_dma::playback(io, aux_mutex, ddma_mutex, subkernel_mutex, routing_table, _id as u32, _timestamp as u64)?; kern_acknowledge() } &kern::DmaAwaitRemoteRequest { id: _id } => { @@ -633,7 +633,7 @@ fn process_kern_message(io: &Io, aux_mutex: &Mutex, session.kernel_state = KernelState::Absent; unsafe { session.congress.cache.unborrow() } #[cfg(has_drtio)] - subkernel::clear_subkernels(io, _subkernel_mutex)?; + subkernel::clear_subkernels(io, subkernel_mutex)?; match stream { None => return Ok(true), @@ -652,7 +652,7 @@ fn process_kern_message(io: &Io, aux_mutex: &Mutex, session.kernel_state = KernelState::Absent; unsafe { session.congress.cache.unborrow() } #[cfg(has_drtio)] - subkernel::clear_subkernels(io, _subkernel_mutex)?; + subkernel::clear_subkernels(io, subkernel_mutex)?; match stream { None => { @@ -675,7 +675,7 @@ fn process_kern_message(io: &Io, aux_mutex: &Mutex, #[cfg(has_drtio)] &kern::SubkernelLoadRunRequest { id, destination: _, run } => { let succeeded = match subkernel::load( - io, aux_mutex, _subkernel_mutex, routing_table, id, run) { + io, aux_mutex, ddma_mutex, subkernel_mutex, routing_table, id, run) { Ok(()) => true, Err(e) => { error!("Error loading subkernel: {}", e); false } }; @@ -683,7 +683,7 @@ fn process_kern_message(io: &Io, aux_mutex: &Mutex, } #[cfg(has_drtio)] &kern::SubkernelAwaitFinishRequest{ id, timeout } => { - let res = subkernel::await_finish(io, aux_mutex, _subkernel_mutex, routing_table, + let res = subkernel::await_finish(io, aux_mutex, ddma_mutex, subkernel_mutex, routing_table, id, timeout); let status = match res { Ok(ref res) => { @@ -705,18 +705,18 @@ fn process_kern_message(io: &Io, aux_mutex: &Mutex, } #[cfg(has_drtio)] &kern::SubkernelMsgSend { id, destination, count, tag, data } => { - subkernel::message_send(io, aux_mutex, _subkernel_mutex, routing_table, id, destination, count, tag, data)?; + subkernel::message_send(io, aux_mutex, ddma_mutex, subkernel_mutex, routing_table, id, destination, count, tag, data)?; kern_acknowledge() } #[cfg(has_drtio)] &kern::SubkernelMsgRecvRequest { id, timeout, tags } => { - let message_received = subkernel::message_await(io, _subkernel_mutex, id as u32, timeout); + let message_received = subkernel::message_await(io, subkernel_mutex, id as u32, timeout); let (status, count) = match message_received { Ok(ref message) => (kern::SubkernelStatus::NoError, message.count), Err(SubkernelError::Timeout) => (kern::SubkernelStatus::Timeout, 0), Err(SubkernelError::IncorrectState) => (kern::SubkernelStatus::IncorrectState, 0), Err(SubkernelError::SubkernelFinished) => { - let res = subkernel::retrieve_finish_status(io, aux_mutex, _subkernel_mutex, + let res = subkernel::retrieve_finish_status(io, aux_mutex, ddma_mutex, subkernel_mutex, routing_table, id as u32)?; if res.comm_lost { (kern::SubkernelStatus::CommLost, 0) @@ -847,7 +847,7 @@ fn flash_kernel_worker(io: &Io, aux_mutex: &Mutex, match result { Ok(kernel) => { // process .ELF or .TAR kernels - let res = process_flash_kernel(io, aux_mutex, subkernel_mutex, routing_table, up_destinations, &mut session, kernel); + let res = process_flash_kernel(io, aux_mutex, subkernel_mutex, ddma_mutex, routing_table, up_destinations, &mut session, kernel); #[cfg(has_drtio)] match res { // wait to establish the DRTIO connection