Soft panic for RTIO PLL reasons #199

Merged
sb10q merged 16 commits from mwojcik/artiq-zynq:pll_error into master 2022-10-21 17:56:34 +08:00
3 changed files with 57 additions and 22 deletions
Showing only changes of commit 3ba6555bb0 - Show all commits

View File

@ -17,13 +17,13 @@ use libboard_zynq::{
}, },
timer::GlobalTimer, timer::GlobalTimer,
}; };
#[cfg(feature = "target_kasli_soc")]
use libboard_zynq::error_led::ErrorLED;
use libcortex_a9::{semaphore::Semaphore, mutex::Mutex, sync_channel::{Sender, Receiver}}; use libcortex_a9::{semaphore::Semaphore, mutex::Mutex, sync_channel::{Sender, Receiver}};
use futures::{select_biased, future::FutureExt}; use futures::{select_biased, future::FutureExt};
use libasync::{smoltcp::{Sockets, TcpStream}, task}; use libasync::{smoltcp::{Sockets, TcpStream}, task};
use libconfig::{Config, net_settings}; use libconfig::{Config, net_settings};
use libboard_artiq::drtio_routing; use libboard_artiq::drtio_routing;
#[cfg(feature = "target_kasli_soc")]
use libboard_zynq::error_led::ErrorLED;
use crate::proto_async::*; use crate::proto_async::*;
use crate::kernel; use crate::kernel;
@ -86,6 +86,7 @@ enum Reply {
static CACHE_STORE: Mutex<BTreeMap<String, Vec<i32>>> = Mutex::new(BTreeMap::new()); static CACHE_STORE: Mutex<BTreeMap<String, Vec<i32>>> = Mutex::new(BTreeMap::new());
static DMA_RECORD_STORE: Mutex<BTreeMap<String, (Vec<u8>, i64)>> = Mutex::new(BTreeMap::new()); static DMA_RECORD_STORE: Mutex<BTreeMap<String, (Vec<u8>, i64)>> = Mutex::new(BTreeMap::new());
static mut MGMT_STARTED: bool = false;
Outdated
Review

Where is this used?

Where is this used?
async fn write_header(stream: &TcpStream, reply: Reply) -> Result<()> { async fn write_header(stream: &TcpStream, reply: Reply) -> Result<()> {
stream.send_slice(&[0x5a, 0x5a, 0x5a, 0x5a, reply.to_u8().unwrap()]).await?; stream.send_slice(&[0x5a, 0x5a, 0x5a, 0x5a, reply.to_u8().unwrap()]).await?;
@ -442,6 +443,9 @@ pub fn main(timer: GlobalTimer, cfg: Config) {
} }
mgmt::start(cfg); mgmt::start(cfg);
unsafe {
MGMT_STARTED = true;
}
task::spawn(async move { task::spawn(async move {
let connection = Rc::new(Semaphore::new(1, 1)); let connection = Rc::new(Semaphore::new(1, 1));
@ -494,17 +498,11 @@ pub fn main(timer: GlobalTimer, cfg: Config) {
} }
pub fn soft_panic_setup(timer: GlobalTimer, cfg: Config, reason: &'static str) { pub fn soft_panic_main(timer: GlobalTimer, cfg: Config) -> ! {
let net_addresses = net_settings::get_addresses(&cfg); let net_addresses = net_settings::get_addresses(&cfg);
info!("network addresses: {}", net_addresses); info!("network addresses: {}", net_addresses);
error!("There has been an error configuring the device: {}. Only mgmt interface will be available.", reason);
#[cfg(feature = "target_kasli_soc")]
{
let mut err_led = ErrorLED::error_led();
err_led.toggle(true);
}
let eth = zynq::eth::Eth::eth0(net_addresses.hardware_addr.0.clone()); let eth = zynq::eth::Eth::eth0(net_addresses.hardware_addr.0.clone());
const RX_LEN: usize = 64; const RX_LEN: usize = 64;
// Number of transmission buffers (minimum is two because with // Number of transmission buffers (minimum is two because with
@ -544,6 +542,13 @@ pub fn soft_panic_setup(timer: GlobalTimer, cfg: Config, reason: &'static str) {
mgmt::start(cfg); mgmt::start(cfg);
//getting eth settings disables the LED - need enable it here
Outdated
Review

Why does it disable the LED?

Why does it disable the LED?

getting eth settings sets up I2C to communicate with the ethernet controller;

I2C setup also resets GPIO:
https://git.m-labs.hk/M-Labs/zynq-rs/src/branch/master/libboard_zynq/src/i2c/mod.rs#L57

and I2C GPIO (MIO 33, 50, 51) is in the same bank as the error led (MIO 37); so resetting the entire port again causes the LED to go off.

getting eth settings sets up I2C to communicate with the ethernet controller; I2C setup also resets GPIO: https://git.m-labs.hk/M-Labs/zynq-rs/src/branch/master/libboard_zynq/src/i2c/mod.rs#L57 and I2C GPIO (MIO 33, 50, 51) is in the same bank as the error led (MIO 37); so resetting the entire port again causes the LED to go off.
Outdated
Review

Okay I see. Maybe make the comment a bit more clear (e.g. "we reinitialized the GPIO controller, so only now can we turn on the error LED")
Also, the non-mgmt panic handler should also turn on the LED to make sure.

Okay I see. Maybe make the comment a bit more clear (e.g. "we reinitialized the GPIO controller, so only now can we turn on the error LED") Also, the non-mgmt panic handler should also turn on the LED to make sure.
#[cfg(feature = "target_kasli_soc")]
{
let mut err_led = ErrorLED::error_led();
err_led.toggle(true);
}
Sockets::run(&mut iface, || { Sockets::run(&mut iface, || {
Instant::from_millis(timer.get_time().0 as i32) Instant::from_millis(timer.get_time().0 as i32)
}); });

View File

@ -122,8 +122,8 @@ pub fn main_core0() {
}; };
task::spawn(report_async_rtio_errors()); task::spawn(report_async_rtio_errors());
match rtio_clocking::init(&mut timer, &cfg) { rtio_clocking::init(&mut timer, &cfg).expect("Could not set up RTIO PLL");
Ok(()) => { comms::main(timer, cfg); }
Err(reason) => { comms::soft_panic_setup(timer, cfg, reason); } comms::main(timer, cfg);
};
} }

View File

@ -3,19 +3,26 @@ use libregister::RegisterR;
use libcortex_a9::regs::MPIDR; use libcortex_a9::regs::MPIDR;
use unwind::backtrace; use unwind::backtrace;
#[cfg(feature = "target_kasli_soc")]
use libboard_zynq::error_led::ErrorLED;
use crate::comms::soft_panic_main;
use libboard_zynq::timer::GlobalTimer;
use libconfig::Config;
static mut PANICKED: [bool; 2] = [false; 2]; static mut PANICKED: [bool; 2] = [false; 2];
static mut SOFT_PANICKED: bool = false;
#[panic_handler] #[panic_handler]
fn panic(info: &core::panic::PanicInfo) -> ! { fn panic(info: &core::panic::PanicInfo) -> ! {
let id = MPIDR.read().cpu_id() as usize; let id = MPIDR.read().cpu_id() as usize;
print!("Core {} ", id); print!("Core {} ", id);
unsafe { #[cfg(feature = "target_kasli_soc")]
if PANICKED[id] { {
println!("nested panic!"); let mut err_led = ErrorLED::error_led();
loop {} err_led.toggle(true);
}
PANICKED[id] = true;
} }
let soft_panicked = unsafe { SOFT_PANICKED };
print!("panic at "); print!("panic at ");
if let Some(location) = info.location() { if let Some(location) = info.location() {
print!("{}:{}:{}", location.file(), location.line(), location.column()); print!("{}:{}:{}", location.file(), location.line(), location.column());
@ -27,6 +34,18 @@ fn panic(info: &core::panic::PanicInfo) -> ! {
} else { } else {
println!(""); println!("");
} }
unsafe {
if PANICKED[id] && (SOFT_PANICKED || id == 1) {
println!("nested panic!");
loop {}
}
SOFT_PANICKED = true;
PANICKED[id] = true;
}
if !soft_panicked && id == 0 {
soft_panic();
}
println!("Backtrace: "); println!("Backtrace: ");
let _ = backtrace(|ip| { let _ = backtrace(|ip| {
// Backtrace gives us the return address, i.e. the address after the delay slot, // Backtrace gives us the return address, i.e. the address after the delay slot,
@ -34,6 +53,17 @@ fn panic(info: &core::panic::PanicInfo) -> ! {
print!("{:#08x} ", ip - 2 * 4); print!("{:#08x} ", ip - 2 * 4);
}); });
println!("\nEnd backtrace"); println!("\nEnd backtrace");
loop {} loop {}
} }
fn soft_panic() -> ! {
Outdated
Review

Had to read that twice...

Maybe reformulate along the lines:
"Write panic info to the log, so that coremgmt can see it"

Had to read that twice... Maybe reformulate along the lines: "Write panic info to the log, so that coremgmt can see it"
let timer = GlobalTimer::start();
let cfg = match Config::new() {
Ok(cfg) => cfg,
Err(_) => {
Config::new_dummy()
}
};
soft_panic_main(timer, cfg);
}