enable network and mgmt during Rust panic, make RTIO PLL lock failure a panic

Closes #198 #200

Making it a soft panic makes it more involved with a bit of code duplication - setting up mgmt requires setting up the interface and sockets. Maybe can be done a bit cleaner.

```
[spaqin@hera:~/m-labs/artiq-zynq]$ artiq_sinara_tester
****** Sinara system tester ******
[...]
ConnectionRefusedError: [Errno 111] Connection refused

[spaqin@hera:~/m-labs/artiq-zynq]$ artiq_coremgmt -D 192.168.1.56 log
[     0.000067s]  INFO(runtime): NAR3/Zynq7000 starting...
[     0.005238s]  INFO(runtime): detected gateware: GenericMaster
[     0.016152s]  INFO(libboard_zynq::i2c): PCA9548 detected
[     0.023004s]  WARN(runtime): config initialization failed: SD error: Card initialization error: No card inserted, check if the card is inserted properly.
[     0.036730s]  WARN(runtime::rtio_clocking): error reading configuration. Falling back to default.
[     0.213000s] ERROR(runtime::rtio_clocking): RTIO PLL failed to lock
[     0.224443s]  INFO(libboard_zynq::i2c): PCA9548 detected
[     0.256197s]  INFO(runtime::comms): network addresses: MAC=e8-eb-1b-13-49-8b IPv4=192.168.1.56 IPv6-LL=fe80::eaeb:1bff:fe13:498b IPv6: no configured address
[     0.270183s] ERROR(runtime::comms): There has been an error configuring the device: RTIO PLL failed to lock. Only mgmt interface will be available.
[     4.000095s]  INFO(libboard_zynq::eth): eth: got Link { speed: S1000, duplex: Full }
[    33.148521s]  INFO(runtime::mgmt): received connection
```

Reviewed-on: #199
Co-authored-by: mwojcik <mw@m-labs.hk>
Co-committed-by: mwojcik <mw@m-labs.hk>

src/runtime/src/comms.rs

@@ -22,6 +22,8 @@ use futures::{select_biased, future::FutureExt};
 use libasync::{smoltcp::{Sockets, TcpStream}, task};
 use libconfig::{Config, net_settings};
 use libboard_artiq::drtio_routing;
+#[cfg(feature = "target_kasli_soc")]
+use libboard_zynq::error_led::ErrorLED;
 
 use crate::proto_async::*;
 use crate::kernel;
@@ -490,3 +492,61 @@ pub fn main(timer: GlobalTimer, cfg: Config) {
         Instant::from_millis(timer.get_time().0 as i32)
     });
 }
+
+
+pub fn soft_panic_main(timer: GlobalTimer, cfg: Config) -> ! {
+
+    let net_addresses = net_settings::get_addresses(&cfg);
+    info!("network addresses: {}", net_addresses);
+
+    let eth = zynq::eth::Eth::eth0(net_addresses.hardware_addr.0.clone());
+    const RX_LEN: usize = 64;
+    // Number of transmission buffers (minimum is two because with
+    // one, duplicate packet transmission occurs)
+    const TX_LEN: usize = 64;
+    let eth = eth.start_rx(RX_LEN);
+    let mut eth = eth.start_tx(TX_LEN);
+
+    let neighbor_cache = NeighborCache::new(alloc::collections::BTreeMap::new());
+    let mut iface = match net_addresses.ipv6_addr {
+        Some(addr) => {
+            let ip_addrs = [
+                IpCidr::new(net_addresses.ipv4_addr, 0),
+                IpCidr::new(net_addresses.ipv6_ll_addr, 0),
+                IpCidr::new(addr, 0)
+            ];
+            EthernetInterfaceBuilder::new(&mut eth)
+                       .ethernet_addr(net_addresses.hardware_addr)
+                       .ip_addrs(ip_addrs)
+                       .neighbor_cache(neighbor_cache)
+                       .finalize()
+        }
+        None => {
+            let ip_addrs = [
+                IpCidr::new(net_addresses.ipv4_addr, 0),
+                IpCidr::new(net_addresses.ipv6_ll_addr, 0)
+            ];
+            EthernetInterfaceBuilder::new(&mut eth)
+                       .ethernet_addr(net_addresses.hardware_addr)
+                       .ip_addrs(ip_addrs)
+                       .neighbor_cache(neighbor_cache)
+                       .finalize()
+        }
+    };
+
+    Sockets::init(32);
+
+    mgmt::start(cfg);
+
+    // getting eth settings disables the LED as it resets GPIO
+    // need to re-enable it here
+    #[cfg(feature = "target_kasli_soc")]
+    {
+        let mut err_led = ErrorLED::error_led();
+        err_led.toggle(true);
+    }
+
+    Sockets::run(&mut iface, || {
+        Instant::from_millis(timer.get_time().0 as i32)
+    });
+}

src/runtime/src/kernel/core1.rs

@@ -222,12 +222,15 @@ extern fn dl_unwind_find_exidx(pc: *const u32, len_ptr: *mut u32) -> *const u32
         if &__text_start as *const u32 <= pc && pc < &__text_end as *const u32 {
             length = (&__exidx_end as *const EXIDX_Entry).offset_from(&__exidx_start) as u32;
             start = &__exidx_start;
-        } else {
+        } else if KERNEL_IMAGE != ptr::null() {
             let exidx = KERNEL_IMAGE.as_ref()
                 .expect("dl_unwind_find_exidx kernel image")
                 .library.get().as_ref().unwrap().exidx();
             length = exidx.len() as u32;
             start = exidx.as_ptr();
+        } else {
+            length = 0;
+            start = ptr::null();
         }
         *len_ptr = length;
     }

src/runtime/src/panic.rs

@@ -3,20 +3,21 @@ use libregister::RegisterR;
 use libcortex_a9::regs::MPIDR;
 use unwind::backtrace;
 
+#[cfg(feature = "target_kasli_soc")]
+use libboard_zynq::error_led::ErrorLED;
+use crate::comms::soft_panic_main;
+use log::error;
+use libboard_zynq::timer::GlobalTimer;
+use libconfig::Config;
+
 static mut PANICKED: [bool; 2] = [false; 2];
+static mut SOFT_PANICKED: bool = false;
 
 #[panic_handler]
 fn panic(info: &core::panic::PanicInfo) -> ! {
     let id = MPIDR.read().cpu_id() as usize;
-    print!("Core {} ", id);
+    let soft_panicked = unsafe { SOFT_PANICKED };
-    unsafe {
+    print!("Core {} panic at ", id);
-        if PANICKED[id] {
-            println!("nested panic!");
-            loop {}
-        }
-        PANICKED[id] = true;
-    }
-    print!("panic at ");
     if let Some(location) = info.location() {
         print!("{}:{}:{}", location.file(), location.line(), location.column());
     } else {
@@ -27,6 +28,20 @@ fn panic(info: &core::panic::PanicInfo) -> ! {
     } else {
         println!("");
     }
+    unsafe {
+        // soft panics only allowed for core 0
+        if PANICKED[id] && (SOFT_PANICKED || id == 1) {
+            println!("nested panic!");
+            loop {}
+        }
+        SOFT_PANICKED = true;
+        PANICKED[id] = true;
+    }
+    #[cfg(feature = "target_kasli_soc")]
+    {
+        let mut err_led = ErrorLED::error_led();
+        err_led.toggle(true);
+    }
     println!("Backtrace: ");
     let _ = backtrace(|ip| {
         // Backtrace gives us the return address, i.e. the address after the delay slot,
@@ -34,6 +49,28 @@ fn panic(info: &core::panic::PanicInfo) -> ! {
         print!("{:#08x} ", ip - 2 * 4);
     });
     println!("\nEnd backtrace");
-
+    if !soft_panicked && id == 0 {
+        soft_panic(info);
+    }
     loop {}
 }
+
+fn soft_panic(info: &core::panic::PanicInfo) -> ! {
+    // write panic info to log, so coremgmt can also read it
+    if let Some(location) = info.location() {
+        error!("panic at {}:{}:{}", location.file(), location.line(), location.column());
+    } else {
+        error!("panic at unknown location");
+    }
+    if let Some(message) = info.message() {
+        error!("panic message: {}", message);
+    }
+    let timer = GlobalTimer::start();
+    let cfg = match Config::new() {
+        Ok(cfg) => cfg,
+        Err(_) => {
+            Config::new_dummy()
+        }
+    };
+    soft_panic_main(timer, cfg);
+}

src/runtime/src/rtio_clocking.rs

@@ -1,4 +1,4 @@
-use log::{info, warn, error};
+use log::{info, warn};
 use libboard_zynq::timer::GlobalTimer;
 use embedded_hal::blocking::delay::DelayMs;
 use libconfig::Config;
@@ -95,7 +95,7 @@ fn init_rtio(timer: &mut GlobalTimer, _clk: RtioClock) {
     if locked {
         info!("RTIO PLL locked");
     } else {
-        error!("RTIO PLL failed to lock");
+        panic!("RTIO PLL failed to lock");
     }
 
     unsafe {