add support for tar flashable (sub)kernels

- for example, current_clock csr can be connected to tx_init.done

flake.lock

@@ -11,11 +11,11 @@
         "src-pythonparser": "src-pythonparser"
       },
       "locked": {
-        "lastModified": 1697537883,
-        "narHash": "sha256-GfadmYHFkczltX+rPf08YpAHjYa/31ZmmVD578BcFow=",
+        "lastModified": 1699353408,
+        "narHash": "sha256-mJZ9PoRsOvN5gtrEl9ogj/+GRCK8O3tZbcyfxKvGG20=",
         "ref": "refs/heads/master",
-        "rev": "b168f0bb4be1697ff100475c20ee304dcc31fcc2",
-        "revCount": 8573,
+        "rev": "ec2b86b08d497a41e75c228385b754c6f3a4a8bf",
+        "revCount": 8585,
         "type": "git",
         "url": "https://github.com/m-labs/artiq.git"
       },
@@ -118,11 +118,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1697226376,
-        "narHash": "sha256-cumLLb1QOUtWieUnLGqo+ylNt3+fU8Lcv5Zl+tYbRUE=",
+        "lastModified": 1699169573,
+        "narHash": "sha256-cvUb1xZkvOp3W2SzylStrTirhVd9zCeo5utJl9nSIhw=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "898cb2064b6e98b8c5499f37e81adbdf2925f7c5",
+        "rev": "aeefe2054617cae501809b82b44a8e8f7be7cc4b",
         "type": "github"
       },
       "original": {
@@ -163,11 +163,11 @@
     "src-migen": {
       "flake": false,
       "locked": {
-        "lastModified": 1697013661,
-        "narHash": "sha256-qNCqgWyE4vTDmyjE2XMJqW1djuBxT25A36AzQfZqluU=",
+        "lastModified": 1699335478,
+        "narHash": "sha256-BsubN4Mfdj02QPK6ZCrl+YOaSg7DaLQdSCVP49ztWik=",
         "owner": "m-labs",
         "repo": "migen",
-        "rev": "aadc19df93b7aa9ca761aaebbb98a11e0cf2d7c7",
+        "rev": "fd0bf5855a1367eab14b0d6f7f8266178e25d78e",
         "type": "github"
       },
       "original": {
@@ -179,11 +179,11 @@
     "src-misoc": {
       "flake": false,
       "locked": {
-        "lastModified": 1693709836,
-        "narHash": "sha256-YiCk05RYLzZu1CYkQ2r7XtjwVEqkUGTQn388uOls9tI=",
+        "lastModified": 1699352904,
+        "narHash": "sha256-SglyTmXOPv8jJOjwAjJrj/WhAkItQfUbvKfUqrynwRg=",
         "ref": "refs/heads/master",
-        "rev": "58dc4ee60d165ce9145cf3d904241fc154b6407f",
-        "revCount": 2448,
+        "rev": "a53859f2167c31ab5225b6c09f30cf05527b94f4",
+        "revCount": 2452,
         "submodules": true,
         "type": "git",
         "url": "https://github.com/m-labs/misoc.git"

flake.nix

@@ -122,6 +122,9 @@
         src = ./src;
         cargoLock = { 
           lockFile = src/Cargo.lock;
+          outputHashes = {
+              "tar-no-std-0.1.8" = "sha256-xm17108v4smXOqxdLvHl9CxTCJslmeogjm4Y87IXFuM=";
+            };
         };
 
         nativeBuildInputs = [

src/Cargo.lock

@@ -2,6 +2,12 @@
 # It is not intended for manual editing.
 version = 3
 
+[[package]]
+name = "arrayvec"
+version = "0.7.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711"
+
 [[package]]
 name = "async-recursion"
 version = "0.3.2"
@@ -474,6 +480,7 @@ dependencies = [
  "log_buffer",
  "num-derive",
  "num-traits",
+ "tar-no-std",
  "unwind",
  "vcell",
  "void",
@@ -538,6 +545,16 @@ dependencies = [
  "unicode-ident",
 ]
 
+[[package]]
+name = "tar-no-std"
+version = "0.1.8"
+source = "git+https://github.com/Spaqin/tar-no-std?rev=2ab6dc5#2ab6dc58e5249c59c4eb03eaf3a119bcdd678d32"
+dependencies = [
+ "arrayvec",
+ "bitflags",
+ "log",
+]
+
 [[package]]
 name = "unicode-ident"
 version = "1.0.5"

src/gateware/kasli_soc.py

@@ -237,12 +237,16 @@ class GenericMaster(SoCCore):
         gtx0 = self.gt_drtio.gtxs[0]
         self.specials += Instance("BUFG", i_I=gtx0.txoutclk, o_O=txout_buf)
 
+        ext_async_rst = Signal()
+
         self.submodules.bootstrap = GTPBootstrapClock(self.platform, clk_freq)
         self.submodules.sys_crg = zynq_clocking.SYSCRG(
             self.platform,
             self.ps7,
             txout_buf,
-            clk_sw=gtx0.tx_init.done)
+            clk_sw=self.gt_drtio.stable_clkin.storage,
+            clk_sw_status=gtx0.tx_init.done,
+            ext_async_rst=ext_async_rst)
         self.csr_devices.append("sys_crg")
         self.crg = self.ps7 # HACK for eem_7series to find the clock
         self.crg.cd_sys = self.sys_crg.cd_sys
@@ -250,6 +254,9 @@ class GenericMaster(SoCCore):
             self.bootstrap.cd_bootstrap.clk, self.sys_crg.cd_sys.clk)
         fix_serdes_timing_path(platform)
 
+        self.comb += ext_async_rst.eq(self.sys_crg.clk_sw_fsm.o_clk_sw & ~gtx0.tx_init.done)
+        self.specials += MultiReg(self.sys_crg.clk_sw_fsm.o_clk_sw & self.sys_crg.mmcm_locked, self.gt_drtio.clk_path_ready, odomain="bootstrap")
+
         self.config["HAS_SI5324"] = None
         self.config["SI5324_SOFT_RESET"] = None
 
@@ -419,12 +426,16 @@ class GenericSatellite(SoCCore):
         gtx0 = self.gt_drtio.gtxs[0]
         self.specials += Instance("BUFG", i_I=gtx0.txoutclk, o_O=txout_buf)
 
+        ext_async_rst = Signal()
+
         self.submodules.bootstrap = GTPBootstrapClock(self.platform, clk_freq)
         self.submodules.sys_crg = zynq_clocking.SYSCRG(
             self.platform, 
             self.ps7,
             txout_buf,
-            clk_sw=gtx0.tx_init.done)
+            clk_sw=self.gt_drtio.stable_clkin.storage,
+            clk_sw_status=gtx0.tx_init.done,
+            ext_async_rst=ext_async_rst)
         platform.add_false_path_constraints(
             self.bootstrap.cd_bootstrap.clk, self.sys_crg.cd_sys.clk)
         self.csr_devices.append("sys_crg")
@@ -433,6 +444,9 @@ class GenericSatellite(SoCCore):
 
         fix_serdes_timing_path(platform)
 
+        self.comb += ext_async_rst.eq(self.sys_crg.clk_sw_fsm.o_clk_sw & ~gtx0.tx_init.done)
+        self.specials += MultiReg(self.sys_crg.clk_sw_fsm.o_clk_sw & self.sys_crg.mmcm_locked, self.gt_drtio.clk_path_ready, odomain="bootstrap")
+
         self.rtio_channels = []
         has_grabber = any(peripheral["type"] == "grabber" for peripheral in description["peripherals"])
         if has_grabber:

src/gateware/zc706.py

@@ -226,6 +226,7 @@ class _MasterBase(SoCCore):
         self.csr_devices.append("gt_drtio")
 
         self.submodules.rtio_tsc = rtio.TSC(glbl_fine_ts_width=3)
+        ext_async_rst = Signal()
         txout_buf = Signal()
         gtx0 = self.gt_drtio.gtxs[0]
         self.specials += Instance("BUFG", i_I=gtx0.txoutclk, o_O=txout_buf)
@@ -234,12 +235,17 @@ class _MasterBase(SoCCore):
             self.platform, 
             self.ps7, 
             txout_buf,
-            clk_sw=gtx0.tx_init.done,
+            clk_sw=self.gt_drtio.stable_clkin.storage,
+            clk_sw_status=gtx0.tx_init.done,
+            ext_async_rst=ext_async_rst,
             freq=clk_freq)
         platform.add_false_path_constraints(
             self.bootstrap.cd_bootstrap.clk, self.sys_crg.cd_sys.clk)
         self.csr_devices.append("sys_crg")
 
+        self.comb += ext_async_rst.eq(self.sys_crg.clk_sw_fsm.o_clk_sw & ~gtx0.tx_init.done)
+        self.specials += MultiReg(self.sys_crg.clk_sw_fsm.o_clk_sw & self.sys_crg.mmcm_locked, self.gt_drtio.clk_path_ready, odomain="bootstrap")
+
         drtio_csr_group = []
         drtioaux_csr_group = []
         drtioaux_memory_group = []
@@ -361,6 +367,7 @@ class _SatelliteBase(SoCCore):
             clk_freq=clk_freq)
         self.csr_devices.append("gt_drtio")
 
+        ext_async_rst = Signal()
         txout_buf = Signal()
         txout_buf.attr.add("keep")
         gtx0 = self.gt_drtio.gtxs[0]
@@ -373,12 +380,17 @@ class _SatelliteBase(SoCCore):
             self.platform, 
             self.ps7, 
             txout_buf,
-            clk_sw=gtx0.tx_init.done,
+            clk_sw=self.gt_drtio.stable_clkin.storage,
+            clk_sw_status=gtx0.tx_init.done,
+            ext_async_rst=ext_async_rst,
             freq=clk_freq)
         platform.add_false_path_constraints(
             self.bootstrap.cd_bootstrap.clk, self.sys_crg.cd_sys.clk)
         self.csr_devices.append("sys_crg")
 
+        self.comb += ext_async_rst.eq(self.sys_crg.clk_sw_fsm.o_clk_sw & ~gtx0.tx_init.done)
+        self.specials += MultiReg(self.sys_crg.clk_sw_fsm.o_clk_sw & self.sys_crg.mmcm_locked, self.gt_drtio.clk_path_ready, odomain="bootstrap")
+
         drtioaux_csr_group = []
         drtioaux_memory_group = []
         drtiorep_csr_group = []

src/gateware/zynq_clocking.py

@@ -65,7 +65,7 @@ class ClockSwitchFSM(Module):
 
 
 class SYSCRG(Module, AutoCSR):
-    def __init__(self, platform, ps7, main_clk, clk_sw=None, freq=125e6):
+    def __init__(self, platform, ps7, main_clk, clk_sw=None, clk_sw_status=None, freq=125e6, ext_async_rst=None, ):
         # assumes bootstrap clock is same freq as main and sys output
         self.clock_domains.cd_sys = ClockDomain()
         self.clock_domains.cd_sys4x = ClockDomain(reset_less=True)
@@ -88,7 +88,7 @@ class SYSCRG(Module, AutoCSR):
         else:
             self.comb += self.clk_sw_fsm.i_clk_sw.eq(clk_sw)
 
-        mmcm_locked = Signal()
+        self.mmcm_locked = Signal()
         mmcm_sys = Signal()
         mmcm_sys4x = Signal()
         mmcm_sys5x = Signal()
@@ -96,7 +96,7 @@ class SYSCRG(Module, AutoCSR):
         mmcm_fb_clk = Signal()
         self.specials += [
             Instance("MMCME2_ADV",
-                p_STARTUP_WAIT="FALSE", o_LOCKED=mmcm_locked,
+                p_STARTUP_WAIT="FALSE", o_LOCKED=self.mmcm_locked,
                 p_BANDWIDTH="HIGH",
                 p_REF_JITTER1=0.001,
                 p_CLKIN1_PERIOD=period, i_CLKIN1=main_clk,
@@ -125,10 +125,19 @@ class SYSCRG(Module, AutoCSR):
             Instance("BUFG", i_I=mmcm_sys, o_O=self.cd_sys.clk),
             Instance("BUFG", i_I=mmcm_sys4x, o_O=self.cd_sys4x.clk),
             Instance("BUFG", i_I=mmcm_clk208, o_O=self.cd_clk200.clk),
-            AsyncResetSynchronizer(self.cd_sys, ~mmcm_locked),
-            AsyncResetSynchronizer(self.cd_clk200, ~mmcm_locked),
         ]
 
+        if ext_async_rst is not None:
+            self.specials += [
+                AsyncResetSynchronizer(self.cd_sys, ~self.mmcm_locked | ext_async_rst),
+                AsyncResetSynchronizer(self.cd_clk200, ~self.mmcm_locked | ext_async_rst),
+            ]
+        else:
+            self.specials += [
+                AsyncResetSynchronizer(self.cd_sys, ~self.mmcm_locked),
+                AsyncResetSynchronizer(self.cd_clk200, ~self.mmcm_locked),
+            ]
+
         reset_counter = Signal(4, reset=15)
         ic_reset = Signal(reset=1)
         self.sync.clk200 += \
@@ -139,4 +148,7 @@ class SYSCRG(Module, AutoCSR):
             )
         self.specials += Instance("IDELAYCTRL", i_REFCLK=ClockSignal("clk200"), i_RST=ic_reset)
 
-        self.comb += self.current_clock.status.eq(self.clk_sw_fsm.o_clk_sw)
+        if clk_sw_status is None:
+            self.comb += self.current_clock.status.eq(self.clk_sw_fsm.o_clk_sw)
+        else:
+            self.comb += self.current_clock.status.eq(clk_sw_status)

src/runtime/Cargo.toml.tpl

@@ -40,4 +40,8 @@ unwind = { path = "../libunwind" }
 libc = { path = "../libc" }
 io = { path = "../libio", features = ["alloc"] }
 ksupport = { path = "../libksupport" }
-libboard_artiq = { path = "../libboard_artiq" }
+libboard_artiq = { path = "../libboard_artiq" }
+
+[dependencies.tar-no-std]
+git = "https://github.com/Spaqin/tar-no-std"
+rev = "2ab6dc5"

src/runtime/src/comms.rs

@@ -3,17 +3,23 @@ use core::{cell::RefCell, fmt, slice, str};
 
 use core_io::Error as IoError;
 use cslice::CSlice;
+#[cfg(has_drtio)]
+use dyld::elf;
 use futures::{future::FutureExt, select_biased};
 #[cfg(has_drtio)]
 use io::Cursor;
 #[cfg(has_drtio)]
 use ksupport::rpc;
 use ksupport::{kernel, resolve_channel_name};
+#[cfg(has_drtio)]
+use libasync::delay;
 use libasync::{smoltcp::{Sockets, TcpStream},
                task};
 use libboard_artiq::drtio_routing;
 #[cfg(feature = "target_kasli_soc")]
 use libboard_zynq::error_led::ErrorLED;
+#[cfg(has_drtio)]
+use libboard_zynq::time::Milliseconds;
 use libboard_zynq::{self as zynq,
                     smoltcp::{self,
                               iface::{EthernetInterfaceBuilder, NeighborCache},
@@ -27,6 +33,8 @@ use libcortex_a9::{mutex::Mutex,
 use log::{error, info, warn};
 use num_derive::{FromPrimitive, ToPrimitive};
 use num_traits::{FromPrimitive, ToPrimitive};
+#[cfg(has_drtio)]
+use tar_no_std::TarArchiveRef;
 
 #[cfg(has_drtio)]
 use crate::pl;
@@ -43,6 +51,8 @@ pub enum Error {
     BufferExhausted,
     #[cfg(has_drtio)]
     SubkernelError(subkernel::Error),
+    #[cfg(has_drtio)]
+    DestinationDown,
 }
 
 pub type Result<T> = core::result::Result<T, Error>;
@@ -57,6 +67,8 @@ impl fmt::Display for Error {
             Error::BufferExhausted => write!(f, "buffer exhausted"),
             #[cfg(has_drtio)]
             Error::SubkernelError(error) => write!(f, "subkernel error: {:?}", error),
+            #[cfg(has_drtio)]
+            Error::DestinationDown => write!(f, "subkernel destination down"),
         }
     }
 }
@@ -538,6 +550,56 @@ async fn handle_run_kernel(
     Ok(())
 }
 
+async fn handle_flash_kernel(
+    buffer: &Vec<u8>,
+    control: &Rc<RefCell<kernel::Control>>,
+    _up_destinations: &Rc<RefCell<[bool; drtio_routing::DEST_COUNT]>>,
+    _aux_mutex: &Rc<Mutex<bool>>,
+    _routing_table: &drtio_routing::RoutingTable,
+    _timer: GlobalTimer,
+) -> Result<()> {
+    if buffer[0] == elf::ELFMAG0 && buffer[1] == elf::ELFMAG1 && buffer[2] == elf::ELFMAG2 && buffer[3] == elf::ELFMAG3
+    {
+        // assume ELF file, proceed as before
+        load_kernel(buffer, control, None).await
+    } else {
+        #[cfg(has_drtio)]
+        {
+            let archive = TarArchiveRef::new(buffer.as_ref());
+            let entries = archive.entries();
+            let mut main_lib: Vec<u8> = Vec::new();
+            for entry in entries {
+                if entry.filename().as_str() == "main.elf" {
+                    main_lib = entry.data().to_vec();
+                } else {
+                    // subkernel filename must be in format:
+                    // "<subkernel id> <destination>.elf"
+                    let filename = entry.filename();
+                    let mut iter = filename.as_str().split_whitespace();
+                    let sid: u32 = iter.next().unwrap().parse().unwrap();
+                    let dest: u8 = iter.next().unwrap().strip_suffix(".elf").unwrap().parse().unwrap();
+                    let up = _up_destinations.borrow()[dest as usize];
+                    if up {
+                        let subkernel_lib = entry.data().to_vec();
+                        subkernel::add_subkernel(sid, dest, subkernel_lib).await;
+                        match subkernel::upload(_aux_mutex, _routing_table, _timer, sid).await {
+                            Ok(_) => (),
+                            Err(_) => return Err(Error::UnexpectedPattern),
+                        }
+                    } else {
+                        return Err(Error::DestinationDown);
+                    }
+                }
+            }
+            load_kernel(&main_lib, control, None).await
+        }
+        #[cfg(not(has_drtio))]
+        {
+            panic!("multi-kernel libraries are not supported in standalone systems");
+        }
+    }
+}
+
 async fn load_kernel(
     buffer: &Vec<u8>,
     control: &Rc<RefCell<kernel::Control>>,
@@ -693,7 +755,6 @@ pub fn main(timer: GlobalTimer, cfg: Config) {
 
     Sockets::init(32);
 
-    // before, mutex was on io, but now that io isn't used...?
     let aux_mutex: Rc<Mutex<bool>> = Rc::new(Mutex::new(false));
     #[cfg(has_drtio)]
     let drtio_routing_table = Rc::new(RefCell::new(drtio_routing::config_routing_table(
@@ -716,9 +777,16 @@ pub fn main(timer: GlobalTimer, cfg: Config) {
     let idle_kernel = Rc::new(cfg.read("idle_kernel").ok());
     if let Ok(buffer) = cfg.read("startup_kernel") {
         info!("Loading startup kernel...");
-        if let Ok(()) = task::block_on(load_kernel(&buffer, &control, None)) {
+        let routing_table = drtio_routing_table.borrow();
+        if let Ok(()) = task::block_on(handle_flash_kernel(
+            &buffer,
+            &control,
+            &up_destinations,
+            &aux_mutex,
+            &routing_table,
+            timer,
+        )) {
             info!("Starting startup kernel...");
-            let routing_table = drtio_routing_table.borrow();
             let _ = task::block_on(handle_run_kernel(
                 None,
                 &control,
@@ -766,8 +834,17 @@ pub fn main(timer: GlobalTimer, cfg: Config) {
                             .map_err(|e| warn!("connection terminated: {}", e));
                         if let Some(buffer) = &*idle_kernel {
                             info!("Loading idle kernel");
-                            let _ = load_kernel(&buffer, &control, None)
-                                .await.map_err(|_| warn!("error loading idle kernel"));
+                            let res = handle_flash_kernel(&buffer, &control, &up_destinations,  &aux_mutex, &routing_table, timer)
+                                .await;
+                            match res {
+                                #[cfg(has_drtio)]
+                                Err(Error::DestinationDown) => {
+                                    let mut countdown = timer.countdown();
+                                    delay(&mut countdown, Milliseconds(500)).await;
+                                }
+                                Err(_) => warn!("error loading idle kernel"),
+                                _ => (),
+                            }
                             info!("Running idle kernel");
                             let _ = handle_run_kernel(None, &control, &up_destinations, &aux_mutex, &routing_table, timer)
                                 .await.map_err(|_| warn!("error running idle kernel"));