Firmware: Add drtio_eem.rs support

- Port from Artiq repo
- Initialize the drtio_eem on main, rtio_clocking
- Driver for eem_transceiver

flake.lock

@@ -11,11 +11,11 @@
         "src-pythonparser": "src-pythonparser"
       },
       "locked": {
-        "lastModified": 1693998161,
-        "narHash": "sha256-GbJbBQF8i0FrBOecaAGJFv+0i37wLPgy0uoeyuiq4IM=",
+        "lastModified": 1696817638,
+        "narHash": "sha256-rbIN4Ll1VX4RXxDBlLGgTiQC6L6jZE9q8HYBxPhwCpY=",
         "ref": "refs/heads/master",
-        "rev": "2f3329181c2579ed77334e5875e8698a804be91d",
-        "revCount": 8495,
+        "rev": "d070826911e86b5ec9de958c4bcd93ef0b7fddb2",
+        "revCount": 8564,
         "type": "git",
         "url": "https://github.com/m-labs/artiq.git"
       },
@@ -71,11 +71,11 @@
     "mozilla-overlay": {
       "flake": false,
       "locked": {
-        "lastModified": 1690536331,
-        "narHash": "sha256-aRIf2FB2GTdfF7gl13WyETmiV/J7EhBGkSWXfZvlxcA=",
+        "lastModified": 1695805681,
+        "narHash": "sha256-1ElPLD8eFfnuIk0G52HGGpRtQZ4QPCjChRlEOfkZ5ro=",
         "owner": "mozilla",
         "repo": "nixpkgs-mozilla",
-        "rev": "db89c8707edcffefcd8e738459d511543a339ff5",
+        "rev": "6eabade97bc28d707a8b9d82ad13ef143836736e",
         "type": "github"
       },
       "original": {
@@ -87,11 +87,11 @@
     "mozilla-overlay_2": {
       "flake": false,
       "locked": {
-        "lastModified": 1690536331,
-        "narHash": "sha256-aRIf2FB2GTdfF7gl13WyETmiV/J7EhBGkSWXfZvlxcA=",
+        "lastModified": 1695805681,
+        "narHash": "sha256-1ElPLD8eFfnuIk0G52HGGpRtQZ4QPCjChRlEOfkZ5ro=",
         "owner": "mozilla",
         "repo": "nixpkgs-mozilla",
-        "rev": "db89c8707edcffefcd8e738459d511543a339ff5",
+        "rev": "6eabade97bc28d707a8b9d82ad13ef143836736e",
         "type": "github"
       },
       "original": {
@@ -118,11 +118,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1693771906,
-        "narHash": "sha256-32EnPCaVjOiEERZ+o/2Ir7JH9pkfwJZJ27SKHNvt4yk=",
+        "lastModified": 1696697597,
+        "narHash": "sha256-q26Qv4DQ+h6IeozF2o1secyQG0jt2VUT3V0K58jr3pg=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "da5adce0ffaff10f6d0fee72a02a5ed9d01b52fc",
+        "rev": "5a237aecb57296f67276ac9ab296a41c23981f56",
         "type": "github"
       },
       "original": {

flake.nix

@@ -338,18 +338,22 @@
       } //
       (build { target = "zc706"; variant = "nist_clock"; }) //
       (build { target = "zc706"; variant = "nist_clock_master"; }) //
+      (build { target = "zc706"; variant = "nist_clock_master_100mhz"; }) //
       (build { target = "zc706"; variant = "nist_clock_satellite"; }) //
       (build { target = "zc706"; variant = "nist_clock_satellite_100mhz"; }) //
       (build { target = "zc706"; variant = "nist_qc2"; }) //
       (build { target = "zc706"; variant = "nist_qc2_master"; }) //
+      (build { target = "zc706"; variant = "nist_qc2_master_100mhz"; }) //
       (build { target = "zc706"; variant = "nist_qc2_satellite"; }) //
       (build { target = "zc706"; variant = "nist_qc2_satellite_100mhz"; }) //
       (build { target = "zc706"; variant = "acpki_nist_clock"; }) //
       (build { target = "zc706"; variant = "acpki_nist_clock_master"; }) //
+      (build { target = "zc706"; variant = "acpki_nist_clock_master_100mhz"; }) //
       (build { target = "zc706"; variant = "acpki_nist_clock_satellite"; }) //
       (build { target = "zc706"; variant = "acpki_nist_clock_satellite_100mhz"; }) //
       (build { target = "zc706"; variant = "acpki_nist_qc2"; }) //
       (build { target = "zc706"; variant = "acpki_nist_qc2_master"; }) //
+      (build { target = "zc706"; variant = "acpki_nist_qc2_master_100mhz"; }) //
       (build { target = "zc706"; variant = "acpki_nist_qc2_satellite"; }) //
       (build { target = "zc706"; variant = "acpki_nist_qc2_satellite_100mhz"; }) //
       (build { target = "kasli_soc"; variant = "demo"; json = ./demo.json; }) //

src/Cargo.lock

@@ -218,6 +218,33 @@ dependencies = [
  "libsupport_zynq",
 ]
 
+[[package]]
+name = "ksupport"
+version = "0.1.0"
+dependencies = [
+ "build_zynq",
+ "byteorder",
+ "core_io",
+ "cslice",
+ "dwarf",
+ "dyld",
+ "io",
+ "libasync",
+ "libboard_artiq",
+ "libboard_zynq",
+ "libc",
+ "libconfig",
+ "libcortex_a9",
+ "libm",
+ "libregister",
+ "libsupport_zynq",
+ "log",
+ "log_buffer",
+ "nb 0.1.3",
+ "unwind",
+ "void",
+]
+
 [[package]]
 name = "libasync"
 version = "0.0.0"
@@ -438,18 +465,17 @@ dependencies = [
  "embedded-hal",
  "futures",
  "io",
+ "ksupport",
  "libasync",
  "libboard_artiq",
  "libboard_zynq",
  "libc",
  "libconfig",
  "libcortex_a9",
- "libm",
  "libregister",
  "libsupport_zynq",
  "log",
  "log_buffer",
- "nb 0.1.3",
  "num-derive",
  "num-traits",
  "unwind",
@@ -471,7 +497,11 @@ name = "satman"
 version = "0.0.0"
 dependencies = [
  "build_zynq",
+ "core_io",
+ "cslice",
  "embedded-hal",
+ "io",
+ "ksupport",
  "libasync",
  "libboard_artiq",
  "libboard_zynq",

src/Cargo.toml

@@ -5,6 +5,7 @@ members = [
   "libdwarf",
   "libio",
   "libunwind",
+  "libksupport",
   "runtime",
   "satman"
 ]

src/gateware/kasli_soc.py

@@ -16,7 +16,7 @@ from artiq.coredevice import jsondesc
 from artiq.gateware import rtio, eem_7series
 from artiq.gateware.rtio.xilinx_clocking import fix_serdes_timing_path
 from artiq.gateware.rtio.phy import ttl_simple
-from artiq.gateware.drtio.transceiver import gtx_7series
+from artiq.gateware.drtio.transceiver import gtx_7series, eem_serdes
 from artiq.gateware.drtio.siphaser import SiPhaser7Series
 from artiq.gateware.drtio.rx_synchronizer import XilinxRXSynchronizer
 from artiq.gateware.drtio import *
@@ -61,13 +61,14 @@ class SMAClkinForward(Module):
         ]
 
 
-class GTP125BootstrapClock(Module):
-    def __init__(self, platform):
+class GTPBootstrapClock(Module):
+    def __init__(self, platform, freq=125e6):
         self.clock_domains.cd_bootstrap = ClockDomain(reset_less=True)
         self.cd_bootstrap.clk.attr.add("keep")
 
         bootstrap_125 = platform.request("clk125_gtp")
         bootstrap_se = Signal()
+        clk_out = Signal()
         platform.add_period_constraint(bootstrap_125.p, 8.0)
         self.specials += [
             Instance("IBUFDS_GTE2",
@@ -77,8 +78,30 @@ class GTP125BootstrapClock(Module):
                 p_CLKCM_CFG="TRUE",
                 p_CLKRCV_TRST="TRUE",
                 p_CLKSWING_CFG=3),
-            Instance("BUFG", i_I=bootstrap_se, o_O=self.cd_bootstrap.clk)
+            Instance("BUFG", i_I=bootstrap_se, o_O=clk_out)
         ]
+        if freq == 125e6:
+            self.comb += self.cd_bootstrap.clk.eq(clk_out)
+        elif freq == 100e6:
+            pll_fb = Signal()
+            pll_out = Signal()
+            self.specials += [
+                Instance("PLLE2_BASE",
+                    p_CLKIN1_PERIOD=8.0,
+                    i_CLKIN1=clk_out,
+                    i_CLKFBIN=pll_fb,
+                    o_CLKFBOUT=pll_fb,
+
+                    # VCO @ 1GHz
+                    p_CLKFBOUT_MULT=8, p_DIVCLK_DIVIDE=1,
+
+                    # 100MHz for bootstrap
+                    p_CLKOUT1_DIVIDE=10, p_CLKOUT1_PHASE=0.0, o_CLKOUT1=pll_out,
+                ),
+                Instance("BUFG", i_I=pll_out, o_O=self.cd_bootstrap.clk)
+            ]
+        else:
+            raise ValueError("Bootstrap frequency must be 100 or 125MHz")
 
 
 class GenericStandalone(SoCCore):
@@ -95,6 +118,7 @@ class GenericStandalone(SoCCore):
         SoCCore.__init__(self, platform=platform, csr_data_width=32, ident=ident, ps_cd_sys=False)
 
         self.config["HW_REV"] = description["hw_rev"]
+        
 
         self.submodules += SMAClkinForward(self.platform)
 
@@ -109,8 +133,7 @@ class GenericStandalone(SoCCore):
                 p_DIFF_TERM="TRUE", p_IBUF_LOW_PWR="FALSE",
                 i_I=clk_synth.p, i_IB=clk_synth.n, o_O=clk_synth_se)
         fix_serdes_timing_path(platform)
-        self.submodules.bootstrap = GTP125BootstrapClock(self.platform)
-
+        self.submodules.bootstrap = GTPBootstrapClock(self.platform, description["rtio_frequency"])
 
         self.submodules.sys_crg = zynq_clocking.SYSCRG(self.platform, self.ps7, clk_synth_se)
         platform.add_false_path_constraints(
@@ -134,7 +157,9 @@ class GenericStandalone(SoCCore):
         self.rtio_channels.append(rtio.LogChannel())
 
         self.submodules.rtio_tsc = rtio.TSC(glbl_fine_ts_width=3)
-        self.submodules.rtio_core = rtio.Core(self.rtio_tsc, self.rtio_channels)
+        self.submodules.rtio_core = rtio.Core(
+            self.rtio_tsc, self.rtio_channels, lane_count=description["sed_lanes"]
+        )
         self.csr_devices.append("rtio_core")
 
         if self.acpki:
@@ -176,6 +201,7 @@ class GenericMaster(SoCCore):
     def __init__(self, description, acpki=False):
         clk_freq = description["rtio_frequency"]
 
+        has_drtio_over_eem = any(peripheral["type"] == "shuttler" for peripheral in description["peripherals"])
         self.acpki = acpki
 
         platform = kasli_soc.Platform()
@@ -203,12 +229,13 @@ class GenericMaster(SoCCore):
         gtx0 = self.gt_drtio.gtxs[0]
         self.specials += Instance("BUFG", i_I=gtx0.txoutclk, o_O=txout_buf)
 
-        self.submodules.bootstrap = GTP125BootstrapClock(self.platform)
+        self.submodules.bootstrap = GTPBootstrapClock(self.platform, clk_freq)
         self.submodules.sys_crg = zynq_clocking.SYSCRG(
             self.platform,
             self.ps7,
             txout_buf,
-            clk_sw=gtx0.tx_init.done)
+            clk_sw=gtx0.tx_init.done,
+            enable_sys5x=has_drtio_over_eem)
         self.csr_devices.append("sys_crg")
         self.crg = self.ps7 # HACK for eem_7series to find the clock
         self.crg.cd_sys = self.sys_crg.cd_sys
@@ -221,6 +248,8 @@ class GenericMaster(SoCCore):
 
         self.rtio_channels = []
         has_grabber = any(peripheral["type"] == "grabber" for peripheral in description["peripherals"])
+        if has_drtio_over_eem:
+            self.eem_drtio_channels = []
         if has_grabber:
             self.grabber_csr_group = []
         eem_7series.add_peripherals(self, description["peripherals"], iostandard=eem_iostandard)
@@ -235,17 +264,17 @@ class GenericMaster(SoCCore):
 
         self.submodules.rtio_tsc = rtio.TSC(glbl_fine_ts_width=3)
 
-        drtio_csr_group = []
-        drtioaux_csr_group = []
-        drtioaux_memory_group = []
+        self.drtio_csr_group = []
+        self.drtioaux_csr_group = []
+        self.drtioaux_memory_group = []
         self.drtio_cri = []
         for i in range(len(self.gt_drtio.channels)):
             core_name = "drtio" + str(i)
             coreaux_name = "drtioaux" + str(i)
             memory_name = "drtioaux" + str(i) + "_mem"
-            drtio_csr_group.append(core_name)
-            drtioaux_csr_group.append(coreaux_name)
-            drtioaux_memory_group.append(memory_name)
+            self.drtio_csr_group.append(core_name)
+            self.drtioaux_csr_group.append(coreaux_name)
+            self.drtioaux_memory_group.append(memory_name)
 
             cdr = ClockDomainsRenamer({"rtio_rx": "rtio_rx" + str(i)})
 
@@ -264,11 +293,14 @@ class GenericMaster(SoCCore):
             self.add_memory_region(memory_name, self.mem_map["csr"] + memory_address, size * 2)
         self.config["HAS_DRTIO"] = None
         self.config["HAS_DRTIO_ROUTING"] = None
-        self.add_csr_group("drtio", drtio_csr_group)
-        self.add_csr_group("drtioaux", drtioaux_csr_group)
-        self.add_memory_group("drtioaux_mem", drtioaux_memory_group)
 
-        self.submodules.rtio_core = rtio.Core(self.rtio_tsc, self.rtio_channels)
+        if has_drtio_over_eem:
+            self.add_eem_drtio(self.eem_drtio_channels)
+        self.add_drtio_cpuif_groups()
+
+        self.submodules.rtio_core = rtio.Core(
+            self.rtio_tsc, self.rtio_channels, lane_count=description["sed_lanes"]
+        )
         self.csr_devices.append("rtio_core")
 
         if self.acpki:
@@ -313,6 +345,42 @@ class GenericMaster(SoCCore):
         self.comb += [self.virtual_leds.get(i).eq(channel.rx_ready)
                 for i, channel in enumerate(self.gt_drtio.channels)]
 
+    def add_eem_drtio(self, eem_drtio_channels):
+        # Must be called before invoking add_rtio() to construct the CRI
+        # interconnect properly
+        self.submodules.eem_transceiver = eem_serdes.EEMSerdes(self.platform, eem_drtio_channels)
+        self.csr_devices.append("eem_transceiver")
+        self.config["HAS_DRTIO_EEM"] = None
+        self.config["EEM_DRTIO_COUNT"] = len(eem_drtio_channels)
+
+        cdr = ClockDomainsRenamer({"rtio_rx": "sys"})
+        for i in range(len(self.eem_transceiver.channels)):
+            channel = i + len(self.gt_drtio.channels)
+            core_name = "drtio" + str(channel)
+            coreaux_name = "drtioaux" + str(channel)
+            memory_name = "drtioaux" + str(channel) + "_mem"
+            self.drtio_csr_group.append(core_name)
+            self.drtioaux_csr_group.append(coreaux_name)
+            self.drtioaux_memory_group.append(memory_name)
+
+            core = cdr(DRTIOMaster(self.rtio_tsc, self.eem_transceiver.channels[i]))
+            setattr(self.submodules, core_name, core)
+            self.drtio_cri.append(core.cri)
+            self.csr_devices.append(core_name)
+
+            coreaux = cdr(drtio_aux_controller.DRTIOAuxControllerBare(core.link_layer))
+            setattr(self.submodules, coreaux_name, coreaux)
+            self.csr_devices.append(coreaux_name)
+
+            size = coreaux.get_mem_size()
+            memory_address = self.axi2csr.register_port(coreaux.get_tx_port(), size)
+            self.axi2csr.register_port(coreaux.get_rx_port(), size)
+            self.add_memory_region(memory_name, self.mem_map["csr"] + memory_address, size * 2)
+
+    def add_drtio_cpuif_groups(self):
+        self.add_csr_group("drtio", self.drtio_csr_group)
+        self.add_csr_group("drtioaux", self.drtioaux_csr_group)
+        self.add_memory_group("drtioaux_mem", self.drtioaux_memory_group)
 
 
 class GenericSatellite(SoCCore):
@@ -344,7 +412,7 @@ class GenericSatellite(SoCCore):
         gtx0 = self.gt_drtio.gtxs[0]
         self.specials += Instance("BUFG", i_I=gtx0.txoutclk, o_O=txout_buf)
 
-        self.submodules.bootstrap = GTP125BootstrapClock(self.platform)
+        self.submodules.bootstrap = GTPBootstrapClock(self.platform, clk_freq)
         self.submodules.sys_crg = zynq_clocking.SYSCRG(
             self.platform, 
             self.ps7,
@@ -437,11 +505,13 @@ class GenericSatellite(SoCCore):
         self.submodules.rtio_dma = dma.DMA(self.ps7.s_axi_hp0)
         self.csr_devices.append("rtio_dma")
 
-        self.submodules.local_io = SyncRTIO(self.rtio_tsc, self.rtio_channels)
+        self.submodules.local_io = SyncRTIO(
+            self.rtio_tsc, self.rtio_channels, lane_count=description["sed_lanes"]
+        )
         self.comb += self.drtiosat.async_errors.eq(self.local_io.async_errors)
 
         self.submodules.cri_con = rtio.CRIInterconnectShared(
-            [self.drtiosat.cri, self.rtio_dma.cri],
+            [self.drtiosat.cri, self.rtio_dma.cri, self.rtio.cri],
             [self.local_io.cri] + self.drtio_cri,
             enable_routing=True)
         self.csr_devices.append("cri_con")

src/gateware/zc706.py

@@ -476,7 +476,7 @@ class _SatelliteBase(SoCCore):
 
         self.submodules.local_io = SyncRTIO(self.rtio_tsc, rtio_channels)
         self.submodules.cri_con = rtio.CRIInterconnectShared(
-            [self.drtiosat.cri, self.rtio_dma.cri],
+            [self.drtiosat.cri, self.rtio_dma.cri, self.rtio.cri],
             [self.local_io.cri] + self.drtio_cri,
             enable_routing=True)
         self.csr_devices.append("cri_con")

src/gateware/zynq_clocking.py

@@ -65,10 +65,13 @@ class ClockSwitchFSM(Module):
 
 
 class SYSCRG(Module, AutoCSR):
-    def __init__(self, platform, ps7, main_clk, clk_sw=None, freq=125e6):
+    def __init__(self, platform, ps7, main_clk, clk_sw=None, freq=125e6, enable_sys5x=False):
         # assumes bootstrap clock is same freq as main and sys output
         self.clock_domains.cd_sys = ClockDomain()
         self.clock_domains.cd_sys4x = ClockDomain(reset_less=True)
+        if enable_sys5x:
+            self.clock_domains.cd_clk200 = ClockDomain()
+            self.clock_domains.cd_sys5x = ClockDomain(reset_less=True)
 
         self.current_clock = CSRStatus()
 
@@ -78,11 +81,6 @@ class SYSCRG(Module, AutoCSR):
 
         period = 1e9/freq
 
-        pll_locked = Signal()
-        pll_sys = Signal()
-        pll_sys4x = Signal()
-        fb_clk = Signal()
-
         self.submodules.clk_sw_fsm = ClockSwitchFSM()
 
         if clk_sw is None:
@@ -91,32 +89,105 @@ class SYSCRG(Module, AutoCSR):
         else:
             self.comb += self.clk_sw_fsm.i_clk_sw.eq(clk_sw)
 
-        self.specials += [
-            Instance("PLLE2_ADV",
-                     p_STARTUP_WAIT="FALSE", o_LOCKED=pll_locked,
-                     p_BANDWIDTH="HIGH",
-                     p_REF_JITTER1=0.001,
-                     p_CLKIN1_PERIOD=period, i_CLKIN1=main_clk,
-                     p_CLKIN2_PERIOD=period, i_CLKIN2=bootstrap_clk,
-                     i_CLKINSEL=self.clk_sw_fsm.o_clk_sw,
+        if enable_sys5x:
+            pll_clk200 = Signal()
+            pll_fb_clk = Signal()
+            pll_locked = Signal()
+            self.specials += [
+                Instance("PLLE2_BASE",
+                    p_CLKIN1_PERIOD=8.0,
+                    i_CLKIN1=bootstrap_clk,
 
-                     # VCO @ 1.5GHz when using 125MHz input
-                     # 1.2GHz for 100MHz (zc706)
-                     p_CLKFBOUT_MULT=12, p_DIVCLK_DIVIDE=1,
-                     i_CLKFBIN=fb_clk,
-                     i_RST=self.clk_sw_fsm.o_reset,
+                    i_CLKFBIN=pll_fb_clk,
+                    o_CLKFBOUT=pll_fb_clk,
+                    o_LOCKED=pll_locked,
 
-                     o_CLKFBOUT=fb_clk,
+                    # VCO @ 1GHz
+                    p_CLKFBOUT_MULT=8, p_DIVCLK_DIVIDE=1,
 
-                     p_CLKOUT0_DIVIDE=3, p_CLKOUT0_PHASE=0.0,
-                     o_CLKOUT0=pll_sys4x,
+                    # 200MHz for IDELAYCTRL
+                    p_CLKOUT0_DIVIDE=5, p_CLKOUT0_PHASE=0.0, o_CLKOUT0=pll_clk200,
+                ),
+                Instance("BUFG", i_I=pll_clk200, o_O=self.cd_clk200.clk),
+                AsyncResetSynchronizer(self.cd_clk200, ~pll_locked),
+            ]
 
-                     p_CLKOUT1_DIVIDE=12, p_CLKOUT1_PHASE=0.0,
-                     o_CLKOUT1=pll_sys),
-            Instance("BUFG", i_I=pll_sys, o_O=self.cd_sys.clk),
-            Instance("BUFG", i_I=pll_sys4x, o_O=self.cd_sys4x.clk),
+            reset_counter = Signal(4, reset=15)
+            ic_reset = Signal(reset=1)
+            self.sync.clk200 += \
+                If(reset_counter != 0,
+                    reset_counter.eq(reset_counter - 1)
+                ).Else(
+                    ic_reset.eq(0)
+                )
+            self.specials += Instance("IDELAYCTRL", i_REFCLK=ClockSignal("clk200"), i_RST=ic_reset)
 
-            AsyncResetSynchronizer(self.cd_sys, ~pll_locked),
-        ]
+            mmcm_locked = Signal()
+            mmcm_sys = Signal()
+            mmcm_sys4x = Signal()
+            mmcm_sys5x = Signal()
+            mmcm_fb_clk = Signal()
+            self.specials += [
+                Instance("MMCME2_ADV",
+                    p_STARTUP_WAIT="FALSE", o_LOCKED=mmcm_locked,
+                    p_BANDWIDTH="HIGH",
+                    p_REF_JITTER1=0.001,
+                    p_CLKIN1_PERIOD=period, i_CLKIN1=main_clk,
+                    p_CLKIN2_PERIOD=period, i_CLKIN2=bootstrap_clk,
+                    i_CLKINSEL=self.clk_sw_fsm.o_clk_sw,
+
+                    # VCO @ 1.25GHz with MULT=10
+                    p_CLKFBOUT_MULT_F=10, p_DIVCLK_DIVIDE=1,
+                    i_CLKFBIN=mmcm_fb_clk,
+                    i_RST=self.clk_sw_fsm.o_reset,
+
+                    o_CLKFBOUT=mmcm_fb_clk,
+
+                    # 500MHz. Must be more than 400MHz as per DDR3 specs.
+                    p_CLKOUT0_DIVIDE_F=2.5, p_CLKOUT0_PHASE=0.0, o_CLKOUT0=mmcm_sys4x,
+
+                    # 125MHz
+                    p_CLKOUT1_DIVIDE=10, p_CLKOUT1_PHASE=0.0, o_CLKOUT1=mmcm_sys,
+
+                    # 625MHz
+                    p_CLKOUT2_DIVIDE=2, p_CLKOUT2_PHASE=0.0, o_CLKOUT2=mmcm_sys5x,
+                ),
+                Instance("BUFG", i_I=mmcm_sys5x, o_O=self.cd_sys5x.clk),
+                Instance("BUFG", i_I=mmcm_sys, o_O=self.cd_sys.clk),
+                Instance("BUFG", i_I=mmcm_sys4x, o_O=self.cd_sys4x.clk),
+                AsyncResetSynchronizer(self.cd_sys, ~mmcm_locked),
+            ]
+        else:
+            pll_locked = Signal()
+            pll_sys = Signal()
+            pll_sys4x = Signal()
+            fb_clk = Signal()
+            self.specials += [
+                Instance("PLLE2_ADV",
+                    p_STARTUP_WAIT="FALSE", o_LOCKED=pll_locked,
+                    p_BANDWIDTH="HIGH",
+                    p_REF_JITTER1=0.001,
+                    p_CLKIN1_PERIOD=period, i_CLKIN1=main_clk,
+                    p_CLKIN2_PERIOD=period, i_CLKIN2=bootstrap_clk,
+                    i_CLKINSEL=self.clk_sw_fsm.o_clk_sw,
+
+                    # VCO @ 1.5GHz when using 125MHz input
+                    # 1.2GHz for 100MHz (zc706)
+                    p_CLKFBOUT_MULT=12, p_DIVCLK_DIVIDE=1,
+                    i_CLKFBIN=fb_clk,
+                    i_RST=self.clk_sw_fsm.o_reset,
+
+                    o_CLKFBOUT=fb_clk,
+
+                    p_CLKOUT0_DIVIDE=3, p_CLKOUT0_PHASE=0.0,
+                    o_CLKOUT0=pll_sys4x,
+
+                    p_CLKOUT1_DIVIDE=12, p_CLKOUT1_PHASE=0.0,
+                    o_CLKOUT1=pll_sys),
+
+                Instance("BUFG", i_I=pll_sys, o_O=self.cd_sys.clk),
+                Instance("BUFG", i_I=pll_sys4x, o_O=self.cd_sys4x.clk),
+                AsyncResetSynchronizer(self.cd_sys, ~pll_locked),
+            ]
 
         self.comb += self.current_clock.status.eq(self.clk_sw_fsm.o_clk_sw)

src/libboard_artiq/Cargo.toml

@@ -25,6 +25,7 @@ void = { version = "1", default-features = false }
 
 io = { path = "../libio", features = ["byteorder"] }
 libboard_zynq = { git = "https://git.m-labs.hk/M-Labs/zynq-rs.git"}
+libsupport_zynq = { default-features = false, features = ["alloc_core"], git = "https://git.m-labs.hk/M-Labs/zynq-rs.git" }
 libregister = { git = "https://git.m-labs.hk/M-Labs/zynq-rs.git" }
 libconfig = { git = "https://git.m-labs.hk/M-Labs/zynq-rs.git", features = ["fat_lfn"] }
 libcortex_a9 = { git = "https://git.m-labs.hk/M-Labs/zynq-rs.git" }

src/libboard_artiq/src/drtio_eem.rs

@@ -0,0 +1,235 @@
+use crate::pl;
+use log::{info, debug, error};
+use embedded_hal::prelude::_embedded_hal_blocking_delay_DelayUs;
+use libconfig::Config;
+use libboard_zynq::timer::GlobalTimer;
+use libsupport_zynq::alloc::format;
+
+struct SerdesConfig {
+    pub delay: [u8; 4],
+}
+
+impl SerdesConfig {
+    pub fn as_bytes(&self) -> &[u8] {
+        unsafe {
+            core::slice::from_raw_parts(
+                (self as *const SerdesConfig) as *const u8,
+                core::mem::size_of::<SerdesConfig>(),
+            )
+        }
+    }
+}
+
+fn select_lane(lane_no: u8) {
+    unsafe {
+        pl::csr::eem_transceiver::lane_sel_write(lane_no);
+    }
+}
+
+fn apply_delay(tap: u8, timer: &mut GlobalTimer) {
+    unsafe {
+        pl::csr::eem_transceiver::dly_cnt_in_write(tap);
+        pl::csr::eem_transceiver::dly_ld_write(1);
+        timer.delay_us(1);
+        assert!(tap as u8 == pl::csr::eem_transceiver::dly_cnt_out_read());
+    }
+}
+
+fn apply_config(config: &SerdesConfig, timer: &mut GlobalTimer) {
+    for lane_no in 0..4 {
+        select_lane(lane_no as u8);
+        apply_delay(config.delay[lane_no], timer);
+    }
+}
+
+
+
+unsafe fn assign_delay(timer: &mut GlobalTimer) -> SerdesConfig {
+    // Select an appropriate delay for lane 0
+    select_lane(0);
+
+    //
+
+    let mut best_dly = None;
+
+    loop {
+        let mut prev = None;
+        for curr_dly in 0..32 {
+            //let read_align = read_align_fn(curr_dly, timer);
+            let curr_low_rate = read_align(curr_dly, timer);
+
+            if let Some(prev_low_rate) = prev {
+                // This is potentially a crossover position
+                if prev_low_rate <= curr_low_rate && curr_low_rate >= 0.5 {
+                    let prev_dev = 0.5 - prev_low_rate;
+                    let curr_dev = curr_low_rate - 0.5;
+                    let selected_idx = if prev_dev < curr_dev {
+                        curr_dly - 1
+                    } else {
+                        curr_dly
+                    };
+
+                    // The setup setup/hold calibration timing (even with
+                    // tolerance) might be invalid in other lanes due to skew.
+                    // 5 taps is very conservative, generally it is 1 or 2
+                    if selected_idx < 5 {
+                        prev = None;
+                        continue;
+                    } else {
+                        best_dly = Some(selected_idx);
+                        break;
+                    }
+                }
+            }
+
+            // Only rising slope from <= 0.5 can result in a rising low rate
+            // crossover at 50%.
+            if curr_low_rate <= 0.5 {
+                prev = Some(curr_low_rate);
+            }
+        }
+
+        if best_dly.is_none() {
+            error!("setup/hold timing calibration failed, retry in 1s...");
+            timer.delay_us(1_000_000);
+        } else {
+            break;
+        }
+    }
+
+    let best_dly = best_dly.unwrap();
+
+    apply_delay(best_dly, timer);
+    let mut delay_list = [best_dly; 4];
+
+    // Assign delay for other lanes
+    for lane_no in 1..=3 {
+        select_lane(lane_no as u8);
+
+        let mut min_deviation = 0.5;
+        let mut min_idx = 0;
+        for dly_delta in -3..=3 {
+            let index = (best_dly as isize + dly_delta) as u8;
+            let low_rate = read_align(index, timer);
+            // abs() from f32 is not available in core library
+            let deviation = if low_rate < 0.5 {
+                0.5 - low_rate
+            } else {
+                low_rate - 0.5
+            };
+
+            if deviation < min_deviation {
+                min_deviation = deviation;
+                min_idx = index;
+            }
+        }
+
+        apply_delay(min_idx, timer);
+        delay_list[lane_no] = min_idx;
+    }
+
+    debug!("setup/hold timing calibration: {:?}", delay_list);
+
+    SerdesConfig {
+        delay: delay_list,
+    }
+}
+
+fn read_align(dly: u8, timer: &mut GlobalTimer) -> f32 {
+    unsafe{
+        apply_delay(dly, timer);
+        pl::csr::eem_transceiver::counter_reset_write(1);
+
+        pl::csr::eem_transceiver::counter_enable_write(1);
+        timer.delay_us(2000);
+        pl::csr::eem_transceiver::counter_enable_write(0);
+
+        let (high, low) = (
+            pl::csr::eem_transceiver::counter_high_count_read(),
+            pl::csr::eem_transceiver::counter_low_count_read(),
+        );
+        if pl::csr::eem_transceiver::counter_overflow_read() == 1 {
+            panic!("Unexpected phase detector counter overflow");
+        }
+
+        low as f32 / (low + high) as f32
+    }
+}
+
+unsafe fn align_comma(timer: &mut GlobalTimer) {
+    loop {
+        for slip in 1..=10 {
+            // The soft transceiver has 2 8b10b decoders, which receives lane
+            // 0/1 and lane 2/3 respectively. The decoder are time-multiplexed
+            // to decode exactly 1 lane each sysclk cycle.
+            //
+            // The decoder decodes lane 0/2 data on odd sysclk cycles, buffer
+            // on even cycles, and vice versa for lane 1/3. Data/Clock latency
+            // could change timing. The extend bit flips the decoding timing,
+            // so lane 0/2 data are decoded on even cycles, and lane 1/3 data
+            // are decoded on odd cycles.
+            //
+            // This is needed because transmitting/receiving a 8b10b character
+            // takes 2 sysclk cycles. Adjusting bitslip only via ISERDES
+            // limits the range to 1 cycle. The wordslip bit extends the range
+            // to 2 sysclk cycles.
+            pl::csr::eem_transceiver::wordslip_write((slip > 5) as u8);
+
+            // Apply a double bitslip since the ISERDES is 2x oversampled.
+            // Bitslip is used for comma alignment purposes once setup/hold
+            // timing is met.
+            pl::csr::eem_transceiver::bitslip_write(1);
+            pl::csr::eem_transceiver::bitslip_write(1);
+            timer.delay_us(1);
+
+            pl::csr::eem_transceiver::comma_align_reset_write(1);
+            timer.delay_us(100);
+
+            if pl::csr::eem_transceiver::comma_read() == 1 {
+                debug!("comma alignment completed after {} bitslips", slip);
+                return;
+            }
+        }
+
+        error!("comma alignment failed, retrying in 1s...");
+        timer.delay_us(1_000_000);
+    }
+}
+
+pub fn init(timer: &mut GlobalTimer, cfg: &Config) {
+    for trx_no in 0..pl::csr::CONFIG_EEM_DRTIO_COUNT {
+        unsafe {
+            pl::csr::eem_transceiver::transceiver_sel_write(trx_no as u8);
+        }
+
+        let key = format!("eem_drtio_delay{}", trx_no);
+        
+        let cfg_read = cfg.read(&key);
+        match cfg_read{
+            Ok(record) => {
+                info!("loading calibrated timing values from sd card");
+                unsafe {
+                    apply_config(&*(record.as_ptr() as *const SerdesConfig), timer);
+                }
+            },
+            Err(_) => {
+                info!("calibrating...");
+                let config = unsafe { assign_delay(timer) };
+                
+                match cfg.write(&key, config.as_bytes().to_vec()){
+                    Ok(()) => {
+                        info!("storing calibration timing values into sd card")
+                    },
+                    Err(e) => {
+                        error!("calibration successful but calibration timing values cannot be stored into sd card. Error:{}", e);
+                    }
+                };
+            }
+        }
+        
+        unsafe {
+            align_comma(timer);
+            pl::csr::eem_transceiver::rx_ready_write(1);
+        }
+    }
+}

src/libboard_artiq/src/drtioaux_proto.rs

@@ -1,8 +1,11 @@
 use core_io::{Error as IoError, Read, Write};
 use io::proto::{ProtoRead, ProtoWrite};
 
-pub const DMA_TRACE_MAX_SIZE: usize = /*max size*/512 - /*CRC*/4 - /*packet ID*/1 - /*trace ID*/4 - /*last*/1 -/*length*/2;
-pub const ANALYZER_MAX_SIZE: usize  = /*max size*/512 - /*CRC*/4 - /*packet ID*/1 - /*last*/1 - /*length*/2;
+// maximum size of arbitrary payloads
+// used by satellite -> master analyzer, subkernel exceptions
+pub const SAT_PAYLOAD_MAX_SIZE: usize  = /*max size*/512 - /*CRC*/4 - /*packet ID*/1 - /*last*/1 - /*length*/2;
+// used by DDMA, subkernel program data (need to provide extra ID and destination)
+pub const MASTER_PAYLOAD_MAX_SIZE: usize = SAT_PAYLOAD_MAX_SIZE - /*destination*/1 - /*ID*/4;
 
 #[derive(Debug)]
 pub enum Error {
@@ -150,7 +153,7 @@ pub enum Packet {
     AnalyzerData {
         last: bool,
         length: u16,
-        data: [u8; ANALYZER_MAX_SIZE],
+        data: [u8; SAT_PAYLOAD_MAX_SIZE],
     },
 
     DmaAddTraceRequest {
@@ -158,7 +161,7 @@ pub enum Packet {
         id: u32,
         last: bool,
         length: u16,
-        trace: [u8; DMA_TRACE_MAX_SIZE],
+        trace: [u8; MASTER_PAYLOAD_MAX_SIZE],
     },
     DmaAddTraceReply {
         succeeded: bool,
@@ -185,6 +188,53 @@ pub enum Packet {
         channel: u32,
         timestamp: u64,
     },
+
+    SubkernelAddDataRequest {
+        destination: u8,
+        id: u32,
+        last: bool,
+        length: u16,
+        data: [u8; MASTER_PAYLOAD_MAX_SIZE],
+    },
+    SubkernelAddDataReply {
+        succeeded: bool,
+    },
+    SubkernelLoadRunRequest {
+        destination: u8,
+        id: u32,
+        run: bool,
+    },
+    SubkernelLoadRunReply {
+        succeeded: bool,
+    },
+    SubkernelStopRequest {
+        destination: u8,
+    },
+    SubkernelStopReply {
+        succeeded: bool,
+    },
+    SubkernelFinished {
+        id: u32,
+        with_exception: bool,
+    },
+    SubkernelExceptionRequest {
+        destination: u8,
+    },
+    SubkernelException {
+        last: bool,
+        length: u16,
+        data: [u8; SAT_PAYLOAD_MAX_SIZE],
+    },
+    SubkernelMessage {
+        destination: u8,
+        id: u32,
+        last: bool,
+        length: u16,
+        data: [u8; MASTER_PAYLOAD_MAX_SIZE],
+    },
+    SubkernelMessageAck {
+        destination: u8,
+    },
 }
 
 impl Packet {
@@ -329,7 +379,7 @@ impl Packet {
             0xa3 => {
                 let last = reader.read_bool()?;
                 let length = reader.read_u16()?;
-                let mut data: [u8; ANALYZER_MAX_SIZE] = [0; ANALYZER_MAX_SIZE];
+                let mut data: [u8; SAT_PAYLOAD_MAX_SIZE] = [0; SAT_PAYLOAD_MAX_SIZE];
                 reader.read_exact(&mut data[0..length as usize])?;
                 Packet::AnalyzerData {
                     last: last,
@@ -343,7 +393,7 @@ impl Packet {
                 let id = reader.read_u32()?;
                 let last = reader.read_bool()?;
                 let length = reader.read_u16()?;
-                let mut trace: [u8; DMA_TRACE_MAX_SIZE] = [0; DMA_TRACE_MAX_SIZE];
+                let mut trace: [u8; MASTER_PAYLOAD_MAX_SIZE] = [0; MASTER_PAYLOAD_MAX_SIZE];
                 reader.read_exact(&mut trace[0..length as usize])?;
                 Packet::DmaAddTraceRequest {
                     destination: destination,
@@ -379,6 +429,75 @@ impl Packet {
                 timestamp: reader.read_u64()?,
             },
 
+            0xc0 => {
+                let destination = reader.read_u8()?;
+                let id = reader.read_u32()?;
+                let last = reader.read_bool()?;
+                let length = reader.read_u16()?;
+                let mut data: [u8; MASTER_PAYLOAD_MAX_SIZE] = [0; MASTER_PAYLOAD_MAX_SIZE];
+                reader.read_exact(&mut data[0..length as usize])?;
+                Packet::SubkernelAddDataRequest {
+                    destination: destination,
+                    id: id,
+                    last: last,
+                    length: length as u16,
+                    data: data,
+                }
+            }
+            0xc1 => Packet::SubkernelAddDataReply {
+                succeeded: reader.read_bool()?,
+            },
+            0xc4 => Packet::SubkernelLoadRunRequest {
+                destination: reader.read_u8()?,
+                id: reader.read_u32()?,
+                run: reader.read_bool()?,
+            },
+            0xc5 => Packet::SubkernelLoadRunReply {
+                succeeded: reader.read_bool()?,
+            },
+            0xc6 => Packet::SubkernelStopRequest {
+                destination: reader.read_u8()?,
+            },
+            0xc7 => Packet::SubkernelStopReply {
+                succeeded: reader.read_bool()?,
+            },
+            0xc8 => Packet::SubkernelFinished {
+                id: reader.read_u32()?,
+                with_exception: reader.read_bool()?,
+            },
+            0xc9 => Packet::SubkernelExceptionRequest {
+                destination: reader.read_u8()?,
+            },
+            0xca => {
+                let last = reader.read_bool()?;
+                let length = reader.read_u16()?;
+                let mut data: [u8; SAT_PAYLOAD_MAX_SIZE] = [0; SAT_PAYLOAD_MAX_SIZE];
+                reader.read_exact(&mut data[0..length as usize])?;
+                Packet::SubkernelException {
+                    last: last,
+                    length: length,
+                    data: data,
+                }
+            }
+            0xcb => {
+                let destination = reader.read_u8()?;
+                let id = reader.read_u32()?;
+                let last = reader.read_bool()?;
+                let length = reader.read_u16()?;
+                let mut data: [u8; MASTER_PAYLOAD_MAX_SIZE] = [0; MASTER_PAYLOAD_MAX_SIZE];
+                reader.read_exact(&mut data[0..length as usize])?;
+                Packet::SubkernelMessage {
+                    destination: destination,
+                    id: id,
+                    last: last,
+                    length: length as u16,
+                    data: data,
+                }
+            }
+            0xcc => Packet::SubkernelMessageAck {
+                destination: reader.read_u8()?,
+            },
+
             ty => return Err(Error::UnknownPacket(ty)),
         })
     }
@@ -648,6 +767,76 @@ impl Packet {
                 writer.write_u32(channel)?;
                 writer.write_u64(timestamp)?;
             }
+
+            Packet::SubkernelAddDataRequest {
+                destination,
+                id,
+                last,
+                data,
+                length,
+            } => {
+                writer.write_u8(0xc0)?;
+                writer.write_u8(destination)?;
+                writer.write_u32(id)?;
+                writer.write_bool(last)?;
+                writer.write_u16(length)?;
+                writer.write_all(&data[0..length as usize])?;
+            }
+            Packet::SubkernelAddDataReply { succeeded } => {
+                writer.write_u8(0xc1)?;
+                writer.write_bool(succeeded)?;
+            }
+            Packet::SubkernelLoadRunRequest { destination, id, run } => {
+                writer.write_u8(0xc4)?;
+                writer.write_u8(destination)?;
+                writer.write_u32(id)?;
+                writer.write_bool(run)?;
+            }
+            Packet::SubkernelLoadRunReply { succeeded } => {
+                writer.write_u8(0xc5)?;
+                writer.write_bool(succeeded)?;
+            }
+            Packet::SubkernelStopRequest { destination } => {
+                writer.write_u8(0xc6)?;
+                writer.write_u8(destination)?;
+            }
+            Packet::SubkernelStopReply { succeeded } => {
+                writer.write_u8(0xc7)?;
+                writer.write_bool(succeeded)?;
+            }
+            Packet::SubkernelFinished { id, with_exception } => {
+                writer.write_u8(0xc8)?;
+                writer.write_u32(id)?;
+                writer.write_bool(with_exception)?;
+            }
+            Packet::SubkernelExceptionRequest { destination } => {
+                writer.write_u8(0xc9)?;
+                writer.write_u8(destination)?;
+            }
+            Packet::SubkernelException { last, length, data } => {
+                writer.write_u8(0xca)?;
+                writer.write_bool(last)?;
+                writer.write_u16(length)?;
+                writer.write_all(&data[0..length as usize])?;
+            }
+            Packet::SubkernelMessage {
+                destination,
+                id,
+                last,
+                data,
+                length,
+            } => {
+                writer.write_u8(0xcb)?;
+                writer.write_u8(destination)?;
+                writer.write_u32(id)?;
+                writer.write_bool(last)?;
+                writer.write_u16(length)?;
+                writer.write_all(&data[0..length as usize])?;
+            }
+            Packet::SubkernelMessageAck { destination } => {
+                writer.write_u8(0xcc)?;
+                writer.write_u8(destination)?;
+            }
         }
         Ok(())
     }

src/libboard_artiq/src/lib.rs

@@ -31,6 +31,8 @@ pub mod mem;
 pub mod pl;
 #[cfg(has_si5324)]
 pub mod si5324;
+#[cfg(has_drtio_eem)]
+pub mod drtio_eem;
 
 use core::{cmp, str};
 

src/libio/cursor.rs

@@ -1,3 +1,6 @@
+#[cfg(feature = "alloc")]
+use alloc::vec::Vec;
+
 use core_io::{Error as IoError, Read, Write};
 
 #[derive(Debug, Clone)]
@@ -64,7 +67,7 @@ impl Write for Cursor<&mut [u8]> {
 }
 
 #[cfg(feature = "alloc")]
-impl Write for Cursor<::alloc::Vec<u8>> {
+impl Write for Cursor<Vec<u8>> {
     #[inline]
     fn write(&mut self, buf: &[u8]) -> Result<usize, IoError> {
         self.inner.extend_from_slice(buf);

src/libio/lib.rs

@@ -1,13 +1,10 @@
 #![no_std]
 #![feature(never_type)]
-#![cfg_attr(feature = "alloc", feature(alloc))]
 
+#[cfg(feature = "alloc")]
 extern crate alloc;
 extern crate core_io;
 
-#[cfg(feature = "alloc")]
-#[macro_use]
-use alloc;
 #[cfg(feature = "byteorder")]
 extern crate byteorder;
 

src/libio/proto.rs

@@ -1,3 +1,4 @@
+#[cfg(feature = "alloc")]
 use alloc::{string::String, vec};
 use core::str::Utf8Error;
 
@@ -50,7 +51,8 @@ pub trait ProtoRead {
     }
 
     #[inline]
-    fn read_bytes(&mut self) -> Result<::alloc::vec::Vec<u8>, Self::ReadError> {
+    #[cfg(feature = "alloc")]
+    fn read_bytes(&mut self) -> Result<vec::Vec<u8>, Self::ReadError> {
         let length = self.read_u32()?;
         let mut value = vec![0; length as usize];
         self.read_exact(&mut value)?;
@@ -58,7 +60,8 @@ pub trait ProtoRead {
     }
 
     #[inline]
-    fn read_string(&mut self) -> Result<::alloc::string::String, ReadStringError<Self::ReadError>> {
+    #[cfg(feature = "alloc")]
+    fn read_string(&mut self) -> Result<String, ReadStringError<Self::ReadError>> {
         let bytes = self.read_bytes().map_err(ReadStringError::Other)?;
         String::from_utf8(bytes).map_err(|err| ReadStringError::Utf8(err.utf8_error()))
     }
@@ -135,6 +138,7 @@ pub trait ProtoWrite {
     }
 
     #[inline]
+    #[cfg(feature = "alloc")]
     fn write_string(&mut self, value: &str) -> Result<(), Self::WriteError> {
         self.write_bytes(value.as_bytes())
     }

src/libksupport/Cargo.toml

@@ -0,0 +1,33 @@
+[package]
+name = "ksupport"
+description = "Kernel support for Zynq-based platforms"
+version = "0.1.0"
+authors = ["M-Labs"]
+edition = "2018"
+
+[build-dependencies]
+build_zynq = { path = "../libbuild_zynq" }
+
+[dependencies]
+cslice = "0.3"
+log = "0.4"
+nb = "0.1"
+core_io = { version = "0.1", features = ["collections"] }
+byteorder = { version = "1.3", default-features = false }
+void = { version = "1", default-features = false }
+log_buffer = { version = "1.2" }
+libm = { version = "0.2", features = ["unstable"] }
+
+libboard_zynq = { git = "https://git.m-labs.hk/M-Labs/zynq-rs.git", features = ["ipv6"]}
+libsupport_zynq = { default-features = false, features = ["alloc_core"], git = "https://git.m-labs.hk/M-Labs/zynq-rs.git" }
+libcortex_a9 = { git = "https://git.m-labs.hk/M-Labs/zynq-rs.git" }
+libasync = { git = "https://git.m-labs.hk/M-Labs/zynq-rs.git" }
+libregister = { git = "https://git.m-labs.hk/M-Labs/zynq-rs.git" }
+libconfig = { git = "https://git.m-labs.hk/M-Labs/zynq-rs.git", features = ["fat_lfn", "ipv6"] }
+
+dyld = { path = "../libdyld" }
+dwarf = { path = "../libdwarf" }
+unwind = { path = "../libunwind" }
+libc = { path = "../libc" }
+io = { path = "../libio" }
+libboard_artiq = { path = "../libboard_artiq" }

src/libksupport/build.rs

@@ -0,0 +1,5 @@
+extern crate build_zynq;
+
+fn main() {
+    build_zynq::cfg();
+}

src/libksupport/src/eh_artiq.rs

@@ -422,7 +422,7 @@ extern "C" fn stop_fn(
 }
 
 // Must be kept in sync with preallocate_runtime_exception_names() in artiq/language/embedding_map.py
-static EXCEPTION_ID_LOOKUP: [(&str, u32); 11] = [
+static EXCEPTION_ID_LOOKUP: [(&str, u32); 12] = [
     ("RuntimeError", 0),
     ("RTIOUnderflow", 1),
     ("RTIOOverflow", 2),
@@ -434,6 +434,7 @@ static EXCEPTION_ID_LOOKUP: [(&str, u32); 11] = [
     ("ZeroDivisionError", 8),
     ("IndexError", 9),
     ("UnwrapNoneError", 10),
+    ("SubkernelError", 11),
 ];
 
 pub fn get_exception_id(name: &str) -> u32 {

src/libksupport/src/kernel/api.rs

@@ -5,6 +5,8 @@ use libc::{c_char, c_int, size_t};
 use libm;
 use log::{info, warn};
 
+#[cfg(has_drtio)]
+use super::subkernel;
 use super::{cache,
             core1::rtio_get_destination_status,
             dma,
@@ -114,6 +116,16 @@ pub fn resolve(required: &[u8]) -> Option<u32> {
         api!(i2c_read = i2c::read),
         api!(i2c_switch_select = i2c::switch_select),
 
+        // subkernel
+        #[cfg(has_drtio)]
+        api!(subkernel_load_run = subkernel::load_run),
+        #[cfg(has_drtio)]
+        api!(subkernel_await_finish = subkernel::await_finish),
+        #[cfg(has_drtio)]
+        api!(subkernel_send_message = subkernel::send_message),
+        #[cfg(has_drtio)]
+        api!(subkernel_await_message = subkernel::await_message),
+
         // Double-precision floating-point arithmetic helper functions
         // RTABI chapter 4.1.2, Table 2
         api!(__aeabi_dadd),

src/libksupport/src/kernel/mod.rs

@@ -3,7 +3,7 @@ use core::ptr;
 
 use libcortex_a9::{mutex::Mutex, semaphore::Semaphore, sync_channel};
 
-use crate::eh_artiq;
+use crate::{eh_artiq, RPCException};
 
 mod control;
 pub use control::Control;
@@ -13,16 +13,17 @@ mod dma;
 mod rpc;
 pub use dma::DmaRecorder;
 mod cache;
+#[cfg(has_drtio)]
+mod subkernel;
 
+#[cfg(has_drtio)]
 #[derive(Debug, Clone)]
-pub struct RPCException {
-    pub id: u32,
-    pub message: u32,
-    pub param: [i64; 3],
-    pub file: u32,
-    pub line: i32,
-    pub column: i32,
-    pub function: u32,
+pub enum SubkernelStatus {
+    NoError,
+    Timeout,
+    IncorrectState,
+    CommLost,
+    OtherError,
 }
 
 #[derive(Debug, Clone)]
@@ -72,6 +73,42 @@ pub enum Message {
     UpDestinationsRequest(i32),
     #[cfg(has_drtio)]
     UpDestinationsReply(bool),
+
+    #[cfg(has_drtio)]
+    SubkernelLoadRunRequest {
+        id: u32,
+        run: bool,
+    },
+    #[cfg(has_drtio)]
+    SubkernelLoadRunReply {
+        succeeded: bool,
+    },
+    #[cfg(has_drtio)]
+    SubkernelAwaitFinishRequest {
+        id: u32,
+        timeout: u64,
+    },
+    #[cfg(has_drtio)]
+    SubkernelAwaitFinishReply {
+        status: SubkernelStatus,
+    },
+    #[cfg(has_drtio)]
+    SubkernelMsgSend {
+        id: u32,
+        data: Vec<u8>,
+    },
+    #[cfg(has_drtio)]
+    SubkernelMsgSent,
+    #[cfg(has_drtio)]
+    SubkernelMsgRecvRequest {
+        id: u32,
+        timeout: u64,
+    },
+    #[cfg(has_drtio)]
+    SubkernelMsgRecvReply {
+        status: SubkernelStatus,
+        count: u8,
+    },
 }
 
 static CHANNEL_0TO1: Mutex<Option<sync_channel::Sender<'static, Message>>> = Mutex::new(None);

src/libksupport/src/kernel/subkernel.rs

@@ -0,0 +1,109 @@
+use alloc::vec::Vec;
+
+use cslice::CSlice;
+
+use super::{Message, SubkernelStatus, KERNEL_CHANNEL_0TO1, KERNEL_CHANNEL_1TO0};
+use crate::{artiq_raise, rpc::send_args};
+
+pub extern "C" fn load_run(id: u32, run: bool) {
+    unsafe {
+        KERNEL_CHANNEL_1TO0
+            .as_mut()
+            .unwrap()
+            .send(Message::SubkernelLoadRunRequest { id: id, run: run });
+    }
+    match unsafe { KERNEL_CHANNEL_0TO1.as_mut().unwrap() }.recv() {
+        Message::SubkernelLoadRunReply { succeeded: true } => (),
+        Message::SubkernelLoadRunReply { succeeded: false } => {
+            artiq_raise!("SubkernelError", "Error loading or running the subkernel")
+        }
+        _ => panic!("Expected SubkernelLoadRunReply after SubkernelLoadRunRequest!"),
+    }
+}
+
+pub extern "C" fn await_finish(id: u32, timeout: u64) {
+    unsafe {
+        KERNEL_CHANNEL_1TO0
+            .as_mut()
+            .unwrap()
+            .send(Message::SubkernelAwaitFinishRequest {
+                id: id,
+                timeout: timeout,
+            });
+    }
+    match unsafe { KERNEL_CHANNEL_0TO1.as_mut().unwrap() }.recv() {
+        Message::SubkernelAwaitFinishReply {
+            status: SubkernelStatus::NoError,
+        } => (),
+        Message::SubkernelAwaitFinishReply {
+            status: SubkernelStatus::IncorrectState,
+        } => artiq_raise!("SubkernelError", "Subkernel not running"),
+        Message::SubkernelAwaitFinishReply {
+            status: SubkernelStatus::Timeout,
+        } => artiq_raise!("SubkernelError", "Subkernel timed out"),
+        Message::SubkernelAwaitFinishReply {
+            status: SubkernelStatus::CommLost,
+        } => artiq_raise!("SubkernelError", "Lost communication with satellite"),
+        Message::SubkernelAwaitFinishReply {
+            status: SubkernelStatus::OtherError,
+        } => artiq_raise!("SubkernelError", "An error occurred during subkernel operation"),
+        _ => panic!("expected SubkernelAwaitFinishReply after SubkernelAwaitFinishRequest"),
+    }
+}
+
+pub extern "C" fn send_message(id: u32, count: u8, tag: &CSlice<u8>, data: *const *const ()) {
+    let mut buffer = Vec::<u8>::new();
+    send_args(&mut buffer, 0, tag.as_ref(), data).expect("RPC encoding failed");
+    // overwrite service tag, include how many tags are in the message
+    buffer[3] = count;
+    unsafe {
+        KERNEL_CHANNEL_1TO0.as_mut().unwrap().send(Message::SubkernelMsgSend {
+            id: id,
+            data: buffer[3..].to_vec(),
+        });
+    }
+    match unsafe { KERNEL_CHANNEL_0TO1.as_mut().unwrap() }.recv() {
+        Message::SubkernelMsgSent => (),
+        _ => panic!("expected SubkernelMsgSent after SubkernelMsgSend"),
+    }
+}
+
+pub extern "C" fn await_message(id: u32, timeout: u64, min: u8, max: u8) {
+    unsafe {
+        KERNEL_CHANNEL_1TO0
+            .as_mut()
+            .unwrap()
+            .send(Message::SubkernelMsgRecvRequest {
+                id: id,
+                timeout: timeout,
+            });
+    }
+    match unsafe { KERNEL_CHANNEL_0TO1.as_mut().unwrap() }.recv() {
+        Message::SubkernelMsgRecvReply {
+            status: SubkernelStatus::NoError,
+            count,
+        } => {
+            if min > count || count > max {
+                artiq_raise!("SubkernelError", "Received more or less arguments than required")
+            }
+        }
+        Message::SubkernelMsgRecvReply {
+            status: SubkernelStatus::IncorrectState,
+            ..
+        } => artiq_raise!("SubkernelError", "Subkernel not running"),
+        Message::SubkernelMsgRecvReply {
+            status: SubkernelStatus::Timeout,
+            ..
+        } => artiq_raise!("SubkernelError", "Subkernel timed out"),
+        Message::SubkernelMsgRecvReply {
+            status: SubkernelStatus::CommLost,
+            ..
+        } => artiq_raise!("SubkernelError", "Lost communication with satellite"),
+        Message::SubkernelMsgRecvReply {
+            status: SubkernelStatus::OtherError,
+            ..
+        } => artiq_raise!("SubkernelError", "An error occurred during subkernel operation"),
+        _ => panic!("expected SubkernelMsgRecvReply after SubkernelMsgRecvRequest"),
+    }
+    // RpcRecvRequest should be called after this to receive message data
+}

src/libksupport/src/lib.rs

@@ -0,0 +1,163 @@
+#![no_std]
+#![feature(c_variadic)]
+#![feature(const_btree_new)]
+#![feature(const_in_array_repeat_expressions)]
+#![feature(naked_functions)]
+#![feature(asm)]
+
+#[macro_use]
+extern crate alloc;
+
+use alloc::{collections::BTreeMap, string::String};
+
+use io::{Cursor, ProtoRead};
+use libasync::block_async;
+use libconfig::Config;
+use log::{error, warn};
+#[cfg(has_drtiosat)]
+pub use pl::csr::drtiosat as rtio_core;
+#[cfg(has_rtio_core)]
+pub use pl::csr::rtio_core;
+use void::Void;
+
+pub mod eh_artiq;
+pub mod i2c;
+pub mod irq;
+pub mod kernel;
+pub mod rpc;
+#[cfg(ki_impl = "csr")]
+#[path = "rtio_csr.rs"]
+pub mod rtio;
+#[cfg(ki_impl = "acp")]
+#[path = "rtio_acp.rs"]
+pub mod rtio;
+#[rustfmt::skip]
+#[path = "../../../build/pl.rs"]
+pub mod pl;
+
+
+#[derive(Debug, Clone)]
+pub struct RPCException {
+    pub id: u32,
+    pub message: u32,
+    pub param: [i64; 3],
+    pub file: u32,
+    pub line: i32,
+    pub column: i32,
+    pub function: u32,
+}
+
+pub static mut SEEN_ASYNC_ERRORS: u8 = 0;
+
+pub const ASYNC_ERROR_COLLISION: u8 = 1 << 0;
+pub const ASYNC_ERROR_BUSY: u8 = 1 << 1;
+pub const ASYNC_ERROR_SEQUENCE_ERROR: u8 = 1 << 2;
+
+pub unsafe fn get_async_errors() -> u8 {
+    let errors = SEEN_ASYNC_ERRORS;
+    SEEN_ASYNC_ERRORS = 0;
+    errors
+}
+
+fn wait_for_async_rtio_error() -> nb::Result<(), Void> {
+    unsafe {
+        #[cfg(has_rtio_core)]
+        let errors = rtio_core::async_error_read();
+        #[cfg(has_drtiosat)]
+        let errors = rtio_core::protocol_error_read();
+        if errors != 0 {
+            Ok(())
+        } else {
+            Err(nb::Error::WouldBlock)
+        }
+    }
+}
+
+pub async fn report_async_rtio_errors() {
+    loop {
+        let _ = block_async!(wait_for_async_rtio_error()).await;
+        unsafe {
+            #[cfg(has_rtio_core)]
+            let errors = rtio_core::async_error_read();
+            #[cfg(has_drtiosat)]
+            let errors = rtio_core::protocol_error_read();
+            if errors & ASYNC_ERROR_COLLISION != 0 {
+                let channel = rtio_core::collision_channel_read();
+                error!(
+                    "RTIO collision involving channel 0x{:04x}:{}",
+                    channel,
+                    resolve_channel_name(channel as u32)
+                );
+            }
+            if errors & ASYNC_ERROR_BUSY != 0 {
+                let channel = rtio_core::busy_channel_read();
+                error!(
+                    "RTIO busy error involving channel 0x{:04x}:{}",
+                    channel,
+                    resolve_channel_name(channel as u32)
+                );
+            }
+            if errors & ASYNC_ERROR_SEQUENCE_ERROR != 0 {
+                let channel = rtio_core::sequence_error_channel_read();
+                error!(
+                    "RTIO sequence error involving channel 0x{:04x}:{}",
+                    channel,
+                    resolve_channel_name(channel as u32)
+                );
+            }
+            SEEN_ASYNC_ERRORS = errors;
+            #[cfg(has_rtio_core)]
+            rtio_core::async_error_write(errors);
+            #[cfg(has_drtiosat)]
+            rtio_core::protocol_error_write(errors);
+        }
+    }
+}
+
+static mut RTIO_DEVICE_MAP: BTreeMap<u32, String> = BTreeMap::new();
+
+fn read_device_map(cfg: &Config) -> BTreeMap<u32, String> {
+    let mut device_map: BTreeMap<u32, String> = BTreeMap::new();
+    let _ = cfg
+        .read("device_map")
+        .and_then(|raw_bytes| {
+            let mut bytes_cr = Cursor::new(raw_bytes);
+            let size = bytes_cr.read_u32().unwrap();
+            for _ in 0..size {
+                let channel = bytes_cr.read_u32().unwrap();
+                let device_name = bytes_cr.read_string().unwrap();
+                if let Some(old_entry) = device_map.insert(channel, device_name.clone()) {
+                    warn!(
+                        "conflicting device map entries for RTIO channel {}: '{}' and '{}'",
+                        channel, old_entry, device_name
+                    );
+                }
+            }
+            Ok(())
+        })
+        .or_else(|err| {
+            warn!(
+                "error reading device map ({}), device names will not be available in RTIO error messages",
+                err
+            );
+            Err(err)
+        });
+    device_map
+}
+
+fn _resolve_channel_name(channel: u32, device_map: &BTreeMap<u32, String>) -> String {
+    match device_map.get(&channel) {
+        Some(val) => val.clone(),
+        None => String::from("unknown"),
+    }
+}
+
+pub fn resolve_channel_name(channel: u32) -> String {
+    _resolve_channel_name(channel, unsafe { &RTIO_DEVICE_MAP })
+}
+
+pub fn setup_device_map(cfg: &Config) {
+    unsafe {
+        RTIO_DEVICE_MAP = read_device_map(cfg);
+    }
+}

src/libksupport/src/rpc.rs

@@ -1,71 +1,62 @@
-use alloc::boxed::Box;
-use core::{future::Future, str};
+use core::str;
 
-use async_recursion::async_recursion;
 use byteorder::{ByteOrder, NativeEndian};
-use core_io::{Error, Write};
+use core_io::{Error, Read, Write};
 use cslice::{CMutSlice, CSlice};
-use io::proto::ProtoWrite;
-use libasync::smoltcp::TcpStream;
-use libboard_zynq::smoltcp;
+use io::{ProtoRead, ProtoWrite};
 use log::trace;
 
 use self::tag::{split_tag, Tag, TagIterator};
-use crate::proto_async;
 
 #[inline]
-fn round_up(val: usize, power_of_two: usize) -> usize {
+pub fn round_up(val: usize, power_of_two: usize) -> usize {
     assert!(power_of_two.is_power_of_two());
     let max_rem = power_of_two - 1;
     (val + max_rem) & (!max_rem)
 }
 
 #[inline]
-unsafe fn round_up_mut<T>(ptr: *mut T, power_of_two: usize) -> *mut T {
+pub unsafe fn round_up_mut<T>(ptr: *mut T, power_of_two: usize) -> *mut T {
     round_up(ptr as usize, power_of_two) as *mut T
 }
 
 #[inline]
-unsafe fn round_up_const<T>(ptr: *const T, power_of_two: usize) -> *const T {
+pub unsafe fn round_up_const<T>(ptr: *const T, power_of_two: usize) -> *const T {
     round_up(ptr as usize, power_of_two) as *const T
 }
 
 #[inline]
-unsafe fn align_ptr<T>(ptr: *const ()) -> *const T {
+pub unsafe fn align_ptr<T>(ptr: *const ()) -> *const T {
     round_up_const(ptr, core::mem::align_of::<T>()) as *const T
 }
 
 #[inline]
-unsafe fn align_ptr_mut<T>(ptr: *mut ()) -> *mut T {
+pub unsafe fn align_ptr_mut<T>(ptr: *mut ()) -> *mut T {
     round_up_mut(ptr, core::mem::align_of::<T>()) as *mut T
 }
 
-/// Reads (deserializes) `length` array or list elements of type `tag` from `stream`,
-/// writing them into the buffer given by `storage`.
-///
-/// `alloc` is used for nested allocations (if elements themselves contain
-/// lists/arrays), see [recv_value].
-#[async_recursion(?Send)]
-async unsafe fn recv_elements<F>(
-    stream: &TcpStream,
-    elt_tag: Tag<'async_recursion>,
+// versions for reader rather than TcpStream
+// they will be made into sync for satellite subkernels later
+unsafe fn recv_elements<F, R>(
+    reader: &mut R,
+    elt_tag: Tag,
     length: usize,
     storage: *mut (),
-    alloc: &(impl Fn(usize) -> F + 'async_recursion),
-) -> Result<(), smoltcp::Error>
+    alloc: &mut F,
+) -> Result<(), Error>
 where
-    F: Future<Output = *mut ()>,
+    F: FnMut(usize) -> *mut (),
+    R: Read + ?Sized,
 {
-    // List of simple types are special-cased in the protocol for performance.
     match elt_tag {
         Tag::Bool => {
             let dest = core::slice::from_raw_parts_mut(storage as *mut u8, length);
-            proto_async::read_chunk(stream, dest).await?;
+            reader.read_exact(dest)?;
         }
         Tag::Int32 => {
             let ptr = storage as *mut u32;
             let dest = core::slice::from_raw_parts_mut(ptr as *mut u8, length * 4);
-            proto_async::read_chunk(stream, dest).await?;
+            reader.read_exact(dest)?;
             drop(dest);
             let dest = core::slice::from_raw_parts_mut(ptr, length);
             NativeEndian::from_slice_u32(dest);
@@ -73,7 +64,7 @@ where
         Tag::Int64 | Tag::Float64 => {
             let ptr = storage as *mut u64;
             let dest = core::slice::from_raw_parts_mut(ptr as *mut u8, length * 8);
-            proto_async::read_chunk(stream, dest).await?;
+            reader.read_exact(dest)?;
             drop(dest);
             let dest = core::slice::from_raw_parts_mut(ptr, length);
             NativeEndian::from_slice_u64(dest);
@@ -81,27 +72,17 @@ where
         _ => {
             let mut data = storage;
             for _ in 0..length {
-                recv_value(stream, elt_tag, &mut data, alloc).await?
+                recv_value(reader, elt_tag, &mut data, alloc)?
             }
         }
     }
     Ok(())
 }
 
-/// Reads (deserializes) a value of type `tag` from `stream`, writing the results to
-/// the kernel-side buffer `data` (the passed pointer to which is incremented to point
-/// past the just-received data). For nested allocations (lists/arrays), `alloc` is
-/// invoked any number of times with the size of the required allocation as a parameter
-/// (which is assumed to be correctly aligned for all payload types).
-#[async_recursion(?Send)]
-async unsafe fn recv_value<F>(
-    stream: &TcpStream,
-    tag: Tag<'async_recursion>,
-    data: &mut *mut (),
-    alloc: &(impl Fn(usize) -> F + 'async_recursion),
-) -> Result<(), smoltcp::Error>
+unsafe fn recv_value<F, R>(reader: &mut R, tag: Tag, data: &mut *mut (), alloc: &mut F) -> Result<(), Error>
 where
-    F: Future<Output = *mut ()>,
+    F: FnMut(usize) -> *mut (),
+    R: Read + ?Sized,
 {
     macro_rules! consume_value {
         ($ty:ty, | $ptr:ident | $map:expr) => {{
@@ -114,22 +95,22 @@ where
     match tag {
         Tag::None => Ok(()),
         Tag::Bool => consume_value!(i8, |ptr| {
-            *ptr = proto_async::read_i8(stream).await?;
+            *ptr = reader.read_u8()? as i8;
             Ok(())
         }),
         Tag::Int32 => consume_value!(i32, |ptr| {
-            *ptr = proto_async::read_i32(stream).await?;
+            *ptr = reader.read_u32()? as i32;
             Ok(())
         }),
         Tag::Int64 | Tag::Float64 => consume_value!(i64, |ptr| {
-            *ptr = proto_async::read_i64(stream).await?;
+            *ptr = reader.read_u64()? as i64;
             Ok(())
         }),
         Tag::String | Tag::Bytes | Tag::ByteArray => {
             consume_value!(CMutSlice<u8>, |ptr| {
-                let length = proto_async::read_i32(stream).await? as usize;
-                *ptr = CMutSlice::new(alloc(length).await as *mut u8, length);
-                proto_async::read_chunk(stream, (*ptr).as_mut()).await?;
+                let length = reader.read_u32()? as usize;
+                *ptr = CMutSlice::new(alloc(length) as *mut u8, length);
+                reader.read_exact((*ptr).as_mut())?;
                 Ok(())
             })
         }
@@ -139,10 +120,8 @@ where
             let mut it = it.clone();
             for _ in 0..arity {
                 let tag = it.next().expect("truncated tag");
-                recv_value(stream, tag, data, alloc).await?
+                recv_value(reader, tag, data, alloc)?
             }
-            // Take into account any tail padding (if element(s) with largest alignment
-            // are not at the end).
             *data = round_up_mut(*data, alignment);
             Ok(())
         }
@@ -154,50 +133,41 @@ where
             }
             consume_value!(*mut List, |ptr_to_list| {
                 let tag = it.clone().next().expect("truncated tag");
-                let length = proto_async::read_i32(stream).await? as usize;
+                let length = reader.read_u32()? as usize;
 
-                // To avoid multiple kernel CPU roundtrips, use a single allocation for
-                // both the pointer/length List (slice) and the backing storage for the
-                // elements. We can assume that alloc() is aligned suitably, so just
-                // need to take into account any extra padding required.
-                // (Note: At the time of writing, there will never actually be any types
-                // with alignment larger than 8 bytes, so storage_offset == 0 always.)
                 let list_size = 4 + 4;
                 let storage_offset = round_up(list_size, tag.alignment());
                 let storage_size = tag.size() * length;
 
-                let allocation = alloc(storage_offset + storage_size).await as *mut u8;
+                let allocation = alloc(storage_offset + storage_size) as *mut u8;
                 *ptr_to_list = allocation as *mut List;
                 let storage = allocation.offset(storage_offset as isize) as *mut ();
 
                 (**ptr_to_list).length = length;
                 (**ptr_to_list).elements = storage;
-                recv_elements(stream, tag, length, storage, alloc).await
+                recv_elements(reader, tag, length, storage, alloc)
             })
         }
         Tag::Array(it, num_dims) => {
             consume_value!(*mut (), |buffer| {
-                // Deserialize length along each dimension and compute total number of
-                // elements.
                 let mut total_len: usize = 1;
                 for _ in 0..num_dims {
-                    let len = proto_async::read_i32(stream).await? as usize;
+                    let len = reader.read_u32()? as usize;
                     total_len *= len;
                     consume_value!(usize, |ptr| *ptr = len)
                 }
 
-                // Allocate backing storage for elements; deserialize them.
                 let elt_tag = it.clone().next().expect("truncated tag");
-                *buffer = alloc(elt_tag.size() * total_len).await;
-                recv_elements(stream, elt_tag, total_len, *buffer, alloc).await
+                *buffer = alloc(elt_tag.size() * total_len);
+                recv_elements(reader, elt_tag, total_len, *buffer, alloc)
             })
         }
         Tag::Range(it) => {
             *data = round_up_mut(*data, tag.alignment());
             let tag = it.clone().next().expect("truncated tag");
-            recv_value(stream, tag, data, alloc).await?;
-            recv_value(stream, tag, data, alloc).await?;
-            recv_value(stream, tag, data, alloc).await?;
+            recv_value(reader, tag, data, alloc)?;
+            recv_value(reader, tag, data, alloc)?;
+            recv_value(reader, tag, data, alloc)?;
             Ok(())
         }
         Tag::Keyword(_) => unreachable!(),
@@ -205,21 +175,17 @@ where
     }
 }
 
-pub async fn recv_return<F>(
-    stream: &TcpStream,
-    tag_bytes: &[u8],
-    data: *mut (),
-    alloc: &impl Fn(usize) -> F,
-) -> Result<(), smoltcp::Error>
+pub fn recv_return<F, R>(reader: &mut R, tag_bytes: &[u8], data: *mut (), alloc: &mut F) -> Result<(), Error>
 where
-    F: Future<Output = *mut ()>,
+    F: FnMut(usize) -> *mut (),
+    R: Read + ?Sized,
 {
     let mut it = TagIterator::new(tag_bytes);
     trace!("recv ...->{}", it);
 
     let tag = it.next().expect("truncated tag");
     let mut data = data;
-    unsafe { recv_value(stream, tag, &mut data, alloc).await? };
+    unsafe { recv_value(reader, tag, &mut data, alloc)? };
 
     Ok(())
 }
@@ -367,7 +333,7 @@ where W: Write + ?Sized {
     Ok(())
 }
 
-mod tag {
+pub mod tag {
     use core::fmt;
 
     pub fn split_tag(tag_bytes: &[u8]) -> (&[u8], &[u8]) {

src/libksupport/src/rtio_acp.rs

@@ -4,7 +4,7 @@ use cslice::CSlice;
 use libcortex_a9::asm;
 use vcell::VolatileCell;
 
-use crate::{artiq_raise, pl::csr, rtio_mgt::resolve_channel_name};
+use crate::{artiq_raise, pl::csr, resolve_channel_name, rtio_core};
 
 pub const RTIO_O_STATUS_WAIT: i32 = 1;
 pub const RTIO_O_STATUS_UNDERFLOW: i32 = 2;
@@ -52,7 +52,7 @@ static mut TRANSACTION_BUFFER: Transaction = Transaction {
 
 pub extern "C" fn init() {
     unsafe {
-        csr::rtio_core::reset_write(1);
+        rtio_core::reset_write(1);
         csr::rtio::engine_addr_base_write(&TRANSACTION_BUFFER as *const Transaction as u32);
         csr::rtio::enable_write(1);
     }

src/libksupport/src/rtio_csr.rs

@@ -2,7 +2,7 @@ use core::ptr::{read_volatile, write_volatile};
 
 use cslice::CSlice;
 
-use crate::{artiq_raise, pl::csr, rtio_mgt::resolve_channel_name};
+use crate::{artiq_raise, pl::csr, resolve_channel_name, rtio_core};
 
 pub const RTIO_O_STATUS_WAIT: u8 = 1;
 pub const RTIO_O_STATUS_UNDERFLOW: u8 = 2;
@@ -20,7 +20,7 @@ pub struct TimestampedData {
 
 pub extern "C" fn init() {
     unsafe {
-        csr::rtio_core::reset_write(1);
+        rtio_core::reset_write(1);
     }
 }
 

src/runtime/Cargo.toml

@@ -18,7 +18,6 @@ num-traits = { version = "0.2", default-features = false }
 num-derive = "0.3"
 cslice = "0.3"
 log = "0.4"
-nb = "0.1"
 embedded-hal = "0.2"
 core_io = { version = "0.1", features = ["collections"] }
 byteorder = { version = "1.3", default-features = false }
@@ -26,7 +25,6 @@ void = { version = "1", default-features = false }
 futures = { version = "0.3", default-features = false, features = ["async-await"] }
 async-recursion = "0.3"
 log_buffer = { version = "1.2" }
-libm = { version = "0.2", features = ["unstable"] }
 vcell = "0.1"
 
 libboard_zynq = { git = "https://git.m-labs.hk/M-Labs/zynq-rs.git", features = ["ipv6"]}
@@ -40,5 +38,6 @@ dyld = { path = "../libdyld" }
 dwarf = { path = "../libdwarf" }
 unwind = { path = "../libunwind" }
 libc = { path = "../libc" }
-io = { path = "../libio" }
+io = { path = "../libio", features = ["alloc"] }
+ksupport = { path = "../libksupport" }
 libboard_artiq = { path = "../libboard_artiq" }

src/runtime/src/comms.rs

@@ -1,8 +1,14 @@
 use alloc::{collections::BTreeMap, rc::Rc, string::String, vec, vec::Vec};
 use core::{cell::RefCell, fmt, slice, str};
 
+use core_io::Error as IoError;
 use cslice::CSlice;
 use futures::{future::FutureExt, select_biased};
+#[cfg(has_drtio)]
+use io::{Cursor, ProtoRead};
+#[cfg(has_drtio)]
+use ksupport::rpc;
+use ksupport::{kernel, resolve_channel_name};
 use libasync::{smoltcp::{Sockets, TcpStream},
                task};
 use libboard_artiq::drtio_routing;
@@ -24,17 +30,19 @@ use num_traits::{FromPrimitive, ToPrimitive};
 
 #[cfg(has_drtio)]
 use crate::pl;
-use crate::{analyzer, kernel, mgmt, moninj,
-            proto_async::*,
-            rpc, rtio_dma,
-            rtio_mgt::{self, resolve_channel_name}};
+use crate::{analyzer, mgmt, moninj, proto_async::*, rpc_async, rtio_dma, rtio_mgt};
+#[cfg(has_drtio)]
+use crate::{subkernel, subkernel::Error as SubkernelError};
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub enum Error {
     NetworkError(smoltcp::Error),
+    IoError,
     UnexpectedPattern,
     UnrecognizedPacket,
     BufferExhausted,
+    #[cfg(has_drtio)]
+    SubkernelError(subkernel::Error),
 }
 
 pub type Result<T> = core::result::Result<T, Error>;
@@ -43,9 +51,12 @@ impl fmt::Display for Error {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         match self {
             Error::NetworkError(error) => write!(f, "network error: {}", error),
+            Error::IoError => write!(f, "io error"),
             Error::UnexpectedPattern => write!(f, "unexpected pattern"),
             Error::UnrecognizedPacket => write!(f, "unrecognized packet"),
             Error::BufferExhausted => write!(f, "buffer exhausted"),
+            #[cfg(has_drtio)]
+            Error::SubkernelError(error) => write!(f, "subkernel error: {:?}", error),
         }
     }
 }
@@ -56,6 +67,19 @@ impl From<smoltcp::Error> for Error {
     }
 }
 
+impl From<IoError> for Error {
+    fn from(_error: IoError) -> Self {
+        Error::IoError
+    }
+}
+
+#[cfg(has_drtio)]
+impl From<subkernel::Error> for Error {
+    fn from(error: subkernel::Error) -> Self {
+        Error::SubkernelError(error)
+    }
+}
+
 #[derive(Debug, FromPrimitive, ToPrimitive)]
 enum Request {
     SystemInfo = 3,
@@ -63,6 +87,7 @@ enum Request {
     RunKernel = 6,
     RPCReply = 7,
     RPCException = 8,
+    UploadSubkernel = 9,
 }
 
 #[derive(Debug, FromPrimitive, ToPrimitive)]
@@ -182,7 +207,7 @@ async fn handle_run_kernel(
                                 kernel::Message::RpcRecvRequest(slot) => slot,
                                 other => panic!("expected root value slot from core1, not {:?}", other),
                             };
-                            rpc::recv_return(stream, &tag, slot, &|size| {
+                            rpc_async::recv_return(stream, &tag, slot, &|size| {
                                 let control = control.clone();
                                 async move {
                                     if size == 0 {
@@ -227,7 +252,7 @@ async fn handle_run_kernel(
                             let function = read_i32(stream).await? as u32;
                             control
                                 .tx
-                                .async_send(kernel::Message::RpcRecvReply(Err(kernel::RPCException {
+                                .async_send(kernel::Message::RpcRecvReply(Err(ksupport::RPCException {
                                     id,
                                     message,
                                     param,
@@ -365,6 +390,124 @@ async fn handle_run_kernel(
                 control.borrow_mut().tx.async_send(reply).await;
             }
             #[cfg(has_drtio)]
+            kernel::Message::SubkernelLoadRunRequest { id, run } => {
+                let succeeded = match subkernel::load(aux_mutex, routing_table, timer, id, run).await {
+                    Ok(()) => true,
+                    Err(e) => {
+                        error!("Error loading subkernel: {:?}", e);
+                        false
+                    }
+                };
+                control
+                    .borrow_mut()
+                    .tx
+                    .async_send(kernel::Message::SubkernelLoadRunReply { succeeded: succeeded })
+                    .await;
+            }
+            #[cfg(has_drtio)]
+            kernel::Message::SubkernelAwaitFinishRequest { id, timeout } => {
+                let res = subkernel::await_finish(aux_mutex, routing_table, timer, id, timeout).await;
+                let status = match res {
+                    Ok(ref res) => {
+                        if res.status == subkernel::FinishStatus::CommLost {
+                            kernel::SubkernelStatus::CommLost
+                        } else if let Some(exception) = &res.exception {
+                            error!("Exception in subkernel");
+                            match stream {
+                                None => (),
+                                Some(stream) => {
+                                    write_chunk(stream, exception).await?;
+                                }
+                            }
+                            // will not be called after exception is served
+                            kernel::SubkernelStatus::OtherError
+                        } else {
+                            kernel::SubkernelStatus::NoError
+                        }
+                    }
+                    Err(SubkernelError::Timeout) => kernel::SubkernelStatus::Timeout,
+                    Err(SubkernelError::IncorrectState) => kernel::SubkernelStatus::IncorrectState,
+                    Err(_) => kernel::SubkernelStatus::OtherError,
+                };
+                control
+                    .borrow_mut()
+                    .tx
+                    .async_send(kernel::Message::SubkernelAwaitFinishReply { status: status })
+                    .await;
+            }
+            #[cfg(has_drtio)]
+            kernel::Message::SubkernelMsgSend { id, data } => {
+                let res = subkernel::message_send(aux_mutex, routing_table, timer, id, data).await;
+                match res {
+                    Ok(_) => (),
+                    Err(e) => {
+                        error!("error sending subkernel message: {:?}", e)
+                    }
+                };
+                control
+                    .borrow_mut()
+                    .tx
+                    .async_send(kernel::Message::SubkernelMsgSent)
+                    .await;
+            }
+            #[cfg(has_drtio)]
+            kernel::Message::SubkernelMsgRecvRequest { id, timeout } => {
+                let message_received = subkernel::message_await(id, timeout, timer).await;
+                let (status, count) = match message_received {
+                    Ok(ref message) => (kernel::SubkernelStatus::NoError, message.count),
+                    Err(SubkernelError::Timeout) => (kernel::SubkernelStatus::Timeout, 0),
+                    Err(SubkernelError::IncorrectState) => (kernel::SubkernelStatus::IncorrectState, 0),
+                    Err(SubkernelError::CommLost) => (kernel::SubkernelStatus::CommLost, 0),
+                    Err(_) => (kernel::SubkernelStatus::OtherError, 0),
+                };
+                control
+                    .borrow_mut()
+                    .tx
+                    .async_send(kernel::Message::SubkernelMsgRecvReply {
+                        status: status,
+                        count: count,
+                    })
+                    .await;
+                if let Ok(message) = message_received {
+                    // receive code almost identical to RPC recv, except we are not reading from a stream
+                    let mut reader = Cursor::new(message.data);
+                    let mut tag: [u8; 1] = [message.tag];
+                    let mut i = 0;
+                    loop {
+                        // kernel has to consume all arguments in the whole message
+                        let slot = match fast_recv(&mut control.borrow_mut().rx).await {
+                            kernel::Message::RpcRecvRequest(slot) => slot,
+                            other => panic!("expected root value slot from core1, not {:?}", other),
+                        };
+                        rpc::recv_return(&mut reader, &tag, slot, &mut |size| {
+                            if size == 0 {
+                                0 as *mut ()
+                            } else {
+                                let mut control = control.borrow_mut();
+                                control.tx.send(kernel::Message::RpcRecvReply(Ok(size)));
+                                match control.rx.recv() {
+                                    kernel::Message::RpcRecvRequest(slot) => slot,
+                                    other => {
+                                        panic!("expected nested value slot from kernel CPU, not {:?}", other)
+                                    }
+                                }
+                            }
+                        })?;
+                        control
+                            .borrow_mut()
+                            .tx
+                            .async_send(kernel::Message::RpcRecvReply(Ok(0)))
+                            .await;
+                        i += 1;
+                        if i < count {
+                            tag[0] = reader.read_u8()?;
+                        } else {
+                            break;
+                        }
+                    }
+                }
+            }
+            #[cfg(has_drtio)]
             kernel::Message::UpDestinationsRequest(destination) => {
                 let result = _up_destinations.borrow()[destination as usize];
                 control
@@ -434,9 +577,13 @@ async fn handle_connection(
         return Err(Error::UnexpectedPattern);
     }
     stream.send_slice("e".as_bytes()).await?;
+    #[cfg(has_drtio)]
+    subkernel::clear_subkernels().await;
     loop {
         let request = read_request(stream, true).await?;
         if request.is_none() {
+            #[cfg(has_drtio)]
+            subkernel::clear_subkernels().await;
             return Ok(());
         }
         let request = request.unwrap();
@@ -460,6 +607,29 @@ async fn handle_connection(
                 )
                 .await?;
             }
+            Request::UploadSubkernel => {
+                #[cfg(has_drtio)]
+                {
+                    let id = read_i32(stream).await? as u32;
+                    let destination = read_i8(stream).await? as u8;
+                    let buffer = read_bytes(stream, 1024 * 1024).await?;
+                    subkernel::add_subkernel(id, destination, buffer).await;
+                    match subkernel::upload(aux_mutex, routing_table, timer, id).await {
+                        Ok(_) => write_header(stream, Reply::LoadCompleted).await?,
+                        Err(_) => {
+                            write_header(stream, Reply::LoadFailed).await?;
+                            write_chunk(stream, b"subkernel failed to load").await?;
+                            return Err(Error::UnexpectedPattern);
+                        }
+                    }
+                }
+                #[cfg(not(has_drtio))]
+                {
+                    write_header(stream, Reply::LoadFailed).await?;
+                    write_chunk(stream, b"No DRTIO on this system, subkernels are not supported").await?;
+                    return Err(Error::UnexpectedPattern);
+                }
+            }
             _ => {
                 error!("unexpected request from host: {:?}", request);
                 return Err(Error::UnrecognizedPacket);
@@ -522,7 +692,8 @@ pub fn main(timer: GlobalTimer, cfg: Config) {
     #[cfg(has_drtio_routing)]
     drtio_routing::interconnect_disable_all();
 
-    rtio_mgt::startup(&aux_mutex, &drtio_routing_table, &up_destinations, timer, &cfg);
+    rtio_mgt::startup(&aux_mutex, &drtio_routing_table, &up_destinations, timer);
+    ksupport::setup_device_map(&cfg);
 
     analyzer::start(&aux_mutex, &drtio_routing_table, &up_destinations, timer);
     moninj::start(timer, &aux_mutex, &drtio_routing_table);

src/runtime/src/main.rs

@@ -2,12 +2,8 @@
 #![no_main]
 #![recursion_limit = "1024"] // for futures_util::select!
 #![feature(alloc_error_handler)]
-#![feature(panic_info_message)]
-#![feature(c_variadic)]
 #![feature(const_btree_new)]
-#![feature(const_in_array_repeat_expressions)]
-#![feature(naked_functions)]
-#![feature(asm)]
+#![feature(panic_info_message)]
 
 #[macro_use]
 extern crate alloc;
@@ -15,95 +11,32 @@ extern crate alloc;
 #[cfg(feature = "target_kasli_soc")]
 use core::cell::RefCell;
 
-use libasync::{block_async, task};
+use ksupport;
+use libasync::task;
 #[cfg(feature = "target_kasli_soc")]
 use libboard_artiq::io_expander;
 use libboard_artiq::{identifier_read, logger, pl};
+#[cfg(has_drtio_eem)]
+use libboard_artiq::drtio_eem;
 use libboard_zynq::{gic, mpcore, timer::GlobalTimer};
 use libconfig::Config;
 use libcortex_a9::l2c::enable_l2_cache;
 use libsupport_zynq::ram;
-use log::{error, info, warn};
-use nb;
-use void::Void;
-
-const ASYNC_ERROR_COLLISION: u8 = 1 << 0;
-const ASYNC_ERROR_BUSY: u8 = 1 << 1;
-const ASYNC_ERROR_SEQUENCE_ERROR: u8 = 1 << 2;
+use log::{info, warn};
 
 mod analyzer;
 mod comms;
-mod eh_artiq;
-mod i2c;
-mod irq;
-mod kernel;
+
 mod mgmt;
 mod moninj;
 mod panic;
 mod proto_async;
-mod rpc;
-#[cfg(ki_impl = "csr")]
-#[path = "rtio_csr.rs"]
-mod rtio;
-#[cfg(ki_impl = "acp")]
-#[path = "rtio_acp.rs"]
-mod rtio;
+mod rpc_async;
 mod rtio_clocking;
 mod rtio_dma;
 mod rtio_mgt;
-
-static mut SEEN_ASYNC_ERRORS: u8 = 0;
-
-pub unsafe fn get_async_errors() -> u8 {
-    let errors = SEEN_ASYNC_ERRORS;
-    SEEN_ASYNC_ERRORS = 0;
-    errors
-}
-
-fn wait_for_async_rtio_error() -> nb::Result<(), Void> {
-    unsafe {
-        if pl::csr::rtio_core::async_error_read() != 0 {
-            Ok(())
-        } else {
-            Err(nb::Error::WouldBlock)
-        }
-    }
-}
-
-async fn report_async_rtio_errors() {
-    loop {
-        let _ = block_async!(wait_for_async_rtio_error()).await;
-        unsafe {
-            let errors = pl::csr::rtio_core::async_error_read();
-            if errors & ASYNC_ERROR_COLLISION != 0 {
-                let channel = pl::csr::rtio_core::collision_channel_read();
-                error!(
-                    "RTIO collision involving channel 0x{:04x}:{}",
-                    channel,
-                    rtio_mgt::resolve_channel_name(channel as u32)
-                );
-            }
-            if errors & ASYNC_ERROR_BUSY != 0 {
-                let channel = pl::csr::rtio_core::busy_channel_read();
-                error!(
-                    "RTIO busy error involving channel 0x{:04x}:{}",
-                    channel,
-                    rtio_mgt::resolve_channel_name(channel as u32)
-                );
-            }
-            if errors & ASYNC_ERROR_SEQUENCE_ERROR != 0 {
-                let channel = pl::csr::rtio_core::sequence_error_channel_read();
-                error!(
-                    "RTIO sequence error involving channel 0x{:04x}:{}",
-                    channel,
-                    rtio_mgt::resolve_channel_name(channel as u32)
-                );
-            }
-            SEEN_ASYNC_ERRORS = errors;
-            pl::csr::rtio_core::async_error_write(errors);
-        }
-    }
-}
+#[cfg(has_drtio)]
+mod subkernel;
 
 #[cfg(feature = "target_kasli_soc")]
 async fn io_expanders_service(
@@ -143,9 +76,9 @@ pub fn main_core0() {
 
     info!("gateware ident: {}", identifier_read(&mut [0; 64]));
 
-    i2c::init();
+    ksupport::i2c::init();
     #[cfg(feature = "target_kasli_soc")]
-    let i2c_bus = unsafe { (i2c::I2C_BUS).as_mut().unwrap() };
+    let i2c_bus = unsafe { (ksupport::i2c::I2C_BUS).as_mut().unwrap() };
 
     #[cfg(feature = "target_kasli_soc")]
     let (mut io_expander0, mut io_expander1);
@@ -178,7 +111,10 @@ pub fn main_core0() {
 
     rtio_clocking::init(&mut timer, &cfg);
 
-    task::spawn(report_async_rtio_errors());
+    #[cfg(has_drtio_eem)]
+    drtio_eem::init(&mut timer, &cfg);
+
+    task::spawn(ksupport::report_async_rtio_errors());
 
     #[cfg(feature = "target_kasli_soc")]
     task::spawn(io_expanders_service(

src/runtime/src/rpc_async.rs

@@ -0,0 +1,197 @@
+use alloc::boxed::Box;
+use core::future::Future;
+
+use async_recursion::async_recursion;
+use byteorder::{ByteOrder, NativeEndian};
+use cslice::CMutSlice;
+use ksupport::rpc::{tag::{Tag, TagIterator},
+                    *};
+use libasync::smoltcp::TcpStream;
+use libboard_zynq::smoltcp;
+use log::trace;
+
+use crate::proto_async;
+
+/// Reads (deserializes) `length` array or list elements of type `tag` from `stream`,
+/// writing them into the buffer given by `storage`.
+///
+/// `alloc` is used for nested allocations (if elements themselves contain
+/// lists/arrays), see [recv_value].
+#[async_recursion(?Send)]
+async unsafe fn recv_elements<F>(
+    stream: &TcpStream,
+    elt_tag: Tag<'async_recursion>,
+    length: usize,
+    storage: *mut (),
+    alloc: &(impl Fn(usize) -> F + 'async_recursion),
+) -> Result<(), smoltcp::Error>
+where
+    F: Future<Output = *mut ()>,
+{
+    // List of simple types are special-cased in the protocol for performance.
+    match elt_tag {
+        Tag::Bool => {
+            let dest = core::slice::from_raw_parts_mut(storage as *mut u8, length);
+            proto_async::read_chunk(stream, dest).await?;
+        }
+        Tag::Int32 => {
+            let ptr = storage as *mut u32;
+            let dest = core::slice::from_raw_parts_mut(ptr as *mut u8, length * 4);
+            proto_async::read_chunk(stream, dest).await?;
+            drop(dest);
+            let dest = core::slice::from_raw_parts_mut(ptr, length);
+            NativeEndian::from_slice_u32(dest);
+        }
+        Tag::Int64 | Tag::Float64 => {
+            let ptr = storage as *mut u64;
+            let dest = core::slice::from_raw_parts_mut(ptr as *mut u8, length * 8);
+            proto_async::read_chunk(stream, dest).await?;
+            drop(dest);
+            let dest = core::slice::from_raw_parts_mut(ptr, length);
+            NativeEndian::from_slice_u64(dest);
+        }
+        _ => {
+            let mut data = storage;
+            for _ in 0..length {
+                recv_value(stream, elt_tag, &mut data, alloc).await?
+            }
+        }
+    }
+    Ok(())
+}
+
+/// Reads (deserializes) a value of type `tag` from `stream`, writing the results to
+/// the kernel-side buffer `data` (the passed pointer to which is incremented to point
+/// past the just-received data). For nested allocations (lists/arrays), `alloc` is
+/// invoked any number of times with the size of the required allocation as a parameter
+/// (which is assumed to be correctly aligned for all payload types).
+#[async_recursion(?Send)]
+async unsafe fn recv_value<F>(
+    stream: &TcpStream,
+    tag: Tag<'async_recursion>,
+    data: &mut *mut (),
+    alloc: &(impl Fn(usize) -> F + 'async_recursion),
+) -> Result<(), smoltcp::Error>
+where
+    F: Future<Output = *mut ()>,
+{
+    macro_rules! consume_value {
+        ($ty:ty, | $ptr:ident | $map:expr) => {{
+            let $ptr = align_ptr_mut::<$ty>(*data);
+            *data = $ptr.offset(1) as *mut ();
+            $map
+        }};
+    }
+
+    match tag {
+        Tag::None => Ok(()),
+        Tag::Bool => consume_value!(i8, |ptr| {
+            *ptr = proto_async::read_i8(stream).await?;
+            Ok(())
+        }),
+        Tag::Int32 => consume_value!(i32, |ptr| {
+            *ptr = proto_async::read_i32(stream).await?;
+            Ok(())
+        }),
+        Tag::Int64 | Tag::Float64 => consume_value!(i64, |ptr| {
+            *ptr = proto_async::read_i64(stream).await?;
+            Ok(())
+        }),
+        Tag::String | Tag::Bytes | Tag::ByteArray => {
+            consume_value!(CMutSlice<u8>, |ptr| {
+                let length = proto_async::read_i32(stream).await? as usize;
+                *ptr = CMutSlice::new(alloc(length).await as *mut u8, length);
+                proto_async::read_chunk(stream, (*ptr).as_mut()).await?;
+                Ok(())
+            })
+        }
+        Tag::Tuple(it, arity) => {
+            let alignment = tag.alignment();
+            *data = round_up_mut(*data, alignment);
+            let mut it = it.clone();
+            for _ in 0..arity {
+                let tag = it.next().expect("truncated tag");
+                recv_value(stream, tag, data, alloc).await?
+            }
+            // Take into account any tail padding (if element(s) with largest alignment
+            // are not at the end).
+            *data = round_up_mut(*data, alignment);
+            Ok(())
+        }
+        Tag::List(it) => {
+            #[repr(C)]
+            struct List {
+                elements: *mut (),
+                length: usize,
+            }
+            consume_value!(*mut List, |ptr_to_list| {
+                let tag = it.clone().next().expect("truncated tag");
+                let length = proto_async::read_i32(stream).await? as usize;
+
+                // To avoid multiple kernel CPU roundtrips, use a single allocation for
+                // both the pointer/length List (slice) and the backing storage for the
+                // elements. We can assume that alloc() is aligned suitably, so just
+                // need to take into account any extra padding required.
+                // (Note: At the time of writing, there will never actually be any types
+                // with alignment larger than 8 bytes, so storage_offset == 0 always.)
+                let list_size = 4 + 4;
+                let storage_offset = round_up(list_size, tag.alignment());
+                let storage_size = tag.size() * length;
+
+                let allocation = alloc(storage_offset + storage_size).await as *mut u8;
+                *ptr_to_list = allocation as *mut List;
+                let storage = allocation.offset(storage_offset as isize) as *mut ();
+
+                (**ptr_to_list).length = length;
+                (**ptr_to_list).elements = storage;
+                recv_elements(stream, tag, length, storage, alloc).await
+            })
+        }
+        Tag::Array(it, num_dims) => {
+            consume_value!(*mut (), |buffer| {
+                // Deserialize length along each dimension and compute total number of
+                // elements.
+                let mut total_len: usize = 1;
+                for _ in 0..num_dims {
+                    let len = proto_async::read_i32(stream).await? as usize;
+                    total_len *= len;
+                    consume_value!(usize, |ptr| *ptr = len)
+                }
+
+                // Allocate backing storage for elements; deserialize them.
+                let elt_tag = it.clone().next().expect("truncated tag");
+                *buffer = alloc(elt_tag.size() * total_len).await;
+                recv_elements(stream, elt_tag, total_len, *buffer, alloc).await
+            })
+        }
+        Tag::Range(it) => {
+            *data = round_up_mut(*data, tag.alignment());
+            let tag = it.clone().next().expect("truncated tag");
+            recv_value(stream, tag, data, alloc).await?;
+            recv_value(stream, tag, data, alloc).await?;
+            recv_value(stream, tag, data, alloc).await?;
+            Ok(())
+        }
+        Tag::Keyword(_) => unreachable!(),
+        Tag::Object => unreachable!(),
+    }
+}
+
+pub async fn recv_return<F>(
+    stream: &TcpStream,
+    tag_bytes: &[u8],
+    data: *mut (),
+    alloc: &impl Fn(usize) -> F,
+) -> Result<(), smoltcp::Error>
+where
+    F: Future<Output = *mut ()>,
+{
+    let mut it = TagIterator::new(tag_bytes);
+    trace!("recv ...->{}", it);
+
+    let tag = it.next().expect("truncated tag");
+    let mut data = data;
+    unsafe { recv_value(stream, tag, &mut data, alloc).await? };
+
+    Ok(())
+}

src/runtime/src/rtio_clocking.rs

@@ -1,4 +1,6 @@
 use embedded_hal::blocking::delay::DelayMs;
+#[cfg(has_si5324)]
+use ksupport::i2c;
 use libboard_artiq::pl;
 #[cfg(has_si5324)]
 use libboard_artiq::si5324;
@@ -8,9 +10,6 @@ use libboard_zynq::timer::GlobalTimer;
 use libconfig::Config;
 use log::{info, warn};
 
-#[cfg(has_si5324)]
-use crate::i2c;
-
 #[derive(Debug, PartialEq, Copy, Clone)]
 #[allow(non_camel_case_types)]
 pub enum RtioClock {
@@ -76,7 +75,7 @@ fn init_rtio(timer: &mut GlobalTimer) {
     }
     // if it's not locked, it will hang at the CSR.
 
-    timer.delay_ms(20); // wait for CPLL/QPLL/SYS PLL lock
+    timer.delay_ms(50); // wait for CPLL/QPLL/SYS PLL lock
     let clk = unsafe { pl::csr::sys_crg::current_clock_read() };
     if clk == 1 {
         info!("SYS CLK switched successfully");
@@ -95,7 +94,7 @@ fn init_drtio(timer: &mut GlobalTimer) {
         pl::csr::gt_drtio::stable_clkin_write(1);
     }
 
-    timer.delay_ms(20); // wait for CPLL/QPLL/SYS PLL lock
+    timer.delay_ms(50); // wait for CPLL/QPLL/SYS PLL lock
     let clk = unsafe { pl::csr::sys_crg::current_clock_read() };
     if clk == 1 {
         info!("SYS CLK switched successfully");
@@ -105,6 +104,9 @@ fn init_drtio(timer: &mut GlobalTimer) {
     unsafe {
         pl::csr::rtio_core::reset_phy_write(1);
         pl::csr::gt_drtio::txenable_write(0xffffffffu32 as _);
+
+        #[cfg(has_drtio_eem)]
+        pl::csr::eem_transceiver::txenable_write(0xffffffffu32 as _);
     }
 }
 
@@ -264,7 +266,10 @@ pub fn init(timer: &mut GlobalTimer, cfg: &Config) {
     {
         let i2c = unsafe { (&mut i2c::I2C_BUS).as_mut().unwrap() };
         match clk {
-            RtioClock::Ext0_Bypass => si5324::bypass(i2c, SI5324_EXT_INPUT, timer).expect("cannot bypass Si5324"),
+            RtioClock::Ext0_Bypass => {
+                info!("bypassing the PLL for RTIO clock");
+                si5324::bypass(i2c, SI5324_EXT_INPUT, timer).expect("cannot bypass Si5324")
+            }
             _ => setup_si5324(i2c, timer, clk),
         }
     }

src/runtime/src/rtio_dma.rs

@@ -2,14 +2,13 @@ use alloc::{collections::BTreeMap, rc::Rc, string::String, vec::Vec};
 #[cfg(has_drtio)]
 use core::mem;
 
+use ksupport::kernel::DmaRecorder;
 #[cfg(has_drtio)]
 use libasync::task;
 use libboard_artiq::drtio_routing::RoutingTable;
 use libboard_zynq::timer::GlobalTimer;
 use libcortex_a9::{cache::dcci_slice, mutex::Mutex};
 
-use crate::kernel::DmaRecorder;
-
 const ALIGNMENT: usize = 16 * 8;
 
 static DMA_RECORD_STORE: Mutex<BTreeMap<String, (u32, Vec<u8>, i64)>> = Mutex::new(BTreeMap::new());

src/runtime/src/rtio_mgt.rs

@@ -1,28 +1,25 @@
-use alloc::{collections::BTreeMap, rc::Rc, string::String};
+use alloc::rc::Rc;
 use core::cell::RefCell;
 
-use io::{Cursor, ProtoRead};
 use libboard_artiq::{drtio_routing, pl::csr};
 use libboard_zynq::timer::GlobalTimer;
-use libconfig::Config;
 use libcortex_a9::mutex::Mutex;
-use log::warn;
-
-static mut RTIO_DEVICE_MAP: BTreeMap<u32, String> = BTreeMap::new();
 
 #[cfg(has_drtio)]
 pub mod drtio {
     use alloc::vec::Vec;
 
     use embedded_hal::blocking::delay::DelayMs;
+    use ksupport::{resolve_channel_name, ASYNC_ERROR_BUSY, ASYNC_ERROR_COLLISION, ASYNC_ERROR_SEQUENCE_ERROR,
+                   SEEN_ASYNC_ERRORS};
     use libasync::{delay, task};
-    use libboard_artiq::{drtioaux::Error, drtioaux_async, drtioaux_async::Packet, drtioaux_proto::DMA_TRACE_MAX_SIZE};
+    use libboard_artiq::{drtioaux::Error, drtioaux_async, drtioaux_async::Packet,
+                         drtioaux_proto::MASTER_PAYLOAD_MAX_SIZE};
     use libboard_zynq::time::Milliseconds;
     use log::{error, info, warn};
 
     use super::*;
-    use crate::{analyzer::remote_analyzer::RemoteBuffer, rtio_dma::remote_dma, ASYNC_ERROR_BUSY,
-                ASYNC_ERROR_COLLISION, ASYNC_ERROR_SEQUENCE_ERROR, SEEN_ASYNC_ERRORS};
+    use crate::{analyzer::remote_analyzer::RemoteBuffer, rtio_dma::remote_dma, subkernel};
 
     pub fn startup(
         aux_mutex: &Rc<Mutex<bool>>,
@@ -44,7 +41,7 @@ pub mod drtio {
         unsafe { (csr::DRTIO[linkno].rx_up_read)() == 1 }
     }
 
-    async fn process_async_packets(packet: Packet) -> Option<Packet> {
+    async fn process_async_packets(aux_mutex: &Mutex<bool>, linkno: u8, packet: Packet) -> Option<Packet> {
         // returns None if an async packet has been consumed
         match packet {
             Packet::DmaPlaybackStatus {
@@ -57,6 +54,25 @@ pub mod drtio {
                 remote_dma::playback_done(id, destination, error, channel, timestamp).await;
                 None
             }
+            Packet::SubkernelFinished { id, with_exception } => {
+                subkernel::subkernel_finished(id, with_exception).await;
+                None
+            }
+            Packet::SubkernelMessage {
+                id,
+                destination: from,
+                last,
+                length,
+                data,
+            } => {
+                subkernel::message_handle_incoming(id, last, length as usize, &data).await;
+                // acknowledge receiving part of the message
+                let _lock = aux_mutex.async_lock().await;
+                drtioaux_async::send(linkno, &Packet::SubkernelMessageAck { destination: from })
+                    .await
+                    .unwrap();
+                None
+            }
             other => Some(other),
         }
     }
@@ -193,7 +209,7 @@ pub mod drtio {
         let _lock = aux_mutex.async_lock().await;
         match drtioaux_async::recv(linkno).await {
             Ok(Some(packet)) => {
-                if let Some(packet) = process_async_packets(packet).await {
+                if let Some(packet) = process_async_packets(aux_mutex, linkno, packet).await {
                     warn!("[LINK#{}] unsolicited aux packet: {:?}", linkno, packet);
                 }
             }
@@ -286,6 +302,8 @@ pub mod drtio {
                                         false,
                                     )
                                     .await;
+                                    subkernel::destination_changed(aux_mutex, routing_table, timer, destination, false)
+                                        .await;
                                 }
                                 Ok(Packet::DestinationOkReply) => (),
                                 Ok(Packet::DestinationSequenceErrorReply { channel }) => {
@@ -315,7 +333,7 @@ pub mod drtio {
                                     );
                                     unsafe { SEEN_ASYNC_ERRORS |= ASYNC_ERROR_BUSY };
                                 }
-                                Ok(packet) => match process_async_packets(packet).await {
+                                Ok(packet) => match process_async_packets(aux_mutex, linkno, packet).await {
                                     Some(packet) => {
                                         error!("[DEST#{}] received unexpected aux packet: {:?}", destination, packet)
                                     }
@@ -328,6 +346,7 @@ pub mod drtio {
                     } else {
                         destination_set_up(routing_table, up_destinations, destination, false).await;
                         remote_dma::destination_changed(aux_mutex, routing_table, timer, destination, false).await;
+                        subkernel::destination_changed(aux_mutex, routing_table, timer, destination, false).await;
                     }
                 } else {
                     if up_links[linkno as usize] {
@@ -347,6 +366,8 @@ pub mod drtio {
                                 init_buffer_space(destination as u8, linkno).await;
                                 remote_dma::destination_changed(aux_mutex, routing_table, timer, destination, true)
                                     .await;
+                                subkernel::destination_changed(aux_mutex, routing_table, timer, destination, true)
+                                    .await;
                             }
                             Ok(packet) => error!("[DEST#{}] received unexpected aux packet: {:?}", destination, packet),
                             Err(e) => error!("[DEST#{}] communication failed ({})", destination, e),
@@ -433,6 +454,36 @@ pub mod drtio {
         }
     }
 
+    async fn partition_data<PacketF, HandlerF>(
+        linkno: u8,
+        aux_mutex: &Rc<Mutex<bool>>,
+        timer: GlobalTimer,
+        data: &[u8],
+        packet_f: PacketF,
+        reply_handler_f: HandlerF,
+    ) -> Result<(), &'static str>
+    where
+        PacketF: Fn(&[u8; MASTER_PAYLOAD_MAX_SIZE], bool, usize) -> Packet,
+        HandlerF: Fn(&Packet) -> Result<(), &'static str>,
+    {
+        let mut i = 0;
+        while i < data.len() {
+            let mut slice: [u8; MASTER_PAYLOAD_MAX_SIZE] = [0; MASTER_PAYLOAD_MAX_SIZE];
+            let len: usize = if i + MASTER_PAYLOAD_MAX_SIZE < data.len() {
+                MASTER_PAYLOAD_MAX_SIZE
+            } else {
+                data.len() - i
+            } as usize;
+            let last = i + len == data.len();
+            slice[..len].clone_from_slice(&data[i..i + len]);
+            i += len;
+            let packet = packet_f(&slice, last, len);
+            let reply = aux_transact(aux_mutex, linkno, &packet, timer).await?;
+            reply_handler_f(&reply)?;
+        }
+        Ok(())
+    }
+
     pub async fn ddma_upload_trace(
         aux_mutex: &Rc<Mutex<bool>>,
         routing_table: &drtio_routing::RoutingTable,
@@ -442,44 +493,25 @@ pub mod drtio {
         trace: &Vec<u8>,
     ) -> Result<(), &'static str> {
         let linkno = routing_table.0[destination as usize][0] - 1;
-        let mut i = 0;
-        while i < trace.len() {
-            let mut trace_slice: [u8; DMA_TRACE_MAX_SIZE] = [0; DMA_TRACE_MAX_SIZE];
-            let len: usize = if i + DMA_TRACE_MAX_SIZE < trace.len() {
-                DMA_TRACE_MAX_SIZE
-            } else {
-                trace.len() - i
-            } as usize;
-            let last = i + len == trace.len();
-            trace_slice[..len].clone_from_slice(&trace[i..i + len]);
-            i += len;
-            let reply = aux_transact(
-                aux_mutex,
-                linkno,
-                &Packet::DmaAddTraceRequest {
-                    id: id,
-                    destination: destination,
-                    last: last,
-                    length: len as u16,
-                    trace: trace_slice,
-                },
-                timer,
-            )
-            .await;
-            match reply {
-                Ok(Packet::DmaAddTraceReply { succeeded: true }) => (),
-                Ok(Packet::DmaAddTraceReply { succeeded: false }) => {
-                    return Err("error adding trace on satellite");
-                }
-                Ok(_) => {
-                    return Err("adding DMA trace failed, unexpected aux packet");
-                }
-                Err(_) => {
-                    return Err("adding DMA trace failed, aux error");
-                }
-            }
-        }
-        Ok(())
+        partition_data(
+            linkno,
+            aux_mutex,
+            timer,
+            trace,
+            |slice, last, len| Packet::DmaAddTraceRequest {
+                id: id,
+                destination: destination,
+                last: last,
+                length: len as u16,
+                trace: *slice,
+            },
+            |reply| match reply {
+                Packet::DmaAddTraceReply { succeeded: true } => Ok(()),
+                Packet::DmaAddTraceReply { succeeded: false } => Err("error adding trace on satellite"),
+                _ => Err("adding DMA trace failed, unexpected aux packet"),
+            },
+        )
+        .await
     }
 
     pub async fn ddma_send_erase(
@@ -608,46 +640,122 @@ pub mod drtio {
         }
         Ok(remote_buffers)
     }
-}
 
-fn read_device_map(cfg: &Config) -> BTreeMap<u32, String> {
-    let mut device_map: BTreeMap<u32, String> = BTreeMap::new();
-    let _ = cfg
-        .read("device_map")
-        .and_then(|raw_bytes| {
-            let mut bytes_cr = Cursor::new(raw_bytes);
-            let size = bytes_cr.read_u32().unwrap();
-            for _ in 0..size {
-                let channel = bytes_cr.read_u32().unwrap();
-                let device_name = bytes_cr.read_string().unwrap();
-                if let Some(old_entry) = device_map.insert(channel, device_name.clone()) {
-                    warn!(
-                        "conflicting device map entries for RTIO channel {}: '{}' and '{}'",
-                        channel, old_entry, device_name
-                    );
-                }
-            }
-            Ok(())
-        })
-        .or_else(|err| {
-            warn!(
-                "error reading device map ({}), device names will not be available in RTIO error messages",
-                err
-            );
-            Err(err)
-        });
-    device_map
-}
-
-fn _resolve_channel_name(channel: u32, device_map: &BTreeMap<u32, String>) -> String {
-    match device_map.get(&channel) {
-        Some(val) => val.clone(),
-        None => String::from("unknown"),
+    pub async fn subkernel_upload(
+        aux_mutex: &Rc<Mutex<bool>>,
+        routing_table: &drtio_routing::RoutingTable,
+        timer: GlobalTimer,
+        id: u32,
+        destination: u8,
+        data: &Vec<u8>,
+    ) -> Result<(), &'static str> {
+        let linkno = routing_table.0[destination as usize][0] - 1;
+        partition_data(
+            linkno,
+            aux_mutex,
+            timer,
+            data,
+            |slice, last, len| Packet::SubkernelAddDataRequest {
+                id: id,
+                destination: destination,
+                last: last,
+                length: len as u16,
+                data: *slice,
+            },
+            |reply| match reply {
+                Packet::SubkernelAddDataReply { succeeded: true } => Ok(()),
+                Packet::SubkernelAddDataReply { succeeded: false } => Err("error adding subkernel on satellite"),
+                _ => Err("adding subkernel failed, unexpected aux packet"),
+            },
+        )
+        .await
     }
-}
 
-pub fn resolve_channel_name(channel: u32) -> String {
-    _resolve_channel_name(channel, unsafe { &RTIO_DEVICE_MAP })
+    pub async fn subkernel_load(
+        aux_mutex: &Rc<Mutex<bool>>,
+        routing_table: &drtio_routing::RoutingTable,
+        timer: GlobalTimer,
+        id: u32,
+        destination: u8,
+        run: bool,
+    ) -> Result<(), &'static str> {
+        let linkno = routing_table.0[destination as usize][0] - 1;
+        let reply = aux_transact(
+            aux_mutex,
+            linkno,
+            &Packet::SubkernelLoadRunRequest {
+                id: id,
+                destination: destination,
+                run: run,
+            },
+            timer,
+        )
+        .await?;
+        match reply {
+            Packet::SubkernelLoadRunReply { succeeded: true } => return Ok(()),
+            Packet::SubkernelLoadRunReply { succeeded: false } => return Err("error on subkernel run request"),
+            _ => return Err("received unexpected aux packet during subkernel run"),
+        }
+    }
+
+    pub async fn subkernel_retrieve_exception(
+        aux_mutex: &Rc<Mutex<bool>>,
+        routing_table: &drtio_routing::RoutingTable,
+        timer: GlobalTimer,
+        destination: u8,
+    ) -> Result<Vec<u8>, &'static str> {
+        let linkno = routing_table.0[destination as usize][0] - 1;
+        let mut remote_data: Vec<u8> = Vec::new();
+        loop {
+            let reply = aux_transact(
+                aux_mutex,
+                linkno,
+                &Packet::SubkernelExceptionRequest {
+                    destination: destination,
+                },
+                timer,
+            )
+            .await?;
+            match reply {
+                Packet::SubkernelException { last, length, data } => {
+                    remote_data.extend(&data[0..length as usize]);
+                    if last {
+                        return Ok(remote_data);
+                    }
+                }
+                _ => return Err("received unexpected aux packet during subkernel exception request"),
+            }
+        }
+    }
+
+    pub async fn subkernel_send_message(
+        aux_mutex: &Rc<Mutex<bool>>,
+        routing_table: &drtio_routing::RoutingTable,
+        timer: GlobalTimer,
+        id: u32,
+        destination: u8,
+        message: &[u8],
+    ) -> Result<(), &'static str> {
+        let linkno = routing_table.0[destination as usize][0] - 1;
+        partition_data(
+            linkno,
+            aux_mutex,
+            timer,
+            message,
+            |slice, last, len| Packet::SubkernelMessage {
+                destination: destination,
+                id: id,
+                last: last,
+                length: len as u16,
+                data: *slice,
+            },
+            |reply| match reply {
+                Packet::SubkernelMessageAck { .. } => Ok(()),
+                _ => Err("sending message to subkernel failed, unexpected aux packet"),
+            },
+        )
+        .await
+    }
 }
 
 #[cfg(not(has_drtio))]
@@ -671,11 +779,7 @@ pub fn startup(
     routing_table: &Rc<RefCell<drtio_routing::RoutingTable>>,
     up_destinations: &Rc<RefCell<[bool; drtio_routing::DEST_COUNT]>>,
     timer: GlobalTimer,
-    cfg: &Config,
 ) {
-    unsafe {
-        RTIO_DEVICE_MAP = read_device_map(cfg);
-    }
     drtio::startup(aux_mutex, routing_table, up_destinations, timer);
     unsafe {
         csr::rtio_core::reset_phy_write(1);

src/runtime/src/subkernel.rs

@@ -0,0 +1,289 @@
+use alloc::{collections::BTreeMap, rc::Rc, vec::Vec};
+
+use libasync::task;
+use libboard_artiq::{drtio_routing::RoutingTable, drtioaux_proto::MASTER_PAYLOAD_MAX_SIZE};
+use libboard_zynq::{time::Milliseconds, timer::GlobalTimer};
+use libcortex_a9::mutex::Mutex;
+use log::error;
+
+use crate::rtio_mgt::drtio;
+
+#[derive(Debug, PartialEq, Clone, Copy)]
+pub enum FinishStatus {
+    Ok,
+    CommLost,
+    Exception,
+}
+
+#[derive(Debug, PartialEq, Clone, Copy)]
+pub enum SubkernelState {
+    NotLoaded,
+    Uploaded,
+    Running,
+    Finished { status: FinishStatus },
+}
+
+#[derive(Debug, PartialEq, Eq, Clone, Copy)]
+pub enum Error {
+    Timeout,
+    IncorrectState,
+    SubkernelNotFound,
+    CommLost,
+    DrtioError(&'static str),
+}
+
+impl From<&'static str> for Error {
+    fn from(value: &'static str) -> Error {
+        Error::DrtioError(value)
+    }
+}
+
+pub struct SubkernelFinished {
+    pub id: u32,
+    pub status: FinishStatus,
+    pub exception: Option<Vec<u8>>,
+}
+
+struct Subkernel {
+    pub destination: u8,
+    pub data: Vec<u8>,
+    pub state: SubkernelState,
+}
+
+impl Subkernel {
+    pub fn new(destination: u8, data: Vec<u8>) -> Self {
+        Subkernel {
+            destination: destination,
+            data: data,
+            state: SubkernelState::NotLoaded,
+        }
+    }
+}
+
+static SUBKERNELS: Mutex<BTreeMap<u32, Subkernel>> = Mutex::new(BTreeMap::new());
+
+pub async fn add_subkernel(id: u32, destination: u8, kernel: Vec<u8>) {
+    SUBKERNELS
+        .async_lock()
+        .await
+        .insert(id, Subkernel::new(destination, kernel));
+}
+
+pub async fn upload(
+    aux_mutex: &Rc<Mutex<bool>>,
+    routing_table: &RoutingTable,
+    timer: GlobalTimer,
+    id: u32,
+) -> Result<(), Error> {
+    if let Some(subkernel) = SUBKERNELS.async_lock().await.get_mut(&id) {
+        drtio::subkernel_upload(
+            aux_mutex,
+            routing_table,
+            timer,
+            id,
+            subkernel.destination,
+            &subkernel.data,
+        )
+        .await?;
+        subkernel.state = SubkernelState::Uploaded;
+        Ok(())
+    } else {
+        Err(Error::SubkernelNotFound)
+    }
+}
+
+pub async fn load(
+    aux_mutex: &Rc<Mutex<bool>>,
+    routing_table: &RoutingTable,
+    timer: GlobalTimer,
+    id: u32,
+    run: bool,
+) -> Result<(), Error> {
+    if let Some(subkernel) = SUBKERNELS.async_lock().await.get_mut(&id) {
+        if subkernel.state != SubkernelState::Uploaded {
+            return Err(Error::IncorrectState);
+        }
+        drtio::subkernel_load(aux_mutex, routing_table, timer, id, subkernel.destination, run).await?;
+        if run {
+            subkernel.state = SubkernelState::Running;
+        }
+        Ok(())
+    } else {
+        Err(Error::SubkernelNotFound)
+    }
+}
+
+pub async fn clear_subkernels() {
+    SUBKERNELS.async_lock().await.clear();
+    MESSAGE_QUEUE.async_lock().await.clear();
+    CURRENT_MESSAGES.async_lock().await.clear();
+}
+
+pub async fn subkernel_finished(id: u32, with_exception: bool) {
+    // called upon receiving DRTIO SubkernelRunDone
+    // may be None if session ends and is cleared
+    if let Some(subkernel) = SUBKERNELS.async_lock().await.get_mut(&id) {
+        subkernel.state = SubkernelState::Finished {
+            status: match with_exception {
+                true => FinishStatus::Exception,
+                false => FinishStatus::Ok,
+            },
+        }
+    }
+}
+
+pub async fn destination_changed(
+    aux_mutex: &Rc<Mutex<bool>>,
+    routing_table: &RoutingTable,
+    timer: GlobalTimer,
+    destination: u8,
+    up: bool,
+) {
+    let mut locked_subkernels = SUBKERNELS.async_lock().await;
+    for (id, subkernel) in locked_subkernels.iter_mut() {
+        if subkernel.destination == destination {
+            if up {
+                match drtio::subkernel_upload(aux_mutex, routing_table, timer, *id, destination, &subkernel.data).await
+                {
+                    Ok(_) => subkernel.state = SubkernelState::Uploaded,
+                    Err(e) => error!("Error adding subkernel on destination {}: {}", destination, e),
+                }
+            } else {
+                subkernel.state = match subkernel.state {
+                    SubkernelState::Running => SubkernelState::Finished {
+                        status: FinishStatus::CommLost,
+                    },
+                    _ => SubkernelState::NotLoaded,
+                }
+            }
+        }
+    }
+}
+
+pub async fn await_finish(
+    aux_mutex: &Rc<Mutex<bool>>,
+    routing_table: &RoutingTable,
+    timer: GlobalTimer,
+    id: u32,
+    timeout: u64,
+) -> Result<SubkernelFinished, Error> {
+    match SUBKERNELS.async_lock().await.get(&id).unwrap().state {
+        SubkernelState::Running | SubkernelState::Finished { .. } => (),
+        _ => return Err(Error::IncorrectState),
+    }
+    let max_time = timer.get_time() + Milliseconds(timeout);
+    while timer.get_time() < max_time {
+        {
+            match SUBKERNELS.async_lock().await.get(&id).unwrap().state {
+                SubkernelState::Finished { .. } => break,
+                _ => (),
+            };
+        }
+        task::r#yield().await;
+    }
+    if timer.get_time() >= max_time {
+        error!("Remote subkernel finish await timed out");
+        return Err(Error::Timeout);
+    }
+    if let Some(subkernel) = SUBKERNELS.async_lock().await.get_mut(&id) {
+        match subkernel.state {
+            SubkernelState::Finished { status } => {
+                subkernel.state = SubkernelState::Uploaded;
+                Ok(SubkernelFinished {
+                    id: id,
+                    status: status,
+                    exception: if status == FinishStatus::Exception {
+                        Some(
+                            drtio::subkernel_retrieve_exception(aux_mutex, routing_table, timer, subkernel.destination)
+                                .await?,
+                        )
+                    } else {
+                        None
+                    },
+                })
+            }
+            _ => Err(Error::IncorrectState),
+        }
+    } else {
+        Err(Error::SubkernelNotFound)
+    }
+}
+
+pub struct Message {
+    from_id: u32,
+    pub count: u8,
+    pub tag: u8,
+    pub data: Vec<u8>,
+}
+
+// FIFO queue of messages
+static MESSAGE_QUEUE: Mutex<Vec<Message>> = Mutex::new(Vec::new());
+// currently under construction message(s) (can be from multiple sources)
+static CURRENT_MESSAGES: Mutex<BTreeMap<u32, Message>> = Mutex::new(BTreeMap::new());
+
+pub async fn message_handle_incoming(id: u32, last: bool, length: usize, data: &[u8; MASTER_PAYLOAD_MAX_SIZE]) {
+    // called when receiving a message from satellite
+    if SUBKERNELS.async_lock().await.get(&id).is_none() {
+        // do not add messages for non-existing or deleted subkernels
+        return;
+    }
+    let mut current_messages = CURRENT_MESSAGES.async_lock().await;
+    match current_messages.get_mut(&id) {
+        Some(message) => message.data.extend(&data[..length]),
+        None => {
+            current_messages.insert(
+                id,
+                Message {
+                    from_id: id,
+                    count: data[0],
+                    tag: data[1],
+                    data: data[2..length].to_vec(),
+                },
+            );
+        }
+    };
+    if last {
+        // when done, remove from working queue
+        MESSAGE_QUEUE
+            .async_lock()
+            .await
+            .push(current_messages.remove(&id).unwrap());
+    }
+}
+
+pub async fn message_await(id: u32, timeout: u64, timer: GlobalTimer) -> Result<Message, Error> {
+    match SUBKERNELS.async_lock().await.get(&id).unwrap().state {
+        SubkernelState::Finished {
+            status: FinishStatus::CommLost,
+        } => return Err(Error::CommLost),
+        SubkernelState::Running | SubkernelState::Finished { .. } => (),
+        _ => return Err(Error::IncorrectState),
+    }
+    let max_time = timer.get_time() + Milliseconds(timeout);
+    while timer.get_time() < max_time {
+        {
+            let mut message_queue = MESSAGE_QUEUE.async_lock().await;
+            for i in 0..message_queue.len() {
+                let msg = &message_queue[i];
+                if msg.from_id == id {
+                    let message = message_queue.remove(i);
+                    return Ok(message);
+                }
+            }
+        }
+        task::r#yield().await;
+    }
+    Err(Error::Timeout)
+}
+
+pub async fn message_send<'a>(
+    aux_mutex: &Rc<Mutex<bool>>,
+    routing_table: &RoutingTable,
+    timer: GlobalTimer,
+    id: u32,
+    message: Vec<u8>,
+) -> Result<(), Error> {
+    let destination = SUBKERNELS.async_lock().await.get(&id).unwrap().destination;
+    // rpc data prepared by the kernel core already
+    Ok(drtio::subkernel_send_message(aux_mutex, routing_table, timer, id, destination, &message).await?)
+}

src/satman/Cargo.toml

@@ -14,6 +14,8 @@ build_zynq = { path = "../libbuild_zynq" }
 
 [dependencies]
 log = { version = "0.4", default-features = false }
+core_io = { version = "0.1", features = ["collections"] }
+cslice = "0.3"
 embedded-hal = "0.2"
 
 libboard_zynq = { git = "https://git.m-labs.hk/M-Labs/zynq-rs.git", features = ["ipv6"]}
@@ -25,4 +27,6 @@ libconfig = { git = "https://git.m-labs.hk/M-Labs/zynq-rs.git", features = ["fat
 
 libboard_artiq = { path = "../libboard_artiq" }
 unwind = { path = "../libunwind" }
-libc = { path = "../libc" }
+libc = { path = "../libc" }
+io = { path = "../libio", features = ["alloc"] }
+ksupport = { path = "../libksupport" }

src/satman/src/analyzer.rs

@@ -1,6 +1,6 @@
 use core::cmp::min;
 
-use libboard_artiq::{drtioaux_proto::ANALYZER_MAX_SIZE, pl::csr};
+use libboard_artiq::{drtioaux_proto::SAT_PAYLOAD_MAX_SIZE, pl::csr};
 use libcortex_a9::cache;
 
 const BUFFER_SIZE: usize = 512 * 1024;
@@ -100,10 +100,10 @@ impl Analyzer {
         }
     }
 
-    pub fn get_data(&mut self, data_slice: &mut [u8; ANALYZER_MAX_SIZE]) -> AnalyzerSliceMeta {
+    pub fn get_data(&mut self, data_slice: &mut [u8; SAT_PAYLOAD_MAX_SIZE]) -> AnalyzerSliceMeta {
         let data = unsafe { &BUFFER.data[..] };
         let i = (self.data_pointer + self.sent_bytes) % BUFFER_SIZE;
-        let len = min(ANALYZER_MAX_SIZE, self.data_len - self.sent_bytes);
+        let len = min(SAT_PAYLOAD_MAX_SIZE, self.data_len - self.sent_bytes);
         let last = self.sent_bytes + len == self.data_len;
 
         if i + len >= BUFFER_SIZE {

src/satman/src/dma.rs

@@ -170,4 +170,8 @@ impl Manager {
             }
         }
     }
+
+    pub fn running(&self) -> bool {
+        self.state == ManagerState::Playback
+    }
 }

src/satman/src/main.rs

@@ -1,13 +1,15 @@
 #![no_std]
 #![no_main]
-#![feature(never_type, panic_info_message, asm, naked_functions)]
-#![feature(alloc_error_handler)]
+#![feature(alloc_error_handler, try_trait, never_type, panic_info_message)]
 
 #[macro_use]
 extern crate log;
-
+extern crate core_io;
+extern crate cslice;
 extern crate embedded_hal;
 
+extern crate io;
+extern crate ksupport;
 extern crate libboard_artiq;
 extern crate libboard_zynq;
 extern crate libcortex_a9;
@@ -18,8 +20,6 @@ extern crate unwind;
 
 extern crate alloc;
 
-use core::sync::atomic::{AtomicBool, Ordering};
-
 use analyzer::Analyzer;
 use dma::Manager as DmaManager;
 use embedded_hal::blocking::delay::DelayUs;
@@ -27,21 +27,22 @@ use embedded_hal::blocking::delay::DelayUs;
 use libboard_artiq::io_expander;
 #[cfg(has_si5324)]
 use libboard_artiq::si5324;
-use libboard_artiq::{drtio_routing, drtioaux, drtioaux_proto::ANALYZER_MAX_SIZE, identifier_read, logger, pl::csr};
+use libboard_artiq::{drtio_routing, drtioaux,
+                     drtioaux_proto::{MASTER_PAYLOAD_MAX_SIZE, SAT_PAYLOAD_MAX_SIZE},
+                     identifier_read, logger,
+                     pl::csr};
 #[cfg(feature = "target_kasli_soc")]
 use libboard_zynq::error_led::ErrorLED;
-use libboard_zynq::{gic, i2c::I2c, mpcore, print, println, stdio, time::Milliseconds, timer::GlobalTimer};
-use libcortex_a9::{asm, interrupt_handler,
-                   l2c::enable_l2_cache,
-                   notify_spin_lock,
-                   regs::{MPIDR, SP},
-                   spin_lock_yield};
-use libregister::{RegisterR, RegisterW};
+use libboard_zynq::{i2c::I2c, print, println, time::Milliseconds, timer::GlobalTimer};
+use libcortex_a9::{l2c::enable_l2_cache, regs::MPIDR};
+use libregister::RegisterR;
 use libsupport_zynq::ram;
+use subkernel::Manager as KernelManager;
 
 mod analyzer;
 mod dma;
 mod repeater;
+mod subkernel;
 
 fn drtiosat_reset(reset: bool) {
     unsafe {
@@ -98,6 +99,7 @@ fn process_aux_packet(
     i2c: &mut I2c,
     dma_manager: &mut DmaManager,
     analyzer: &mut Analyzer,
+    kernel_manager: &mut KernelManager,
 ) -> Result<(), drtioaux::Error> {
     // In the code below, *_chan_sel_write takes an u8 if there are fewer than 256 channels,
     // and u16 otherwise; hence the `as _` conversion.
@@ -116,11 +118,9 @@ fn process_aux_packet(
             drtioaux::send(0, &drtioaux::Packet::ResetAck)
         }
 
-        drtioaux::Packet::DestinationStatusRequest {
-            destination: _destination,
-        } => {
+        drtioaux::Packet::DestinationStatusRequest { destination } => {
             #[cfg(has_drtio_routing)]
-            let hop = _routing_table.0[_destination as usize][*_rank as usize];
+            let hop = _routing_table.0[destination as usize][*_rank as usize];
             #[cfg(not(has_drtio_routing))]
             let hop = 0;
 
@@ -133,13 +133,40 @@ fn process_aux_packet(
                     drtioaux::send(
                         0,
                         &drtioaux::Packet::DmaPlaybackStatus {
-                            destination: _destination,
+                            destination: destination,
                             id: status.id,
                             error: status.error,
                             channel: status.channel,
                             timestamp: status.timestamp,
                         },
                     )?;
+                } else if let Some(subkernel_finished) = kernel_manager.get_last_finished() {
+                    info!(
+                        "subkernel {} finished, with exception: {}",
+                        subkernel_finished.id, subkernel_finished.with_exception
+                    );
+                    drtioaux::send(
+                        0,
+                        &drtioaux::Packet::SubkernelFinished {
+                            id: subkernel_finished.id,
+                            with_exception: subkernel_finished.with_exception,
+                        },
+                    )?;
+                } else if kernel_manager.message_is_ready() {
+                    let mut data_slice: [u8; MASTER_PAYLOAD_MAX_SIZE] = [0; MASTER_PAYLOAD_MAX_SIZE];
+                    match kernel_manager.message_get_slice(&mut data_slice) {
+                        Some(meta) => drtioaux::send(
+                            0,
+                            &drtioaux::Packet::SubkernelMessage {
+                                destination: destination,
+                                id: kernel_manager.get_current_id().unwrap(),
+                                last: meta.last,
+                                length: meta.len as u16,
+                                data: data_slice,
+                            },
+                        )?,
+                        None => warn!("subkernel message is ready but no message is present"),
+                    }
                 } else {
                     let errors;
                     unsafe {
@@ -180,7 +207,7 @@ fn process_aux_packet(
                         let repno = hop - 1;
                         match _repeaters[repno].aux_forward(
                             &drtioaux::Packet::DestinationStatusRequest {
-                                destination: _destination,
+                                destination: destination,
                             },
                             timer,
                         ) {
@@ -450,7 +477,7 @@ fn process_aux_packet(
             destination: _destination,
         } => {
             forward!(_routing_table, _destination, *_rank, _repeaters, &packet, timer);
-            let mut data_slice: [u8; ANALYZER_MAX_SIZE] = [0; ANALYZER_MAX_SIZE];
+            let mut data_slice: [u8; SAT_PAYLOAD_MAX_SIZE] = [0; SAT_PAYLOAD_MAX_SIZE];
             let meta = analyzer.get_data(&mut data_slice);
             drtioaux::send(
                 0,
@@ -487,10 +514,98 @@ fn process_aux_packet(
             timestamp,
         } => {
             forward!(_routing_table, _destination, *_rank, _repeaters, &packet, timer);
-            let succeeded = dma_manager.playback(id, timestamp).is_ok();
+            let succeeded = if !kernel_manager.running() {
+                dma_manager.playback(id, timestamp).is_ok()
+            } else {
+                false
+            };
             drtioaux::send(0, &drtioaux::Packet::DmaPlaybackReply { succeeded: succeeded })
         }
 
+        drtioaux::Packet::SubkernelAddDataRequest {
+            destination: _destination,
+            id,
+            last,
+            length,
+            data,
+        } => {
+            forward!(_routing_table, _destination, *_rank, _repeaters, &packet, timer);
+            let succeeded = kernel_manager.add(id, last, &data, length as usize).is_ok();
+            drtioaux::send(0, &drtioaux::Packet::SubkernelAddDataReply { succeeded: succeeded })
+        }
+        drtioaux::Packet::SubkernelLoadRunRequest {
+            destination: _destination,
+            id,
+            run,
+        } => {
+            forward!(_routing_table, _destination, *_rank, _repeaters, &packet, timer);
+            let mut succeeded = kernel_manager.load(id).is_ok();
+            // allow preloading a kernel with delayed run
+            if run {
+                if dma_manager.running() {
+                    // cannot run kernel while DDMA is running
+                    succeeded = false;
+                } else {
+                    succeeded |= kernel_manager.run(id).is_ok();
+                }
+            }
+            drtioaux::send(0, &drtioaux::Packet::SubkernelLoadRunReply { succeeded: succeeded })
+        }
+        drtioaux::Packet::SubkernelExceptionRequest {
+            destination: _destination,
+        } => {
+            forward!(_routing_table, _destination, *_rank, _repeaters, &packet, timer);
+            let mut data_slice: [u8; SAT_PAYLOAD_MAX_SIZE] = [0; SAT_PAYLOAD_MAX_SIZE];
+            let meta = kernel_manager.exception_get_slice(&mut data_slice);
+            drtioaux::send(
+                0,
+                &drtioaux::Packet::SubkernelException {
+                    last: meta.last,
+                    length: meta.len,
+                    data: data_slice,
+                },
+            )
+        }
+        drtioaux::Packet::SubkernelMessage {
+            destination,
+            id: _id,
+            last,
+            length,
+            data,
+        } => {
+            forward!(_routing_table, destination, *_rank, _repeaters, &packet, timer);
+            kernel_manager.message_handle_incoming(last, length as usize, &data);
+            drtioaux::send(
+                0,
+                &drtioaux::Packet::SubkernelMessageAck {
+                    destination: destination,
+                },
+            )
+        }
+        drtioaux::Packet::SubkernelMessageAck {
+            destination: _destination,
+        } => {
+            forward!(_routing_table, _destination, *_rank, _repeaters, &packet, timer);
+            if kernel_manager.message_ack_slice() {
+                let mut data_slice: [u8; MASTER_PAYLOAD_MAX_SIZE] = [0; MASTER_PAYLOAD_MAX_SIZE];
+                if let Some(meta) = kernel_manager.message_get_slice(&mut data_slice) {
+                    drtioaux::send(
+                        0,
+                        &drtioaux::Packet::SubkernelMessage {
+                            destination: *_rank,
+                            id: kernel_manager.get_current_id().unwrap(),
+                            last: meta.last,
+                            length: meta.len as u16,
+                            data: data_slice,
+                        },
+                    )?;
+                } else {
+                    error!("Error receiving message slice");
+                }
+            }
+            Ok(())
+        }
+
         _ => {
             warn!("received unexpected aux packet");
             Ok(())
@@ -506,6 +621,7 @@ fn process_aux_packets(
     i2c: &mut I2c,
     dma_manager: &mut DmaManager,
     analyzer: &mut Analyzer,
+    kernel_manager: &mut KernelManager,
 ) {
     let result = drtioaux::recv(0).and_then(|packet| {
         if let Some(packet) = packet {
@@ -518,6 +634,7 @@ fn process_aux_packets(
                 i2c,
                 dma_manager,
                 analyzer,
+                kernel_manager,
             )
         } else {
             Ok(())
@@ -626,8 +743,8 @@ pub extern "C" fn main_core0() -> i32 {
 
     ram::init_alloc_core0();
 
-    let mut i2c = I2c::i2c0();
-    i2c.init().expect("I2C initialization failed");
+    ksupport::i2c::init();
+    let mut i2c = unsafe { (ksupport::i2c::I2C_BUS).as_mut().unwrap() };
 
     #[cfg(feature = "target_kasli_soc")]
     let (mut io_expander0, mut io_expander1);
@@ -682,6 +799,8 @@ pub extern "C" fn main_core0() -> i32 {
 
     let mut hardware_tick_ts = 0;
 
+    let mut control = ksupport::kernel::Control::start();
+
     loop {
         while !drtiosat_link_rx_up() {
             drtiosat_process_errors();
@@ -709,12 +828,12 @@ pub extern "C" fn main_core0() -> i32 {
             si5324::siphaser::calibrate_skew(&mut timer).expect("failed to calibrate skew");
         }
 
-        // DMA manager created here, so when link is dropped, all DMA traces
+        // Various managers created here, so when link is dropped, all DMA traces
         // are cleared out for a clean slate on subsequent connections,
         // without a manual intervention.
         let mut dma_manager = DmaManager::new();
-        // same for RTIO Analyzer
         let mut analyzer = Analyzer::new();
+        let mut kernel_manager = KernelManager::new(&mut control);
 
         drtioaux::reset(0);
         drtiosat_reset(false);
@@ -730,6 +849,7 @@ pub extern "C" fn main_core0() -> i32 {
                 &mut i2c,
                 &mut dma_manager,
                 &mut analyzer,
+                &mut kernel_manager,
             );
             #[allow(unused_mut)]
             for mut rep in repeaters.iter_mut() {
@@ -756,6 +876,7 @@ pub extern "C" fn main_core0() -> i32 {
                     error!("aux packet error: {:?}", e);
                 }
             }
+            kernel_manager.process_kern_requests(rank, timer);
         }
 
         drtiosat_reset_phy(true);
@@ -771,46 +892,8 @@ extern "C" {
     static mut __stack1_start: u32;
 }
 
-interrupt_handler!(IRQ, irq, __irq_stack0_start, __irq_stack1_start, {
-    if MPIDR.read().cpu_id() == 1 {
-        let mpcore = mpcore::RegisterBlock::mpcore();
-        let mut gic = gic::InterruptController::gic(mpcore);
-        let id = gic.get_interrupt_id();
-        if id.0 == 0 {
-            gic.end_interrupt(id);
-            asm::exit_irq();
-            SP.write(&mut __stack1_start as *mut _ as u32);
-            asm::enable_irq();
-            CORE1_RESTART.store(false, Ordering::Relaxed);
-            notify_spin_lock();
-            main_core1();
-        }
-        stdio::drop_uart();
-    }
-    loop {}
-});
-
 static mut PANICKED: [bool; 2] = [false; 2];
 
-static CORE1_RESTART: AtomicBool = AtomicBool::new(false);
-
-pub fn restart_core1() {
-    let mut interrupt_controller = gic::InterruptController::gic(mpcore::RegisterBlock::mpcore());
-    CORE1_RESTART.store(true, Ordering::Relaxed);
-    interrupt_controller.send_sgi(gic::InterruptId(0), gic::CPUCore::Core1.into());
-    while CORE1_RESTART.load(Ordering::Relaxed) {
-        spin_lock_yield();
-    }
-}
-
-#[no_mangle]
-pub fn main_core1() {
-    let mut interrupt_controller = gic::InterruptController::gic(mpcore::RegisterBlock::mpcore());
-    interrupt_controller.enable_interrupts();
-
-    loop {}
-}
-
 #[no_mangle]
 pub extern "C" fn exception(_vect: u32, _regs: *const u32, pc: u32, ea: u32) {
     fn hexdump(addr: u32) {
@@ -866,23 +949,3 @@ pub fn panic_fmt(info: &core::panic::PanicInfo) -> ! {
 
     loop {}
 }
-
-// linker symbols
-extern "C" {
-    static __text_start: u32;
-    static __text_end: u32;
-    static __exidx_start: u32;
-    static __exidx_end: u32;
-}
-
-#[no_mangle]
-extern "C" fn dl_unwind_find_exidx(_pc: *const u32, len_ptr: *mut u32) -> *const u32 {
-    let length;
-    let start: *const u32;
-    unsafe {
-        length = (&__exidx_end as *const u32).offset_from(&__exidx_start) as u32;
-        start = &__exidx_start;
-        *len_ptr = length;
-    }
-    start
-}

src/satman/src/subkernel.rs

@@ -0,0 +1,692 @@
+use alloc::{collections::{BTreeMap, VecDeque},
+            format,
+            string::{String, ToString},
+            vec::Vec};
+use core::{cmp::min, option::NoneError, slice, str};
+
+use core_io::{Error as IoError, Write};
+use cslice::AsCSlice;
+use io::{Cursor, ProtoRead, ProtoWrite};
+use ksupport::{eh_artiq, kernel, rpc};
+use libboard_artiq::{drtioaux_proto::{MASTER_PAYLOAD_MAX_SIZE, SAT_PAYLOAD_MAX_SIZE},
+                     pl::csr};
+use libboard_zynq::{time::Milliseconds, timer::GlobalTimer};
+use libcortex_a9::sync_channel::Receiver;
+use log::warn;
+
+#[derive(Debug, Clone, Copy, PartialEq)]
+enum KernelState {
+    Absent,
+    Loaded,
+    Running,
+    MsgAwait(Milliseconds),
+    MsgSending,
+}
+
+#[derive(Debug)]
+pub enum Error {
+    Load(String),
+    KernelNotFound,
+    Unexpected(String),
+    NoMessage,
+    AwaitingMessage,
+    SubkernelIoError,
+    KernelException(Sliceable),
+}
+
+impl From<NoneError> for Error {
+    fn from(_: NoneError) -> Error {
+        Error::KernelNotFound
+    }
+}
+
+impl From<IoError> for Error {
+    fn from(_value: IoError) -> Error {
+        Error::SubkernelIoError
+    }
+}
+
+impl From<()> for Error {
+    fn from(_: ()) -> Error {
+        Error::NoMessage
+    }
+}
+
+macro_rules! unexpected {
+    ($($arg:tt)*) => (return Err(Error::Unexpected(format!($($arg)*))));
+}
+
+/* represents data that has to be sent to Master */
+#[derive(Debug)]
+pub struct Sliceable {
+    it: usize,
+    data: Vec<u8>,
+}
+
+/* represents interkernel messages */
+struct Message {
+    count: u8,
+    tag: u8,
+    data: Vec<u8>,
+}
+
+#[derive(PartialEq)]
+enum OutMessageState {
+    NoMessage,
+    MessageReady,
+    MessageBeingSent,
+    MessageSent,
+    MessageAcknowledged,
+}
+
+/* for dealing with incoming and outgoing interkernel messages */
+struct MessageManager {
+    out_message: Option<Sliceable>,
+    out_state: OutMessageState,
+    in_queue: VecDeque<Message>,
+    in_buffer: Option<Message>,
+}
+
+// Per-run state
+struct Session {
+    id: u32,
+    kernel_state: KernelState,
+    last_exception: Option<Sliceable>,
+    messages: MessageManager,
+}
+
+impl Session {
+    pub fn new(id: u32) -> Session {
+        Session {
+            id: id,
+            kernel_state: KernelState::Absent,
+            last_exception: None,
+            messages: MessageManager::new(),
+        }
+    }
+
+    fn running(&self) -> bool {
+        match self.kernel_state {
+            KernelState::Absent | KernelState::Loaded => false,
+            KernelState::Running | KernelState::MsgAwait { .. } | KernelState::MsgSending => true,
+        }
+    }
+}
+
+#[derive(Debug)]
+struct KernelLibrary {
+    library: Vec<u8>,
+    complete: bool,
+}
+
+pub struct Manager<'a> {
+    kernels: BTreeMap<u32, KernelLibrary>,
+    session: Session,
+    control: &'a mut kernel::Control,
+    cache: BTreeMap<String, Vec<i32>>,
+    last_finished: Option<SubkernelFinished>,
+}
+
+pub struct SubkernelFinished {
+    pub id: u32,
+    pub with_exception: bool,
+}
+
+pub struct SliceMeta {
+    pub len: u16,
+    pub last: bool,
+}
+
+macro_rules! get_slice_fn {
+    ($name:tt, $size:expr) => {
+        pub fn $name(&mut self, data_slice: &mut [u8; $size]) -> SliceMeta {
+            if self.data.len() == 0 {
+                return SliceMeta { len: 0, last: true };
+            }
+            let len = min($size, self.data.len() - self.it);
+            let last = self.it + len == self.data.len();
+
+            data_slice[..len].clone_from_slice(&self.data[self.it..self.it + len]);
+            self.it += len;
+
+            SliceMeta {
+                len: len as u16,
+                last: last,
+            }
+        }
+    };
+}
+
+impl Sliceable {
+    pub fn new(data: Vec<u8>) -> Sliceable {
+        Sliceable { it: 0, data: data }
+    }
+
+    get_slice_fn!(get_slice_sat, SAT_PAYLOAD_MAX_SIZE);
+    get_slice_fn!(get_slice_master, MASTER_PAYLOAD_MAX_SIZE);
+}
+
+impl MessageManager {
+    pub fn new() -> MessageManager {
+        MessageManager {
+            out_message: None,
+            out_state: OutMessageState::NoMessage,
+            in_queue: VecDeque::new(),
+            in_buffer: None,
+        }
+    }
+
+    pub fn handle_incoming(&mut self, last: bool, length: usize, data: &[u8; MASTER_PAYLOAD_MAX_SIZE]) {
+        // called when receiving a message from master
+        match self.in_buffer.as_mut() {
+            Some(message) => message.data.extend(&data[..length]),
+            None => {
+                self.in_buffer = Some(Message {
+                    count: data[0],
+                    tag: data[1],
+                    data: data[2..length].to_vec(),
+                });
+            }
+        };
+        if last {
+            // when done, remove from working queue
+            self.in_queue.push_back(self.in_buffer.take().unwrap());
+        }
+    }
+
+    pub fn is_outgoing_ready(&mut self) -> bool {
+        // called by main loop, to see if there's anything to send, will send it afterwards
+        match self.out_state {
+            OutMessageState::MessageReady => {
+                self.out_state = OutMessageState::MessageBeingSent;
+                true
+            }
+            _ => false,
+        }
+    }
+
+    pub fn was_message_acknowledged(&mut self) -> bool {
+        match self.out_state {
+            OutMessageState::MessageAcknowledged => {
+                self.out_state = OutMessageState::NoMessage;
+                true
+            }
+            _ => false,
+        }
+    }
+
+    pub fn get_outgoing_slice(&mut self, data_slice: &mut [u8; MASTER_PAYLOAD_MAX_SIZE]) -> Option<SliceMeta> {
+        if self.out_state != OutMessageState::MessageBeingSent {
+            return None;
+        }
+        let meta = self.out_message.as_mut()?.get_slice_master(data_slice);
+        if meta.last {
+            // clear the message slot
+            self.out_message = None;
+            // notify kernel with a flag that message is sent
+            self.out_state = OutMessageState::MessageSent;
+        }
+        Some(meta)
+    }
+
+    pub fn ack_slice(&mut self) -> bool {
+        // returns whether or not there's more to be sent
+        match self.out_state {
+            OutMessageState::MessageBeingSent => true,
+            OutMessageState::MessageSent => {
+                self.out_state = OutMessageState::MessageAcknowledged;
+                false
+            }
+            _ => {
+                warn!("received unsolicited SubkernelMessageAck");
+                false
+            }
+        }
+    }
+
+    pub fn accept_outgoing(&mut self, message: Vec<u8>) -> Result<(), Error> {
+        // service tag skipped in kernel
+        self.out_message = Some(Sliceable::new(message));
+        self.out_state = OutMessageState::MessageReady;
+        Ok(())
+    }
+
+    pub fn get_incoming(&mut self) -> Option<Message> {
+        self.in_queue.pop_front()
+    }
+}
+
+impl<'a> Manager<'_> {
+    pub fn new(control: &mut kernel::Control) -> Manager {
+        Manager {
+            kernels: BTreeMap::new(),
+            session: Session::new(0),
+            control: control,
+            cache: BTreeMap::new(),
+            last_finished: None,
+        }
+    }
+
+    pub fn add(&mut self, id: u32, last: bool, data: &[u8], data_len: usize) -> Result<(), Error> {
+        let kernel = match self.kernels.get_mut(&id) {
+            Some(kernel) => {
+                if kernel.complete {
+                    // replace entry
+                    self.kernels.remove(&id);
+                    self.kernels.insert(
+                        id,
+                        KernelLibrary {
+                            library: Vec::new(),
+                            complete: false,
+                        },
+                    );
+                    self.kernels.get_mut(&id)?
+                } else {
+                    kernel
+                }
+            }
+            None => {
+                self.kernels.insert(
+                    id,
+                    KernelLibrary {
+                        library: Vec::new(),
+                        complete: false,
+                    },
+                );
+                self.kernels.get_mut(&id)?
+            }
+        };
+        kernel.library.extend(&data[0..data_len]);
+
+        kernel.complete = last;
+        Ok(())
+    }
+
+    pub fn running(&self) -> bool {
+        self.session.running()
+    }
+
+    pub fn get_current_id(&self) -> Option<u32> {
+        match self.running() {
+            true => Some(self.session.id),
+            false => None,
+        }
+    }
+
+    pub fn run(&mut self, id: u32) -> Result<(), Error> {
+        info!("starting subkernel #{}", id);
+        if self.session.kernel_state != KernelState::Loaded || self.session.id != id {
+            self.load(id)?;
+        }
+        self.session.kernel_state = KernelState::Running;
+        unsafe {
+            csr::cri_con::selected_write(2);
+        }
+
+        self.control.tx.send(kernel::Message::StartRequest);
+        Ok(())
+    }
+
+    pub fn message_handle_incoming(&mut self, last: bool, length: usize, slice: &[u8; MASTER_PAYLOAD_MAX_SIZE]) {
+        if !self.running() {
+            return;
+        }
+        self.session.messages.handle_incoming(last, length, slice);
+    }
+
+    pub fn message_get_slice(&mut self, slice: &mut [u8; MASTER_PAYLOAD_MAX_SIZE]) -> Option<SliceMeta> {
+        if !self.running() {
+            return None;
+        }
+        self.session.messages.get_outgoing_slice(slice)
+    }
+
+    pub fn message_ack_slice(&mut self) -> bool {
+        if !self.running() {
+            warn!("received unsolicited SubkernelMessageAck");
+            return false;
+        }
+        self.session.messages.ack_slice()
+    }
+
+    pub fn message_is_ready(&mut self) -> bool {
+        self.session.messages.is_outgoing_ready()
+    }
+
+    pub fn load(&mut self, id: u32) -> Result<(), Error> {
+        if self.session.id == id && self.session.kernel_state == KernelState::Loaded {
+            return Ok(());
+        }
+        if !self.kernels.get(&id)?.complete {
+            return Err(Error::KernelNotFound);
+        }
+        self.session = Session::new(id);
+        self.control.restart();
+
+        self.control
+            .tx
+            .send(kernel::Message::LoadRequest(self.kernels.get(&id)?.library.clone()));
+        let reply = self.control.rx.recv();
+        match reply {
+            kernel::Message::LoadCompleted => Ok(()),
+            kernel::Message::LoadFailed => Err(Error::Load("kernel load failed".to_string())),
+            _ => Err(Error::Load(format!(
+                "unexpected kernel CPU reply to load request: {:?}",
+                reply
+            ))),
+        }
+    }
+
+    pub fn exception_get_slice(&mut self, data_slice: &mut [u8; SAT_PAYLOAD_MAX_SIZE]) -> SliceMeta {
+        match self.session.last_exception.as_mut() {
+            Some(exception) => exception.get_slice_sat(data_slice),
+            None => SliceMeta { len: 0, last: true },
+        }
+    }
+
+    pub fn get_last_finished(&mut self) -> Option<SubkernelFinished> {
+        self.last_finished.take()
+    }
+
+    fn kernel_stop(&mut self) {
+        self.session.kernel_state = KernelState::Absent;
+        unsafe {
+            csr::cri_con::selected_write(0);
+        }
+    }
+
+    fn runtime_exception(&mut self, cause: Error) {
+        let raw_exception: Vec<u8> = Vec::new();
+        let mut writer = Cursor::new(raw_exception);
+        match write_exception(
+            &mut writer,
+            &[Some(eh_artiq::Exception {
+                id: 11, // SubkernelError, defined in ksupport
+                message: format!("in subkernel id {}: {:?}", self.session.id, cause).as_c_slice(),
+                param: [0, 0, 0],
+                file: file!().as_c_slice(),
+                line: line!(),
+                column: column!(),
+                function: format!("subkernel id {}", self.session.id).as_c_slice(),
+            })],
+            &[eh_artiq::StackPointerBacktrace {
+                stack_pointer: 0,
+                initial_backtrace_size: 0,
+                current_backtrace_size: 0,
+            }],
+            &[],
+            0,
+        ) {
+            Ok(_) => self.session.last_exception = Some(Sliceable::new(writer.into_inner())),
+            Err(_) => error!("Error writing exception data"),
+        }
+        self.kernel_stop();
+    }
+
+    pub fn process_kern_requests(&mut self, rank: u8, timer: GlobalTimer) {
+        if !self.running() {
+            return;
+        }
+
+        match self.process_external_messages(timer) {
+            Ok(()) => (),
+            Err(Error::AwaitingMessage) => return, // kernel still waiting, do not process kernel messages
+            Err(Error::KernelException(exception)) => {
+                self.session.last_exception = Some(exception);
+                self.last_finished = Some(SubkernelFinished {
+                    id: self.session.id,
+                    with_exception: true,
+                });
+            }
+            Err(e) => {
+                error!("Error while running processing external messages: {:?}", e);
+                self.runtime_exception(e);
+                self.last_finished = Some(SubkernelFinished {
+                    id: self.session.id,
+                    with_exception: true,
+                });
+            }
+        }
+
+        match self.process_kern_message(rank, timer) {
+            Ok(true) => {
+                self.last_finished = Some(SubkernelFinished {
+                    id: self.session.id,
+                    with_exception: false,
+                });
+            }
+            Ok(false) | Err(Error::NoMessage) => (),
+            Err(Error::KernelException(exception)) => {
+                self.session.last_exception = Some(exception);
+                self.last_finished = Some(SubkernelFinished {
+                    id: self.session.id,
+                    with_exception: true,
+                });
+            }
+            Err(e) => {
+                error!("Error while running kernel: {:?}", e);
+                self.runtime_exception(e);
+                self.last_finished = Some(SubkernelFinished {
+                    id: self.session.id,
+                    with_exception: true,
+                });
+            }
+        }
+    }
+
+    fn process_kern_message(&mut self, rank: u8, timer: GlobalTimer) -> Result<bool, Error> {
+        let reply = self.control.rx.try_recv()?;
+        match reply {
+            kernel::Message::KernelFinished(_async_errors) => {
+                self.kernel_stop();
+                return Ok(true);
+            }
+            kernel::Message::KernelException(exceptions, stack_pointers, backtrace, async_errors) => {
+                error!("exception in kernel");
+                for exception in exceptions {
+                    error!("{:?}", exception.unwrap());
+                }
+                error!("stack pointers: {:?}", stack_pointers);
+                error!("backtrace: {:?}", backtrace);
+                let buf: Vec<u8> = Vec::new();
+                let mut writer = Cursor::new(buf);
+                match write_exception(&mut writer, exceptions, stack_pointers, backtrace, async_errors) {
+                    Ok(()) => (),
+                    Err(_) => error!("Error writing exception data"),
+                }
+                self.kernel_stop();
+                return Err(Error::KernelException(Sliceable::new(writer.into_inner())));
+            }
+            kernel::Message::CachePutRequest(key, value) => {
+                self.cache.insert(key, value);
+            }
+            kernel::Message::CacheGetRequest(key) => {
+                const DEFAULT: Vec<i32> = Vec::new();
+                let value = self.cache.get(&key).unwrap_or(&DEFAULT).clone();
+                self.control.tx.send(kernel::Message::CacheGetReply(value));
+            }
+            kernel::Message::SubkernelMsgSend { id: _, data } => {
+                self.session.messages.accept_outgoing(data)?;
+                self.session.kernel_state = KernelState::MsgSending;
+            }
+            kernel::Message::SubkernelMsgRecvRequest { id: _, timeout } => {
+                let max_time = timer.get_time() + Milliseconds(timeout);
+                self.session.kernel_state = KernelState::MsgAwait(max_time);
+            }
+            kernel::Message::UpDestinationsRequest(destination) => {
+                self.control
+                    .tx
+                    .send(kernel::Message::UpDestinationsReply(destination == (rank as i32)));
+            }
+            _ => {
+                unexpected!("unexpected message from core1 while kernel was running: {:?}", reply);
+            }
+        }
+        Ok(false)
+    }
+
+    fn process_external_messages(&mut self, timer: GlobalTimer) -> Result<(), Error> {
+        match self.session.kernel_state {
+            KernelState::MsgAwait(timeout) => {
+                if timer.get_time() > timeout {
+                    self.control.tx.send(kernel::Message::SubkernelMsgRecvReply {
+                        status: kernel::SubkernelStatus::Timeout,
+                        count: 0,
+                    });
+                    self.session.kernel_state = KernelState::Running;
+                    return Ok(());
+                }
+                if let Some(message) = self.session.messages.get_incoming() {
+                    self.control.tx.send(kernel::Message::SubkernelMsgRecvReply {
+                        status: kernel::SubkernelStatus::NoError,
+                        count: message.count,
+                    });
+                    self.session.kernel_state = KernelState::Running;
+                    self.pass_message_to_kernel(&message, timer)
+                } else {
+                    Err(Error::AwaitingMessage)
+                }
+            }
+            KernelState::MsgSending => {
+                if self.session.messages.was_message_acknowledged() {
+                    self.session.kernel_state = KernelState::Running;
+                    self.control.tx.send(kernel::Message::SubkernelMsgSent);
+                    Ok(())
+                } else {
+                    Err(Error::AwaitingMessage)
+                }
+            }
+            _ => Ok(()),
+        }
+    }
+
+    fn pass_message_to_kernel(&mut self, message: &Message, timer: GlobalTimer) -> Result<(), Error> {
+        let mut reader = Cursor::new(&message.data);
+        let mut tag: [u8; 1] = [message.tag];
+        let mut i = message.count;
+        loop {
+            let slot = match recv_w_timeout(&mut self.control.rx, timer, 100)? {
+                kernel::Message::RpcRecvRequest(slot) => slot,
+                other => unexpected!("expected root value slot from core1, not {:?}", other),
+            };
+            let mut exception: Option<Sliceable> = None;
+            let mut unexpected: Option<String> = None;
+            rpc::recv_return(&mut reader, &tag, slot, &mut |size| {
+                if size == 0 {
+                    0 as *mut ()
+                } else {
+                    self.control.tx.send(kernel::Message::RpcRecvReply(Ok(size)));
+                    match recv_w_timeout(&mut self.control.rx, timer, 100) {
+                        Ok(kernel::Message::RpcRecvRequest(slot)) => slot,
+                        Ok(kernel::Message::KernelException(exceptions, stack_pointers, backtrace, async_errors)) => {
+                            let buf: Vec<u8> = Vec::new();
+                            let mut writer = Cursor::new(buf);
+                            match write_exception(&mut writer, exceptions, stack_pointers, backtrace, async_errors) {
+                                Ok(()) => {
+                                    exception = Some(Sliceable::new(writer.into_inner()));
+                                }
+                                Err(_) => {
+                                    unexpected = Some("Error writing exception data".to_string());
+                                }
+                            };
+                            0 as *mut ()
+                        }
+                        other => {
+                            unexpected = Some(format!("expected nested value slot from kernel CPU, not {:?}", other));
+                            0 as *mut ()
+                        }
+                    }
+                }
+            })?;
+            if let Some(exception) = exception {
+                self.kernel_stop();
+                return Err(Error::KernelException(exception));
+            } else if let Some(unexpected) = unexpected {
+                self.kernel_stop();
+                unexpected!("{}", unexpected);
+            }
+            self.control.tx.send(kernel::Message::RpcRecvReply(Ok(0)));
+            i -= 1;
+            if i == 0 {
+                break;
+            } else {
+                // update the tag for next read
+                tag[0] = reader.read_u8()?;
+            }
+        }
+        Ok(())
+    }
+}
+
+fn write_exception<W>(
+    writer: &mut W,
+    exceptions: &[Option<eh_artiq::Exception>],
+    stack_pointers: &[eh_artiq::StackPointerBacktrace],
+    backtrace: &[(usize, usize)],
+    async_errors: u8,
+) -> Result<(), Error>
+where
+    W: Write + ?Sized,
+{
+    /* header */
+    writer.write_bytes(&[0x5a, 0x5a, 0x5a, 0x5a, /*Reply::KernelException*/ 9])?;
+    writer.write_u32(exceptions.len() as u32)?;
+    for exception in exceptions.iter() {
+        let exception = exception.as_ref().unwrap();
+        writer.write_u32(exception.id)?;
+
+        if exception.message.len() == usize::MAX {
+            // exception with host string
+            writer.write_u32(u32::MAX)?;
+            writer.write_u32(exception.message.as_ptr() as u32)?;
+        } else {
+            let msg =
+                str::from_utf8(unsafe { slice::from_raw_parts(exception.message.as_ptr(), exception.message.len()) })
+                    .unwrap()
+                    .replace(
+                        "{rtio_channel_info:0}",
+                        &format!(
+                            "0x{:04x}:{}",
+                            exception.param[0],
+                            ksupport::resolve_channel_name(exception.param[0] as u32)
+                        ),
+                    );
+            writer.write_string(&msg)?;
+        }
+        writer.write_u64(exception.param[0] as u64)?;
+        writer.write_u64(exception.param[1] as u64)?;
+        writer.write_u64(exception.param[2] as u64)?;
+        writer.write_bytes(exception.file.as_ref())?;
+        writer.write_u32(exception.line)?;
+        writer.write_u32(exception.column)?;
+        writer.write_bytes(exception.function.as_ref())?;
+    }
+
+    for sp in stack_pointers.iter() {
+        writer.write_u32(sp.stack_pointer as u32)?;
+        writer.write_u32(sp.initial_backtrace_size as u32)?;
+        writer.write_u32(sp.current_backtrace_size as u32)?;
+    }
+    writer.write_u32(backtrace.len() as u32)?;
+    for &(addr, sp) in backtrace {
+        writer.write_u32(addr as u32)?;
+        writer.write_u32(sp as u32)?;
+    }
+    writer.write_u8(async_errors as u8)?;
+    Ok(())
+}
+
+fn recv_w_timeout(
+    rx: &mut Receiver<'_, kernel::Message>,
+    timer: GlobalTimer,
+    timeout: u64,
+) -> Result<kernel::Message, Error> {
+    let max_time = timer.get_time() + Milliseconds(timeout);
+    while timer.get_time() < max_time {
+        match rx.try_recv() {
+            Err(_) => (),
+            Ok(message) => return Ok(message),
+        }
+    }
+    Err(Error::NoMessage)
+}