lower opt level to avoid axi burst

Co-authored-by: newell <newell.jensen@gmail.com>
Co-committed-by: newell <newell.jensen@gmail.com>

README.md

@@ -59,10 +59,20 @@ Notes:
 - Firmware type must be either ``runtime`` for DRTIO-less or DRTIO master variants, or ``satman`` for DRTIO satellite.
 - If the board is connected to the local machine, use the ``local_run.sh`` script.
 
+Pre-Commit Hooks
+----------------
+
+You are strongly recommended to use the provided pre-commit hooks to automatically reformat files and check for non-optimal Rust/C/C++ practices. Run `pre-commit install` to install the hook and `pre-commit` will automatically run `cargo fmt`, `cargo clippy`, and `clang-format` for you.
+
+Several things to note:
+
+- If `cargo fmt`, `cargo clippy`, or `clang-format` returns an error, the pre-commit hook will fail. You should fix all errors before trying to commit again.
+- If `cargo fmt` or `clang-format` reformats some files, the pre-commit hook will also fail. You should review the changes and, if satisfied, try to commit again.
+
 License
 -------
 
-Copyright (C) 2019-2023 M-Labs Limited.
+Copyright (C) 2019-2024 M-Labs Limited.
 
 ARTIQ is free software: you can redistribute it and/or modify
 it under the terms of the GNU Lesser General Public License as published by

examples/ebaz4205/device_db.py

@@ -0,0 +1,70 @@
+core_addr = "192.168.1.57"
+
+device_db = {
+    "core": {
+        "type": "local",
+        "module": "artiq.coredevice.core",
+        "class": "Core",
+        "arguments": {
+            "host": core_addr,
+            "ref_period": 1e-9,
+            "target": "cortexa9",
+        },
+    },
+    "core_log": {
+        "type": "controller",
+        "host": "::1",
+        "port": 1068,
+        "command": "aqctl_corelog -p {port} --bind {bind} " + core_addr,
+    },
+    "core_moninj": {
+        "type": "controller",
+        "host": "::1",
+        "port_proxy": 1383,
+        "port": 1384,
+        "command": "aqctl_moninj_proxy --port-proxy {port_proxy} --port-control {port} --bind {bind} "
+        + core_addr,
+    },
+    "core_analyzer": {
+        "type": "controller",
+        "host": "::1",
+        "port_proxy": 1385,
+        "port": 1386,
+        "command": "aqctl_coreanalyzer_proxy --port-proxy {port_proxy} --port-control {port} --bind {bind} "
+        + core_addr,
+    },
+    "core_cache": {
+        "type": "local",
+        "module": "artiq.coredevice.cache",
+        "class": "CoreCache",
+    },
+    "core_dma": {"type": "local", "module": "artiq.coredevice.dma", "class": "CoreDMA"},
+    "led0": {
+        "type": "local",
+        "module": "artiq.coredevice.ttl",
+        "class": "TTLOut",
+        "arguments": {"channel": 0},
+    },
+    "led1": {
+        "type": "local",
+        "module": "artiq.coredevice.ttl",
+        "class": "TTLOut",
+        "arguments": {"channel": 1},
+    },
+}
+
+
+device_db.update(
+    spi0={
+        "type": "local",
+        "module": "artiq.coredevice.spi2",
+        "class": "SPIMaster",
+        "arguments": {"channel": 2},
+    },
+    dds0={
+        "type": "local",
+        "module": "artiq.coredevice.ad9834",
+        "class": "AD9834",
+        "arguments": {"spi_device": "spi0"},
+    },
+)

flake.lock

@@ -3,19 +3,19 @@
     "artiq": {
       "inputs": {
         "artiq-comtools": "artiq-comtools",
-        "mozilla-overlay": "mozilla-overlay",
         "nixpkgs": "nixpkgs",
+        "rust-overlay": "rust-overlay",
         "sipyco": "sipyco",
         "src-migen": "src-migen",
         "src-misoc": "src-misoc",
         "src-pythonparser": "src-pythonparser"
       },
       "locked": {
-        "lastModified": 1697537883,
+        "lastModified": 1729249162,
-        "narHash": "sha256-GfadmYHFkczltX+rPf08YpAHjYa/31ZmmVD578BcFow=",
+        "narHash": "sha256-o3UROeulca0x5LQ6PlVILKEnZfhLElUA6qnNJ4Uwjmk=",
         "ref": "refs/heads/master",
-        "rev": "b168f0bb4be1697ff100475c20ee304dcc31fcc2",
+        "rev": "02235b2d80780de2ce0bb3cea09fe05be191fa5b",
-        "revCount": 8573,
+        "revCount": 9039,
         "type": "git",
         "url": "https://github.com/m-labs/artiq.git"
       },
@@ -37,11 +37,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1693473687,
+        "lastModified": 1720768567,
-        "narHash": "sha256-BdLddCWbvoEyakcGwhph9b5dIU1iA0hCQV7KYgU8nos=",
+        "narHash": "sha256-3VoK7o5MtHtbHLrc6Pv+eQWFtaz5Gd/YWyV5TD3c5Ss=",
         "owner": "m-labs",
         "repo": "artiq-comtools",
-        "rev": "f522ef3dbc65961f17b2d3d41e927409d970fd79",
+        "rev": "f93570d8f2ed5a3cfb3e1c16ab00f2540551e994",
         "type": "github"
       },
       "original": {
@@ -55,11 +55,11 @@
         "systems": "systems"
       },
       "locked": {
-        "lastModified": 1692799911,
+        "lastModified": 1710146030,
-        "narHash": "sha256-3eihraek4qL744EvQXsK1Ha6C3CR7nnT8X2qWap4RNk=",
+        "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=",
         "owner": "numtide",
         "repo": "flake-utils",
-        "rev": "f9e7cf818399d17d347f847525c5a5a8032e4e44",
+        "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a",
         "type": "github"
       },
       "original": {
@@ -71,11 +71,11 @@
     "mozilla-overlay": {
       "flake": false,
       "locked": {
-        "lastModified": 1695805681,
+        "lastModified": 1704373101,
-        "narHash": "sha256-1ElPLD8eFfnuIk0G52HGGpRtQZ4QPCjChRlEOfkZ5ro=",
+        "narHash": "sha256-+gi59LRWRQmwROrmE1E2b3mtocwueCQqZ60CwLG+gbg=",
         "owner": "mozilla",
         "repo": "nixpkgs-mozilla",
-        "rev": "6eabade97bc28d707a8b9d82ad13ef143836736e",
+        "rev": "9b11a87c0cc54e308fa83aac5b4ee1816d5418a2",
         "type": "github"
       },
       "original": {
@@ -87,27 +87,11 @@
     "mozilla-overlay_2": {
       "flake": false,
       "locked": {
-        "lastModified": 1695805681,
+        "lastModified": 1704373101,
-        "narHash": "sha256-1ElPLD8eFfnuIk0G52HGGpRtQZ4QPCjChRlEOfkZ5ro=",
+        "narHash": "sha256-+gi59LRWRQmwROrmE1E2b3mtocwueCQqZ60CwLG+gbg=",
         "owner": "mozilla",
         "repo": "nixpkgs-mozilla",
-        "rev": "6eabade97bc28d707a8b9d82ad13ef143836736e",
+        "rev": "9b11a87c0cc54e308fa83aac5b4ee1816d5418a2",
-        "type": "github"
-      },
-      "original": {
-        "owner": "mozilla",
-        "repo": "nixpkgs-mozilla",
-        "type": "github"
-      }
-    },
-    "mozilla-overlay_3": {
-      "flake": false,
-      "locked": {
-        "lastModified": 1695805681,
-        "narHash": "sha256-1ElPLD8eFfnuIk0G52HGGpRtQZ4QPCjChRlEOfkZ5ro=",
-        "owner": "mozilla",
-        "repo": "nixpkgs-mozilla",
-        "rev": "6eabade97bc28d707a8b9d82ad13ef143836736e",
         "type": "github"
       },
       "original": {
@@ -118,16 +102,16 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1697226376,
+        "lastModified": 1728492678,
-        "narHash": "sha256-cumLLb1QOUtWieUnLGqo+ylNt3+fU8Lcv5Zl+tYbRUE=",
+        "narHash": "sha256-9UTxR8eukdg+XZeHgxW5hQA9fIKHsKCdOIUycTryeVw=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "898cb2064b6e98b8c5499f37e81adbdf2925f7c5",
+        "rev": "5633bcff0c6162b9e4b5f1264264611e950c8ec7",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "nixos-23.05",
+        "ref": "nixos-unstable",
         "repo": "nixpkgs",
         "type": "github"
       }
@@ -135,10 +119,32 @@
     "root": {
       "inputs": {
         "artiq": "artiq",
-        "mozilla-overlay": "mozilla-overlay_2",
+        "mozilla-overlay": "mozilla-overlay",
         "zynq-rs": "zynq-rs"
       }
     },
+    "rust-overlay": {
+      "inputs": {
+        "nixpkgs": [
+          "artiq",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1719454714,
+        "narHash": "sha256-MojqG0lyUINkEk0b3kM2drsU5vyaF8DFZe/FAlZVOGs=",
+        "owner": "oxalica",
+        "repo": "rust-overlay",
+        "rev": "d1c527659cf076ecc4b96a91c702d080b213801e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "oxalica",
+        "ref": "snapshot/2024-08-01",
+        "repo": "rust-overlay",
+        "type": "github"
+      }
+    },
     "sipyco": {
       "inputs": {
         "nixpkgs": [
@@ -147,11 +153,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1697528004,
+        "lastModified": 1728371104,
-        "narHash": "sha256-FFa2MbhAJEjwY58uOs0swvgymfjubHyWba6Q0X6CbB0=",
+        "narHash": "sha256-PPnAyDedUQ7Og/Cby9x5OT9wMkNGTP8GS53V6N/dk4w=",
         "owner": "m-labs",
         "repo": "sipyco",
-        "rev": "c0a7ed350ccfb85474217057fc47b3f258ca8d99",
+        "rev": "094a6cd63ffa980ef63698920170e50dc9ba77fd",
         "type": "github"
       },
       "original": {
@@ -163,11 +169,11 @@
     "src-migen": {
       "flake": false,
       "locked": {
-        "lastModified": 1697013661,
+        "lastModified": 1727677091,
-        "narHash": "sha256-qNCqgWyE4vTDmyjE2XMJqW1djuBxT25A36AzQfZqluU=",
+        "narHash": "sha256-Zg3SQnTwMM/VkOGKogbPyuCC2NhLy8HB2SPEUWWNgCU=",
         "owner": "m-labs",
         "repo": "migen",
-        "rev": "aadc19df93b7aa9ca761aaebbb98a11e0cf2d7c7",
+        "rev": "c19ae9f8ae162ffe2d310a92bfce53ac2a821bc8",
         "type": "github"
       },
       "original": {
@@ -179,11 +185,11 @@
     "src-misoc": {
       "flake": false,
       "locked": {
-        "lastModified": 1693709836,
+        "lastModified": 1728978817,
-        "narHash": "sha256-YiCk05RYLzZu1CYkQ2r7XtjwVEqkUGTQn388uOls9tI=",
+        "narHash": "sha256-b4633jrhh4i+KunZq4kNlyhdm9BCsEJwKs+6KINKV2o=",
         "ref": "refs/heads/master",
-        "rev": "58dc4ee60d165ce9145cf3d904241fc154b6407f",
+        "rev": "386b544776b66cea148da67d06a4b3a4151179f9",
-        "revCount": 2448,
+        "revCount": 2459,
         "submodules": true,
         "type": "git",
         "url": "https://github.com/m-labs/misoc.git"
@@ -227,18 +233,18 @@
     },
     "zynq-rs": {
       "inputs": {
-        "mozilla-overlay": "mozilla-overlay_3",
+        "mozilla-overlay": "mozilla-overlay_2",
         "nixpkgs": [
           "artiq",
           "nixpkgs"
         ]
       },
       "locked": {
-        "lastModified": 1697795161,
+        "lastModified": 1728110308,
-        "narHash": "sha256-p89w9eoFJ2VFTDZ5Mrv5vsH0E1Ko9z1C6Ett281hCHg=",
+        "narHash": "sha256-MAoFbcDgr+ZjptFCWfthK+tTnR1NcfuO6tvYhNM2Pwo=",
         "ref": "refs/heads/master",
-        "rev": "be672ab662d8134ee11412a651864824f6483d4a",
+        "rev": "cc20478d91e30e1448a4304df7003caed2981b71",
-        "revCount": 630,
+        "revCount": 651,
         "type": "git",
         "url": "https://git.m-labs.hk/m-labs/zynq-rs"
       },

flake.nix

@@ -17,11 +17,11 @@
 
     fastnumbers = pkgs.python3Packages.buildPythonPackage rec {
       pname = "fastnumbers";
-      version = "2.2.1";
+      version = "5.1.0";
 
       src = pkgs.python3Packages.fetchPypi {
         inherit pname version;
-        sha256 = "0j15i54p7nri6hkzn1wal9pxri4pgql01wgjccig6ar0v5jjbvsy";
+        sha256 = "sha256-4JLTP4uVwxcaL7NOV57+DFSwKQ3X+W/6onYkN2AdkKc=";
       };
     };
 
@@ -74,10 +74,11 @@
 
       propagatedBuildInputs = with pkgs.python3Packages; [ setuptools click numpy toolz jinja2 ramda artiqpkgs.migen artiqpkgs.misoc ];
 
-      checkInputs = with pkgs.python3Packages; [ pytest-runner pytestCheckHook pytest-timeout ];
+      checkInputs = with pkgs.python3Packages; [ pytestCheckHook pytest-timeout ];
 
       # migen/misoc version checks are broken with pyproject for some reason
       postPatch = ''
+        sed -i "1,4d" pyproject.toml
         substituteInPlace pyproject.toml \
           --replace '"migen@git+https://github.com/m-labs/migen",' ""
         substituteInPlace pyproject.toml \
@@ -112,7 +113,7 @@
       "nist_clock_satellite" "nist_qc2_satellite" "acpki_nist_clock_satellite" "acpki_nist_qc2_satellite" 
       "nist_clock_satellite_100mhz" "nist_qc2_satellite_100mhz" "acpki_nist_clock_satellite_100mhz" "acpki_nist_qc2_satellite_100mhz"
     ];
-    build = { target, variant, json ? null }: let
+    board-package-set = { target, variant, json ? null }: let
       szl = zynqpkgs."${target}-szl";
       fsbl = zynqpkgs."${target}-fsbl";
       fwtype = if builtins.elem variant sat_variants then "satman" else "runtime";
@@ -122,18 +123,22 @@
         src = ./src;
         cargoLock = { 
           lockFile = src/Cargo.lock;
+          outputHashes = {
+              "tar-no-std-0.1.8" = "sha256-xm17108v4smXOqxdLvHl9CxTCJslmeogjm4Y87IXFuM=";
+              "fatfs-0.4.0" = "sha256-P7IgvhwTPXtNhcyv8cFqwO2UdaEcCGJY7UBG6+yBFSg=";
+            };
         };
 
         nativeBuildInputs = [
           pkgs.gnumake
           (pkgs.python3.withPackages(ps: [ ps.jsonschema artiqpkgs.migen migen-axi artiqpkgs.misoc artiqpkgs.artiq ]))
           zynqpkgs.cargo-xbuild
-          pkgs.llvmPackages_9.llvm
+          pkgs.llvmPackages_18.llvm
-          pkgs.llvmPackages_9.clang-unwrapped
+          pkgs.llvmPackages_18.clang-unwrapped
         ];
         buildPhase = ''
           export XARGO_RUST_SRC="${rust}/lib/rustlib/src/rust/library"
-          export CLANG_EXTRA_INCLUDE_DIR="${pkgs.llvmPackages_9.clang-unwrapped.lib}/lib/clang/9.0.1/include"
+          export CLANG_EXTRA_INCLUDE_DIR="${pkgs.llvmPackages_18.clang-unwrapped.lib}/lib/clang/18/include"
           export CARGO_HOME=$(mktemp -d cargo-home.XXX)
           export ZYNQ_RS=${zynq-rs}
           make TARGET=${target} GWARGS="${if json == null then "-V ${variant}" else json}" ${fwtype}
@@ -149,6 +154,7 @@
 
         doCheck = false;
         dontFixup = true;
+        auditable = false;
       };
       gateware = pkgs.runCommand "${target}-${variant}-gateware"
         {
@@ -268,7 +274,7 @@
     };
 
     # for hitl-tests
-    zc706-nist_qc2 = (build { target = "zc706"; variant = "nist_qc2"; });
+    zc706-nist_qc2 = (board-package-set { target = "zc706"; variant = "nist_qc2"; });
     zc706-hitl-tests = pkgs.stdenv.mkDerivation {
       name = "zc706-hitl-tests";
 
@@ -335,29 +341,30 @@
       {
         inherit fastnumbers artiq-netboot ramda migen-axi binutils-arm;
       } //
-      (build { target = "zc706"; variant = "nist_clock"; }) //
+      (board-package-set { target = "zc706"; variant = "nist_clock"; }) //
-      (build { target = "zc706"; variant = "nist_clock_master"; }) //
+      (board-package-set { target = "zc706"; variant = "nist_clock_master"; }) //
-      (build { target = "zc706"; variant = "nist_clock_master_100mhz"; }) //
+      (board-package-set { target = "zc706"; variant = "nist_clock_master_100mhz"; }) //
-      (build { target = "zc706"; variant = "nist_clock_satellite"; }) //
+      (board-package-set { target = "zc706"; variant = "nist_clock_satellite"; }) //
-      (build { target = "zc706"; variant = "nist_clock_satellite_100mhz"; }) //
+      (board-package-set { target = "zc706"; variant = "nist_clock_satellite_100mhz"; }) //
-      (build { target = "zc706"; variant = "nist_qc2"; }) //
+      (board-package-set { target = "zc706"; variant = "nist_qc2"; }) //
-      (build { target = "zc706"; variant = "nist_qc2_master"; }) //
+      (board-package-set { target = "zc706"; variant = "nist_qc2_master"; }) //
-      (build { target = "zc706"; variant = "nist_qc2_master_100mhz"; }) //
+      (board-package-set { target = "zc706"; variant = "nist_qc2_master_100mhz"; }) //
-      (build { target = "zc706"; variant = "nist_qc2_satellite"; }) //
+      (board-package-set { target = "zc706"; variant = "nist_qc2_satellite"; }) //
-      (build { target = "zc706"; variant = "nist_qc2_satellite_100mhz"; }) //
+      (board-package-set { target = "zc706"; variant = "nist_qc2_satellite_100mhz"; }) //
-      (build { target = "zc706"; variant = "acpki_nist_clock"; }) //
+      (board-package-set { target = "zc706"; variant = "acpki_nist_clock"; }) //
-      (build { target = "zc706"; variant = "acpki_nist_clock_master"; }) //
+      (board-package-set { target = "zc706"; variant = "acpki_nist_clock_master"; }) //
-      (build { target = "zc706"; variant = "acpki_nist_clock_master_100mhz"; }) //
+      (board-package-set { target = "zc706"; variant = "acpki_nist_clock_master_100mhz"; }) //
-      (build { target = "zc706"; variant = "acpki_nist_clock_satellite"; }) //
+      (board-package-set { target = "zc706"; variant = "acpki_nist_clock_satellite"; }) //
-      (build { target = "zc706"; variant = "acpki_nist_clock_satellite_100mhz"; }) //
+      (board-package-set { target = "zc706"; variant = "acpki_nist_clock_satellite_100mhz"; }) //
-      (build { target = "zc706"; variant = "acpki_nist_qc2"; }) //
+      (board-package-set { target = "zc706"; variant = "acpki_nist_qc2"; }) //
-      (build { target = "zc706"; variant = "acpki_nist_qc2_master"; }) //
+      (board-package-set { target = "zc706"; variant = "acpki_nist_qc2_master"; }) //
-      (build { target = "zc706"; variant = "acpki_nist_qc2_master_100mhz"; }) //
+      (board-package-set { target = "zc706"; variant = "acpki_nist_qc2_master_100mhz"; }) //
-      (build { target = "zc706"; variant = "acpki_nist_qc2_satellite"; }) //
+      (board-package-set { target = "zc706"; variant = "acpki_nist_qc2_satellite"; }) //
-      (build { target = "zc706"; variant = "acpki_nist_qc2_satellite_100mhz"; }) //
+      (board-package-set { target = "zc706"; variant = "acpki_nist_qc2_satellite_100mhz"; }) //
-      (build { target = "kasli_soc"; variant = "demo"; json = ./demo.json; }) //
+      (board-package-set { target = "kasli_soc"; variant = "demo"; json = ./demo.json; }) //
-      (build { target = "kasli_soc"; variant = "master"; json = ./kasli-soc-master.json; }) //
+      (board-package-set { target = "kasli_soc"; variant = "master"; json = ./kasli-soc-master.json; }) //
-      (build { target = "kasli_soc"; variant = "satellite"; json = ./kasli-soc-satellite.json; });
+      (board-package-set { target = "kasli_soc"; variant = "satellite"; json = ./kasli-soc-satellite.json; }) //
+      (board-package-set { target = "ebaz4205"; variant = "base"; });
 
     hydraJobs = packages.x86_64-linux // { inherit zc706-hitl-tests; inherit gateware-sim; inherit fmt-check; };
 
@@ -365,8 +372,8 @@
       name = "artiq-zynq-dev-shell";
       buildInputs = with pkgs; [
         rust
-        llvmPackages_9.llvm
+        llvmPackages_18.llvm
-        llvmPackages_9.clang-unwrapped
+        llvmPackages_18.clang-unwrapped
         gnumake
         cacert
         zynqpkgs.cargo-xbuild
@@ -377,15 +384,16 @@
         artiqpkgs.artiq
         artiqpkgs.vivado
         binutils-arm
+        pre-commit
       ];
       XARGO_RUST_SRC = "${rust}/lib/rustlib/src/rust/library";
-      CLANG_EXTRA_INCLUDE_DIR = "${pkgs.llvmPackages_9.clang-unwrapped.lib}/lib/clang/9.0.1/include";
+      CLANG_EXTRA_INCLUDE_DIR = "${pkgs.llvmPackages_18.clang-unwrapped.lib}/lib/clang/18/include";
       ZYNQ_RS = "${zynq-rs}";
       OPENOCD_ZYNQ = "${zynq-rs}/openocd";
       SZL = "${zynqpkgs.szl}";
     };
 
-    makeArtiqZynqPackage = build;
+    makeArtiqZynqPackage = board-package-set;
 
   };
 }

src/.clang-format

@@ -0,0 +1,32 @@
+BasedOnStyle: LLVM
+
+Language: Cpp
+Standard: Cpp11
+
+AccessModifierOffset: -1
+AlignEscapedNewlines: Left
+AlwaysBreakAfterReturnType: None
+AlwaysBreakTemplateDeclarations: Yes
+AllowAllParametersOfDeclarationOnNextLine: false
+AllowShortFunctionsOnASingleLine: Inline
+BinPackParameters: false
+BreakBeforeBinaryOperators: NonAssignment
+BreakBeforeTernaryOperators: true
+BreakConstructorInitializers: AfterColon
+BreakInheritanceList: AfterColon
+ColumnLimit: 120
+ConstructorInitializerAllOnOneLineOrOnePerLine: true
+ContinuationIndentWidth: 4
+DerivePointerAlignment: false
+IndentCaseLabels: true
+IndentPPDirectives: None
+IndentWidth: 4
+MaxEmptyLinesToKeep: 1
+PointerAlignment: Left
+ReflowComments: true
+SortIncludes: false
+SortUsingDeclarations: true
+SpaceAfterTemplateKeyword: false
+SpacesBeforeTrailingComments: 2
+TabWidth: 4
+UseTab: Never

src/.clippy.toml

@@ -0,0 +1 @@
+doc-valid-idents = ["CPython", "NumPy", ".."]

src/.pre-commit-config.yaml

@@ -0,0 +1,32 @@
+# See https://pre-commit.com for more information
+# See https://pre-commit.com/hooks.html for more hooks
+
+default_stages: [commit]
+
+repos:
+  - repo: local
+    hooks:
+      - id: cargo-fmt
+        name: artiq-zynq cargo format
+        entry: nix
+        language: system
+        types: [file, rust]
+        pass_filenames: false
+        description: Runs cargo fmt on the codebase.
+        args: [develop, -c, cargo, fmt, --manifest-path, src/Cargo.toml, --all]
+      - id: cargo-clippy
+        name: artiq-zynq cargo clippy
+        entry: nix
+        language: system
+        types: [file, rust]
+        pass_filenames: false
+        description: Runs cargo clippy on the codebase.
+        args: [develop, -c, cargo, clippy, --manifest-path, src/Cargo.toml, --tests]
+  - repo: https://github.com/pre-commit/mirrors-clang-format
+    rev: v19.1.0
+    hooks:
+      - id: clang-format
+        name: artiq-zynq clang-format
+        description: Runs clang-format on the codebase.
+        files: \.(cpp|h|hpp|c)$
+        args: [-style=file, -fallback-style=none, -assume-filename=src/.clang-format]

src/Cargo.lock

@@ -2,6 +2,21 @@
 # It is not intended for manual editing.
 version = 3
 
+[[package]]
+name = "approx"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cab112f0a86d568ea0e627cc1d6be74a1e9cd55214684db5561995f6dad897c6"
+dependencies = [
+ "num-traits",
+]
+
+[[package]]
+name = "arrayvec"
+version = "0.7.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711"
+
 [[package]]
 name = "async-recursion"
 version = "0.3.2"
@@ -31,6 +46,12 @@ version = "1.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
 
+[[package]]
+name = "bitflags"
+version = "2.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de"
+
 [[package]]
 name = "build_const"
 version = "0.2.2"
@@ -67,18 +88,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
 [[package]]
 name = "compiler_builtins"
-version = "0.1.39"
+version = "0.1.109"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3748f82c7d366a0b4950257d19db685d4958d2fa27c6d164a3f069fec42b748b"
+checksum = "f11973008a8cf741fe6d22f339eba21fd0ca81e2760a769ba8243ed6c21edd7e"
-
-[[package]]
-name = "core_io"
-version = "0.1.20210325"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "97f8932064288cc79feb4d343a399d353a6f6f001e586ece47fe518a9e8507df"
-dependencies = [
- "rustc_version",
-]
 
 [[package]]
 name = "crc"
@@ -126,13 +138,10 @@ dependencies = [
 
 [[package]]
 name = "fatfs"
-version = "0.3.5"
+version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
+source = "git+https://github.com/rafalh/rust-fatfs?rev=85f06e0#85f06e08edbd3368e1b0562f2fc1b6d178bf7b8a"
-checksum = "e18f80a87439240dac45d927fd8f8081b6f1e34c03e97271189fa8a8c2e96c8f"
 dependencies = [
- "bitflags",
+ "bitflags 2.6.0",
- "byteorder",
- "core_io",
  "log",
 ]
 
@@ -214,7 +223,6 @@ name = "io"
 version = "0.0.0"
 dependencies = [
  "byteorder",
- "core_io",
  "libsupport_zynq",
 ]
 
@@ -224,7 +232,6 @@ version = "0.1.0"
 dependencies = [
  "build_zynq",
  "byteorder",
- "core_io",
  "cslice",
  "dwarf",
  "dyld",
@@ -240,6 +247,7 @@ dependencies = [
  "libsupport_zynq",
  "log",
  "log_buffer",
+ "nalgebra",
  "nb 0.1.3",
  "unwind",
  "vcell",
@@ -262,7 +270,6 @@ name = "libboard_artiq"
 version = "0.0.0"
 dependencies = [
  "build_zynq",
- "core_io",
  "crc",
  "embedded-hal",
  "io",
@@ -306,7 +313,6 @@ dependencies = [
 name = "libconfig"
 version = "0.1.0"
 dependencies = [
- "core_io",
  "fatfs",
  "libboard_zynq",
  "log",
@@ -376,6 +382,20 @@ version = "0.7.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c75de51135344a4f8ed3cfe2720dc27736f7711989703a0b43aadf3753c55577"
 
+[[package]]
+name = "nalgebra"
+version = "0.33.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "26aecdf64b707efd1310e3544d709c5c0ac61c13756046aaaba41be5c4f66a3b"
+dependencies = [
+ "approx",
+ "num-complex",
+ "num-rational",
+ "num-traits",
+ "simba",
+ "typenum",
+]
+
 [[package]]
 name = "nb"
 version = "0.1.3"
@@ -391,6 +411,15 @@ version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "546c37ac5d9e56f55e73b677106873d9d9f5190605e41a856503623648488cae"
 
+[[package]]
+name = "num-complex"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "26873667bbbb7c5182d4a37c1add32cdf09f841af72da53318fdb81543c15085"
+dependencies = [
+ "num-traits",
+]
+
 [[package]]
 name = "num-derive"
 version = "0.3.3"
@@ -402,6 +431,26 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "num-integer"
+version = "0.1.46"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f"
+dependencies = [
+ "num-traits",
+]
+
+[[package]]
+name = "num-rational"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d41702bd167c2df5520b384281bc111a4b5efcf7fbc4c9c222c815b07e0a6a6a"
+dependencies = [
+ "autocfg",
+ "num-integer",
+ "num-traits",
+]
+
 [[package]]
 name = "num-traits"
 version = "0.2.15"
@@ -409,8 +458,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd"
 dependencies = [
  "autocfg",
+ "libm",
 ]
 
+[[package]]
+name = "paste"
+version = "1.0.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a"
+
 [[package]]
 name = "pin-project-lite"
 version = "0.2.9"
@@ -454,7 +510,6 @@ dependencies = [
  "async-recursion",
  "build_zynq",
  "byteorder",
- "core_io",
  "cslice",
  "dwarf",
  "dyld",
@@ -474,26 +529,17 @@ dependencies = [
  "log_buffer",
  "num-derive",
  "num-traits",
+ "tar-no-std",
  "unwind",
  "vcell",
  "void",
 ]
 
-[[package]]
-name = "rustc_version"
-version = "0.1.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c5f5376ea5e30ce23c03eb77cbe4962b988deead10910c372b226388b594c084"
-dependencies = [
- "semver",
-]
-
 [[package]]
 name = "satman"
 version = "0.0.0"
 dependencies = [
  "build_zynq",
- "core_io",
  "cslice",
  "embedded-hal",
  "io",
@@ -511,10 +557,16 @@ dependencies = [
 ]
 
 [[package]]
-name = "semver"
+name = "simba"
-version = "0.1.20"
+version = "0.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d4f410fedcf71af0345d7607d246e7ad15faaadd49d240ee3b24e5dc21a820ac"
+checksum = "b3a386a501cd104797982c15ae17aafe8b9261315b5d07e3ec803f2ea26be0fa"
+dependencies = [
+ "approx",
+ "num-complex",
+ "num-traits",
+ "paste",
+]
 
 [[package]]
 name = "smoltcp"
@@ -522,7 +574,7 @@ version = "0.7.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3e4a069bef843d170df47e7c0a8bf8d037f217d9f5b325865acc3e466ffe40d3"
 dependencies = [
- "bitflags",
+ "bitflags 1.3.2",
  "byteorder",
  "managed",
 ]
@@ -538,6 +590,22 @@ dependencies = [
  "unicode-ident",
 ]
 
+[[package]]
+name = "tar-no-std"
+version = "0.1.8"
+source = "git+https://git.m-labs.hk/M-Labs/tar-no-std?rev=2ab6dc5#2ab6dc58e5249c59c4eb03eaf3a119bcdd678d32"
+dependencies = [
+ "arrayvec",
+ "bitflags 1.3.2",
+ "log",
+]
+
+[[package]]
+name = "typenum"
+version = "1.17.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825"
+
 [[package]]
 name = "unicode-ident"
 version = "1.0.5"

src/Cargo.toml

@@ -14,5 +14,5 @@ members = [
 panic = "abort"
 debug = true
 codegen-units = 1
-opt-level = 2
+opt-level = 1
 lto = true

src/gateware/ddmtd.py

@@ -0,0 +1,119 @@
+from migen import *
+from migen.genlib.cdc import PulseSynchronizer, MultiReg
+from misoc.interconnect.csr import *
+
+
+class DDMTDSampler(Module):
+    def __init__(self, cd_ref, main_clk_se):
+        self.ref_beating = Signal()
+        self.main_beating = Signal()
+
+        # # #
+
+        ref_clk = Signal()
+        self.specials +=[
+            # ISERDESE2 can only be driven from fabric via IDELAYE2 (see UG471)
+            Instance("IDELAYE2",
+                    p_DELAY_SRC="DATAIN",
+                    p_HIGH_PERFORMANCE_MODE="TRUE",
+                    p_REFCLK_FREQUENCY=208.3,   # REFCLK frequency from IDELAYCTRL
+                    p_IDELAY_VALUE=0,
+
+                    i_DATAIN=cd_ref.clk,
+
+                    o_DATAOUT=ref_clk
+            ),
+            Instance("ISERDESE2",
+                    p_IOBDELAY="IFD",   # use DDLY as input
+                    p_DATA_RATE="SDR",
+                    p_DATA_WIDTH=2,     # min is 2
+                    p_NUM_CE=1,
+
+                    i_DDLY=ref_clk,
+                    i_CE1=1,
+                    i_CLK=ClockSignal("helper"),
+                    i_CLKDIV=ClockSignal("helper"),
+
+                    o_Q1=self.ref_beating
+            ),
+            Instance("ISERDESE2",
+                    p_DATA_RATE="SDR",
+                    p_DATA_WIDTH=2,     # min is 2
+                    p_NUM_CE=1,
+
+                    i_D=main_clk_se,
+                    i_CE1=1,
+                    i_CLK=ClockSignal("helper"),
+                    i_CLKDIV=ClockSignal("helper"),
+
+                    o_Q1=self.main_beating,
+            ),
+        ]
+
+
+class DDMTDDeglitcherMedianEdge(Module):
+    def __init__(self, counter, input_signal, stable_0_period=100, stable_1_period=100):
+        self.tag = Signal(len(counter))
+        self.detect = Signal()
+
+        stable_0_counter = Signal(reset=stable_0_period - 1, max=stable_0_period)
+        stable_1_counter = Signal(reset=stable_1_period - 1, max=stable_1_period)
+
+        # # #
+
+        # Based on CERN's median edge deglitcher FSM
+        # https://white-rabbit.web.cern.ch/documents/Precise_time_and_frequency_transfer_in_a_White_Rabbit_network.pdf (p.72)
+        fsm = ClockDomainsRenamer("helper")(FSM(reset_state="WAIT_STABLE_0"))
+        self.submodules += fsm
+
+        fsm.act("WAIT_STABLE_0",
+            If(stable_0_counter != 0,
+                NextValue(stable_0_counter, stable_0_counter - 1)
+            ).Else(
+                NextValue(stable_0_counter, stable_0_period - 1),
+                NextState("WAIT_EDGE")
+            ),
+            If(input_signal,
+                NextValue(stable_0_counter, stable_0_period - 1)
+            ),
+        )
+        fsm.act("WAIT_EDGE",
+            If(input_signal,
+                NextValue(self.tag, counter),
+                NextState("GOT_EDGE")
+            )
+        )
+        fsm.act("GOT_EDGE",
+            If(stable_1_counter != 0,
+                NextValue(stable_1_counter, stable_1_counter - 1)
+            ).Else(
+                NextValue(stable_1_counter, stable_1_period - 1),
+                self.detect.eq(1),
+                NextState("WAIT_STABLE_0")
+            ),
+            If(~input_signal,
+                NextValue(self.tag, self.tag + 1),
+                NextValue(stable_1_counter, stable_1_period - 1)
+            ),
+        )
+        
+
+class DDMTD(Module):
+    def __init__(self, counter, input_signal):
+
+        # in helper clock domain
+        self.h_tag = Signal(len(counter))
+        self.h_tag_update = Signal()
+
+        # # #
+
+        deglitcher = DDMTDDeglitcherMedianEdge(counter, input_signal)
+        self.submodules += deglitcher
+
+        self.sync.helper += [
+            self.h_tag_update.eq(0),
+            If(deglitcher.detect,
+                self.h_tag_update.eq(1),
+                self.h_tag.eq(deglitcher.tag)
+               )
+        ]

src/gateware/drtio_aux_controller.py

@@ -1,12 +1,12 @@
 """Auxiliary controller, common to satellite and master"""
 
-from artiq.gateware.drtio.aux_controller import Transmitter, Receiver
+from artiq.gateware.drtio.aux_controller import (max_packet, aux_buffer_count,
+    Transmitter, Receiver)
 from migen.fhdl.simplify import FullMemoryWE
 from misoc.interconnect.csr import *
 from migen_axi.interconnect.sram import SRAM
 from migen_axi.interconnect import axi
 
-max_packet = 1024
 
 class _DRTIOAuxControllerBase(Module):
     def __init__(self, link_layer):
@@ -27,12 +27,12 @@ class DRTIOAuxControllerAxi(_DRTIOAuxControllerBase):
         tx_sdram_if = SRAM(self.transmitter.mem, read_only=False)
         rx_sdram_if = SRAM(self.receiver.mem, read_only=True)
         aw_decoder = axi.AddressDecoder(self.bus.aw,
-            [(lambda a: a[log2_int(max_packet)] == 0, tx_sdram_if.bus.aw),
+            [(lambda a: a[log2_int(max_packet*aux_buffer_count)] == 0, tx_sdram_if.bus.aw),
-             (lambda a: a[log2_int(max_packet)] == 1, rx_sdram_if.bus.aw)],
+             (lambda a: a[log2_int(max_packet*aux_buffer_count)] == 1, rx_sdram_if.bus.aw)],
             register=True)
         ar_decoder = axi.AddressDecoder(self.bus.ar,
-            [(lambda a: a[log2_int(max_packet)] == 0, tx_sdram_if.bus.ar),
+            [(lambda a: a[log2_int(max_packet*aux_buffer_count)] == 0, tx_sdram_if.bus.ar),
-             (lambda a: a[log2_int(max_packet)] == 1, rx_sdram_if.bus.ar)],
+             (lambda a: a[log2_int(max_packet*aux_buffer_count)] == 1, rx_sdram_if.bus.ar)],
             register=True)
         # unlike wb, axi address decoder only connects ar/aw lanes,
         # the rest must also be connected!
@@ -82,4 +82,4 @@ class DRTIOAuxControllerBare(_DRTIOAuxControllerBase):
         return self.receiver.mem.get_port(write_capable=False)
 
     def get_mem_size(self):
-        return max_packet
+        return max_packet*aux_buffer_count

src/gateware/ebaz4205.py

@@ -0,0 +1,288 @@
+#!/usr/bin/env python
+
+import argparse
+
+import analyzer
+import dma
+from artiq.gateware import rtio
+from artiq.gateware.rtio.phy import dds, spi2, ttl_simple
+from artiq.gateware.rtio.xilinx_clocking import fix_serdes_timing_path
+from config import write_csr_file, write_mem_file, write_rustc_cfg_file
+from migen import *
+from migen.build.generic_platform import IOStandard, Misc, Pins, Subsignal
+from migen.build.platforms import ebaz4205
+from migen_axi.integration.soc_core import SoCCore
+from misoc.interconnect.csr import *
+
+_ps = [
+    (
+        "ps",
+        0,
+        Subsignal("clk", Pins("E7"), IOStandard("LVCMOS33"), Misc("SLEW=FAST")),
+        Subsignal("por_b", Pins("C7"), IOStandard("LVCMOS33"), Misc("SLEW=FAST")),
+        Subsignal("srst_b", Pins("B10"), IOStandard("LVCMOS18"), Misc("SLEW=FAST")),
+    )
+]
+
+_ddr = [
+    (
+        "ddr",
+        0,
+        Subsignal(
+            "a",
+            Pins("N2 K2 M3 K3 M4 L1 L4 K4 K1 J4 F5 G4 E4 D4 F4"),
+            IOStandard("SSTL15"),
+        ),
+        Subsignal("ba", Pins("L5 R4 J5"), IOStandard("SSTL15")),
+        Subsignal("cas_n", Pins("P5"), IOStandard("SSTL15")),
+        Subsignal("cke", Pins("N3"), IOStandard("SSTL15")),
+        Subsignal("cs_n", Pins("N1"), IOStandard("SSTL15")),
+        Subsignal("ck_n", Pins("M2"), IOStandard("DIFF_SSTL15"), Misc("SLEW=FAST")),
+        Subsignal("ck_p", Pins("L2"), IOStandard("DIFF_SSTL15"), Misc("SLEW=FAST")),
+        # Pins "T1 Y1" not connected
+        Subsignal("dm", Pins("A1 F1"), IOStandard("SSTL15_T_DCI"), Misc("SLEW=FAST")),
+        Subsignal(
+            "dq",
+            Pins("C3 B3 A2 A4 D3 D1 C1 E1 E2 E3 G3 H3 J3 H2 H1 J1"),
+            # Pins "P1 P3 R3 R1 T4 U4 U2 U3 V1 Y3 W1 Y4 Y2 W3 V2 V3" not connected
+            IOStandard("SSTL15_T_DCI"),
+            Misc("SLEW=FAST"),
+        ),
+        Subsignal(
+            "dqs_n",
+            Pins("B2 F2"),  # Pins "T2 W4" not connected
+            IOStandard("DIFF_SSTL15_T_DCI"),
+            Misc("SLEW=FAST"),
+        ),
+        Subsignal(
+            "dqs_p",
+            Pins("C2 G2"),  # Pins "R2 W5" not connected
+            IOStandard("DIFF_SSTL15_T_DCI"),
+            Misc("SLEW=FAST"),
+        ),
+        Subsignal("vrn", Pins("G5"), IOStandard("SSTL15_T_DCI"), Misc("SLEW=FAST")),
+        Subsignal("vrp", Pins("H5"), IOStandard("SSTL15_T_DCI"), Misc("SLEW=FAST")),
+        Subsignal("drst_n", Pins("B4"), IOStandard("SSTL15"), Misc("SLEW=FAST")),
+        Subsignal("odt", Pins("N5"), IOStandard("SSTL15")),
+        Subsignal("ras_n", Pins("P4"), IOStandard("SSTL15")),
+        Subsignal("we_n", Pins("M5"), IOStandard("SSTL15")),
+    )
+]
+
+# Connector J3
+_i2c = [
+    (
+        "i2c",
+        0,
+        Subsignal("scl", Pins("U12"), IOStandard("LVCMOS33")),
+        Subsignal("sda", Pins("V13"), IOStandard("LVCMOS33")),
+    )
+]
+
+_spi = [
+    (
+        "spi",
+        0,
+        Subsignal("clk", Pins("V20")),
+        Subsignal("mosi", Pins("U20")),
+        Subsignal("cs_n", Pins("P19")),
+        IOStandard("LVCMOS33"),
+    )
+]
+
+
+class EBAZ4205(SoCCore):
+    def __init__(self, rtio_clk=125e6, acpki=False):
+        self.acpki = acpki
+
+        platform = ebaz4205.Platform()
+        platform.toolchain.bitstream_commands.extend(
+            [
+                "set_property BITSTREAM.GENERAL.COMPRESS True [current_design]",
+            ]
+        )
+        platform.add_extension(_ps)
+        platform.add_extension(_ddr)
+        platform.add_extension(_i2c)
+        platform.add_extension(_spi)
+
+        gmii = platform.request("gmii")
+        platform.add_period_constraint(gmii.rx_clk, 10)
+        platform.add_period_constraint(gmii.tx_clk, 10)
+        platform.add_platform_command(
+            "set_property CLOCK_DEDICATED_ROUTE FALSE [get_nets gmii_tx_clk_IBUF]"
+        )
+
+        ident = self.__class__.__name__
+        if self.acpki:
+            ident = "acpki_" + ident
+        SoCCore.__init__(self, platform=platform, csr_data_width=32, ident=ident)
+        fix_serdes_timing_path(platform)
+        self.config["RTIO_FREQUENCY"] = str(rtio_clk / 1e6)
+        platform.add_period_constraint(self.ps7.cd_sys.clk, 10)
+
+        self.comb += [
+            self.ps7.enet0.enet.gmii.tx_clk.eq(gmii.tx_clk),
+            self.ps7.enet0.enet.gmii.rx_clk.eq(gmii.rx_clk),
+        ]
+        self.clock_domains.cd_eth_rx = ClockDomain(reset_less=False)
+        self.clock_domains.cd_eth_tx = ClockDomain(reset_less=False)
+        self.comb += [
+            ClockSignal("eth_rx").eq(gmii.rx_clk),
+            ClockSignal("eth_tx").eq(gmii.tx_clk),
+        ]
+        self.sync.eth_tx += [
+            gmii.txd.eq(self.ps7.enet0.enet.gmii.txd),
+            gmii.tx_en.eq(self.ps7.enet0.enet.gmii.tx_en),
+        ]
+        self.sync.eth_rx += [
+            self.ps7.enet0.enet.gmii.rxd.eq(gmii.rxd),
+            self.ps7.enet0.enet.gmii.rx_dv.eq(gmii.rx_dv),
+        ]
+
+        # MDIO
+        mdio = platform.request("mdio")
+        self.comb += mdio.mdc.eq(self.ps7.enet0.enet.mdio.mdc)
+        self.specials += Instance(
+            "IOBUF",
+            i_I=self.ps7.enet0.enet.mdio.o,
+            io_IO=mdio.mdio,
+            o_O=self.ps7.enet0.enet.mdio.i,
+            i_T=~self.ps7.enet0.enet.mdio.t_n,
+        )
+
+        # I2C
+        i2c = self.platform.request("i2c")
+        self.specials += [
+            # SCL
+            Instance(
+                "IOBUF",
+                i_I=self.ps7.i2c0.scl.o,
+                io_IO=i2c.scl,
+                o_O=self.ps7.i2c0.scl.i,
+                i_T=~self.ps7.i2c0.scl.t_n,
+            ),
+            # SDA
+            Instance(
+                "IOBUF",
+                i_I=self.ps7.i2c0.sda.o,
+                io_IO=i2c.sda,
+                o_O=self.ps7.i2c0.sda.i,
+                i_T=~self.ps7.i2c0.sda.t_n,
+            ),
+        ]
+
+        self.rtio_channels = []
+        for i in (0, 1):
+            print("USER LED at RTIO channel 0x{:06x}".format(len(self.rtio_channels)))
+            user_led = self.platform.request("user_led", i)
+            phy = ttl_simple.Output(user_led)
+            self.submodules += phy
+            self.rtio_channels.append(rtio.Channel.from_phy(phy))
+
+        print("SPI at RTIO channel 0x{:06x}".format(len(self.rtio_channels)))
+        spi_phy = spi2.SPIMaster(platform.request("spi"))
+        self.submodules += spi_phy
+        self.rtio_channels.append(rtio.Channel.from_phy(spi_phy, ififo_depth=4))
+        self.config["RTIO_LOG_CHANNEL"] = len(self.rtio_channels)
+        self.rtio_channels.append(rtio.LogChannel())
+
+        self.submodules.rtio_tsc = rtio.TSC(glbl_fine_ts_width=3)
+        self.submodules.rtio_core = rtio.Core(self.rtio_tsc, self.rtio_channels)
+        self.csr_devices.append("rtio_core")
+        if self.acpki:
+            import acpki
+
+            self.config["KI_IMPL"] = "acp"
+            self.submodules.rtio = acpki.KernelInitiator(
+                self.rtio_tsc,
+                bus=self.ps7.s_axi_acp,
+                user=self.ps7.s_axi_acp_user,
+                evento=self.ps7.event.o,
+            )
+            self.csr_devices.append("rtio")
+        else:
+            self.config["KI_IMPL"] = "csr"
+            self.submodules.rtio = rtio.KernelInitiator(self.rtio_tsc, now64=True)
+            self.csr_devices.append("rtio")
+
+        self.submodules.rtio_dma = dma.DMA(self.ps7.s_axi_hp0)
+        self.csr_devices.append("rtio_dma")
+
+        self.submodules.cri_con = rtio.CRIInterconnectShared(
+            [self.rtio.cri, self.rtio_dma.cri],
+            [self.rtio_core.cri],
+            enable_routing=True,
+        )
+        self.csr_devices.append("cri_con")
+
+        self.submodules.rtio_moninj = rtio.MonInj(self.rtio_channels)
+        self.csr_devices.append("rtio_moninj")
+
+        self.submodules.rtio_analyzer = analyzer.Analyzer(
+            self.rtio_tsc, self.rtio_core.cri, self.ps7.s_axi_hp1
+        )
+        self.csr_devices.append("rtio_analyzer")
+
+
+class BASE(EBAZ4205):
+    def __init__(self, rtio_clk, acpki):
+        EBAZ4205.__init__(self, rtio_clk, acpki)
+
+
+VARIANTS = {cls.__name__.lower(): cls for cls in [BASE]}
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="ARTIQ port to the EBAZ4205 control card of Ebit E9+ BTC miner"
+    )
+    parser.add_argument(
+        "-r", default=None, help="build Rust interface into the specified file"
+    )
+    parser.add_argument(
+        "-m", default=None, help="build Rust memory interface into the specified file"
+    )
+    parser.add_argument(
+        "-c",
+        default=None,
+        help="build Rust compiler configuration into the specified file",
+    )
+    parser.add_argument(
+        "-g", default=None, help="build gateware into the specified directory"
+    )
+    parser.add_argument("--rtio-clk", default=125e6, help="RTIO Clock Frequency (Hz)")
+    parser.add_argument(
+        "-V",
+        "--variant",
+        default="base",
+        help="variant: " "[acpki_]base" "(default: %(default)s)",
+    )
+    args = parser.parse_args()
+
+    rtio_clk = int(args.rtio_clk)
+    variant = args.variant.lower()
+    acpki = variant.startswith("acpki_")
+    if acpki:
+        variant = variant[6:]
+
+    try:
+        cls = VARIANTS[variant]
+    except KeyError:
+        raise SystemExit("Invalid variant (-V/--variant)")
+
+    soc = cls(rtio_clk=rtio_clk, acpki=acpki)
+    soc.finalize()
+
+    if args.r is not None:
+        write_csr_file(soc, args.r)
+    if args.m is not None:
+        write_mem_file(soc, args.m)
+    if args.c is not None:
+        write_rustc_cfg_file(soc, args.c)
+    if args.g is not None:
+        soc.build(build_dir=args.g)
+
+
+if __name__ == "__main__":
+    main()

src/gateware/kasli_soc.py

@@ -20,6 +20,7 @@ from artiq.gateware.drtio.transceiver import gtx_7series, eem_serdes
 from artiq.gateware.drtio.siphaser import SiPhaser7Series
 from artiq.gateware.drtio.rx_synchronizer import XilinxRXSynchronizer
 from artiq.gateware.drtio import *
+from artiq.gateware.wrpll import wrpll
 
 import dma
 import analyzer
@@ -108,6 +109,7 @@ class GenericStandalone(SoCCore):
     def __init__(self, description, acpki=False):
         self.acpki = acpki
         clk_freq = description["rtio_frequency"]
+        with_wrpll = description["enable_wrpll"]
 
         platform = kasli_soc.Platform()
         platform.toolchain.bitstream_commands.extend([
@@ -119,13 +121,6 @@ class GenericStandalone(SoCCore):
         SoCCore.__init__(self, platform=platform, csr_data_width=32, ident=ident, ps_cd_sys=False)
 
         self.config["HW_REV"] = description["hw_rev"]
-        
-
-        self.submodules += SMAClkinForward(self.platform)
-
-        self.config["HAS_SI5324"] = None
-        self.config["SI5324_SOFT_RESET"] = None
-
         clk_synth = platform.request("cdr_clk_clean_fabric")
         clk_synth_se = Signal()
         clk_synth_se_buf = Signal()
@@ -140,6 +135,7 @@ class GenericStandalone(SoCCore):
         ]
         fix_serdes_timing_path(platform)
         self.submodules.bootstrap = GTPBootstrapClock(self.platform, clk_freq)
+        self.config["RTIO_FREQUENCY"] = str(clk_freq/1e6)
         self.config["CLOCK_FREQUENCY"] = int(clk_freq)
 
         self.submodules.sys_crg = zynq_clocking.SYSCRG(self.platform, self.ps7, clk_synth_se_buf)
@@ -149,6 +145,23 @@ class GenericStandalone(SoCCore):
         self.crg = self.ps7 # HACK for eem_7series to find the clock
         self.crg.cd_sys = self.sys_crg.cd_sys
 
+        if with_wrpll:
+            self.submodules.wrpll_refclk = wrpll.FrequencyMultiplier(platform.request("sma_clkin"))
+            self.submodules.wrpll = wrpll.WRPLL(
+                platform=self.platform,
+                cd_ref=self.wrpll_refclk.cd_ref,
+                main_clk_se=clk_synth_se)
+            self.csr_devices.append("wrpll_refclk")
+            self.csr_devices.append("wrpll")
+            self.comb += self.ps7.core.core0.nfiq.eq(self.wrpll.ev.irq)
+            self.config["HAS_SI549"] = None
+            self.config["WRPLL_REF_CLK"] = "SMA_CLKIN"
+        else:
+            self.submodules += SMAClkinForward(self.platform)
+            self.config["HAS_SI5324"] = None
+            self.config["SI5324_SOFT_RESET"] = None
+
+
         self.rtio_channels = []
         has_grabber = any(peripheral["type"] == "grabber" for peripheral in description["peripherals"])
         if has_grabber:
@@ -207,6 +220,7 @@ class GenericStandalone(SoCCore):
 class GenericMaster(SoCCore):
     def __init__(self, description, acpki=False):
         clk_freq = description["rtio_frequency"]
+        with_wrpll = description["enable_wrpll"]
 
         has_drtio_over_eem = any(peripheral["type"] == "shuttler" for peripheral in description["peripherals"])
         self.acpki = acpki
@@ -222,8 +236,6 @@ class GenericMaster(SoCCore):
 
         self.config["HW_REV"] = description["hw_rev"]
 
-        self.submodules += SMAClkinForward(self.platform)
-
         data_pads = [platform.request("sfp", i) for i in range(4)]
 
         self.submodules.gt_drtio = gtx_7series.GTX(
@@ -231,18 +243,23 @@ class GenericMaster(SoCCore):
             pads=data_pads,
             clk_freq=clk_freq)
         self.csr_devices.append("gt_drtio")
+        self.config["RTIO_FREQUENCY"] = str(clk_freq/1e6)
         self.config["CLOCK_FREQUENCY"] = int(clk_freq)
 
         txout_buf = Signal()
         gtx0 = self.gt_drtio.gtxs[0]
         self.specials += Instance("BUFG", i_I=gtx0.txoutclk, o_O=txout_buf)
 
+        ext_async_rst = Signal()
+
         self.submodules.bootstrap = GTPBootstrapClock(self.platform, clk_freq)
         self.submodules.sys_crg = zynq_clocking.SYSCRG(
             self.platform,
             self.ps7,
             txout_buf,
-            clk_sw=gtx0.tx_init.done)
+            clk_sw=self.gt_drtio.stable_clkin.storage,
+            clk_sw_status=gtx0.tx_init.done,
+            ext_async_rst=ext_async_rst)
         self.csr_devices.append("sys_crg")
         self.crg = self.ps7 # HACK for eem_7series to find the clock
         self.crg.cd_sys = self.sys_crg.cd_sys
@@ -250,8 +267,28 @@ class GenericMaster(SoCCore):
             self.bootstrap.cd_bootstrap.clk, self.sys_crg.cd_sys.clk)
         fix_serdes_timing_path(platform)
 
-        self.config["HAS_SI5324"] = None
+        self.comb += ext_async_rst.eq(self.sys_crg.clk_sw_fsm.o_clk_sw & ~gtx0.tx_init.done)
-        self.config["SI5324_SOFT_RESET"] = None
+        self.specials += MultiReg(self.sys_crg.clk_sw_fsm.o_clk_sw & self.sys_crg.mmcm_locked, self.gt_drtio.clk_path_ready, odomain="bootstrap")
+
+        if with_wrpll:
+            clk_synth = platform.request("cdr_clk_clean_fabric")
+            clk_synth_se = Signal()
+            platform.add_period_constraint(clk_synth.p, 8.0)
+            self.specials += Instance("IBUFGDS", p_DIFF_TERM="TRUE", p_IBUF_LOW_PWR="FALSE", i_I=clk_synth.p, i_IB=clk_synth.n, o_O=clk_synth_se)
+            self.submodules.wrpll_refclk = wrpll.FrequencyMultiplier(platform.request("sma_clkin"))
+            self.submodules.wrpll = wrpll.WRPLL(
+                platform=self.platform,
+                cd_ref=self.wrpll_refclk.cd_ref,
+                main_clk_se=clk_synth_se)
+            self.csr_devices.append("wrpll_refclk")
+            self.csr_devices.append("wrpll")
+            self.comb += self.ps7.core.core0.nfiq.eq(self.wrpll.ev.irq)
+            self.config["HAS_SI549"] = None
+            self.config["WRPLL_REF_CLK"] = "SMA_CLKIN"
+        else:
+            self.submodules += SMAClkinForward(self.platform)
+            self.config["HAS_SI5324"] = None
+            self.config["SI5324_SOFT_RESET"] = None
 
         self.rtio_channels = []
         has_grabber = any(peripheral["type"] == "grabber" for peripheral in description["peripherals"])
@@ -393,6 +430,7 @@ class GenericMaster(SoCCore):
 class GenericSatellite(SoCCore):
     def __init__(self, description, acpki=False):
         clk_freq = description["rtio_frequency"]
+        with_wrpll = description["enable_wrpll"]
 
         self.acpki = acpki
 
@@ -419,12 +457,16 @@ class GenericSatellite(SoCCore):
         gtx0 = self.gt_drtio.gtxs[0]
         self.specials += Instance("BUFG", i_I=gtx0.txoutclk, o_O=txout_buf)
 
+        ext_async_rst = Signal()
+
         self.submodules.bootstrap = GTPBootstrapClock(self.platform, clk_freq)
         self.submodules.sys_crg = zynq_clocking.SYSCRG(
             self.platform, 
             self.ps7,
             txout_buf,
-            clk_sw=gtx0.tx_init.done)
+            clk_sw=self.gt_drtio.stable_clkin.storage,
+            clk_sw_status=gtx0.tx_init.done,
+            ext_async_rst=ext_async_rst)
         platform.add_false_path_constraints(
             self.bootstrap.cd_bootstrap.clk, self.sys_crg.cd_sys.clk)
         self.csr_devices.append("sys_crg")
@@ -433,6 +475,9 @@ class GenericSatellite(SoCCore):
 
         fix_serdes_timing_path(platform)
 
+        self.comb += ext_async_rst.eq(self.sys_crg.clk_sw_fsm.o_clk_sw & ~gtx0.tx_init.done)
+        self.specials += MultiReg(self.sys_crg.clk_sw_fsm.o_clk_sw & self.sys_crg.mmcm_locked, self.gt_drtio.clk_path_ready, odomain="bootstrap")
+
         self.rtio_channels = []
         has_grabber = any(peripheral["type"] == "grabber" for peripheral in description["peripherals"])
         if has_grabber:
@@ -515,7 +560,10 @@ class GenericSatellite(SoCCore):
         self.submodules.local_io = SyncRTIO(
             self.rtio_tsc, self.rtio_channels, lane_count=description["sed_lanes"]
         )
-        self.comb += self.drtiosat.async_errors.eq(self.local_io.async_errors)
+        self.comb += [
+            self.drtiosat.async_errors.eq(self.local_io.async_errors),
+            self.local_io.sed_spread_enable.eq(self.drtiosat.sed_spread_enable.storage)
+        ]
 
         self.submodules.cri_con = rtio.CRIInterconnectShared(
             [self.drtiosat.cri, self.rtio_dma.cri, self.rtio.cri],
@@ -537,14 +585,30 @@ class GenericSatellite(SoCCore):
         self.config["RTIO_FREQUENCY"] = str(clk_freq/1e6)
         self.config["CLOCK_FREQUENCY"] = int(clk_freq)
 
-        self.submodules.siphaser = SiPhaser7Series(
+        if with_wrpll:
-            si5324_clkin=platform.request("cdr_clk"),
+            clk_synth = platform.request("cdr_clk_clean_fabric")
-            rx_synchronizer=self.rx_synchronizer,
+            clk_synth_se = Signal()
-            ultrascale=False,
+            platform.add_period_constraint(clk_synth.p, 8.0)
-            rtio_clk_freq=self.gt_drtio.rtio_clk_freq)
+            self.specials += Instance("IBUFGDS", p_DIFF_TERM="TRUE", p_IBUF_LOW_PWR="FALSE", i_I=clk_synth.p, i_IB=clk_synth.n, o_O=clk_synth_se)
-        self.csr_devices.append("siphaser")
+            self.submodules.wrpll = wrpll.WRPLL(
-        self.config["HAS_SI5324"] = None
+                platform=self.platform,
-        self.config["SI5324_SOFT_RESET"] = None
+                cd_ref=self.gt_drtio.cd_rtio_rx0,
+                main_clk_se=clk_synth_se)
+            self.submodules.wrpll_skewtester = wrpll.SkewTester(self.rx_synchronizer)
+            self.csr_devices.append("wrpll_skewtester")
+            self.csr_devices.append("wrpll")
+            self.comb += self.ps7.core.core0.nfiq.eq(self.wrpll.ev.irq)
+            self.config["HAS_SI549"] = None
+            self.config["WRPLL_REF_CLK"] = "GT_CDR"
+        else:
+            self.submodules.siphaser = SiPhaser7Series(
+                si5324_clkin=platform.request("cdr_clk"),
+                rx_synchronizer=self.rx_synchronizer,
+                ultrascale=False,
+                rtio_clk_freq=self.gt_drtio.rtio_clk_freq)
+            self.csr_devices.append("siphaser")
+            self.config["HAS_SI5324"] = None
+            self.config["SI5324_SOFT_RESET"] = None
 
         gtx0 = self.gt_drtio.gtxs[0]
         platform.add_false_path_constraints(

src/gateware/zc706.py

@@ -226,6 +226,7 @@ class _MasterBase(SoCCore):
         self.csr_devices.append("gt_drtio")
 
         self.submodules.rtio_tsc = rtio.TSC(glbl_fine_ts_width=3)
+        ext_async_rst = Signal()
         txout_buf = Signal()
         gtx0 = self.gt_drtio.gtxs[0]
         self.specials += Instance("BUFG", i_I=gtx0.txoutclk, o_O=txout_buf)
@@ -234,12 +235,17 @@ class _MasterBase(SoCCore):
             self.platform, 
             self.ps7, 
             txout_buf,
-            clk_sw=gtx0.tx_init.done,
+            clk_sw=self.gt_drtio.stable_clkin.storage,
+            clk_sw_status=gtx0.tx_init.done,
+            ext_async_rst=ext_async_rst,
             freq=clk_freq)
         platform.add_false_path_constraints(
             self.bootstrap.cd_bootstrap.clk, self.sys_crg.cd_sys.clk)
         self.csr_devices.append("sys_crg")
 
+        self.comb += ext_async_rst.eq(self.sys_crg.clk_sw_fsm.o_clk_sw & ~gtx0.tx_init.done)
+        self.specials += MultiReg(self.sys_crg.clk_sw_fsm.o_clk_sw & self.sys_crg.mmcm_locked, self.gt_drtio.clk_path_ready, odomain="bootstrap")
+
         drtio_csr_group = []
         drtioaux_csr_group = []
         drtioaux_memory_group = []
@@ -361,6 +367,7 @@ class _SatelliteBase(SoCCore):
             clk_freq=clk_freq)
         self.csr_devices.append("gt_drtio")
 
+        ext_async_rst = Signal()
         txout_buf = Signal()
         txout_buf.attr.add("keep")
         gtx0 = self.gt_drtio.gtxs[0]
@@ -373,12 +380,17 @@ class _SatelliteBase(SoCCore):
             self.platform, 
             self.ps7, 
             txout_buf,
-            clk_sw=gtx0.tx_init.done,
+            clk_sw=self.gt_drtio.stable_clkin.storage,
+            clk_sw_status=gtx0.tx_init.done,
+            ext_async_rst=ext_async_rst,
             freq=clk_freq)
         platform.add_false_path_constraints(
             self.bootstrap.cd_bootstrap.clk, self.sys_crg.cd_sys.clk)
         self.csr_devices.append("sys_crg")
 
+        self.comb += ext_async_rst.eq(self.sys_crg.clk_sw_fsm.o_clk_sw & ~gtx0.tx_init.done)
+        self.specials += MultiReg(self.sys_crg.clk_sw_fsm.o_clk_sw & self.sys_crg.mmcm_locked, self.gt_drtio.clk_path_ready, odomain="bootstrap")
+
         drtioaux_csr_group = []
         drtioaux_memory_group = []
         drtiorep_csr_group = []
@@ -475,6 +487,10 @@ class _SatelliteBase(SoCCore):
         self.csr_devices.append("rtio_dma")
 
         self.submodules.local_io = SyncRTIO(self.rtio_tsc, rtio_channels)
+        self.comb += [
+            self.drtiosat.async_errors.eq(self.local_io.async_errors),
+            self.local_io.sed_spread_enable.eq(self.drtiosat.sed_spread_enable.storage)
+        ]
         self.submodules.cri_con = rtio.CRIInterconnectShared(
             [self.drtiosat.cri, self.rtio_dma.cri, self.rtio.cri],
             [self.local_io.cri] + self.drtio_cri,

src/gateware/zynq_clocking.py

@@ -65,7 +65,7 @@ class ClockSwitchFSM(Module):
 
 
 class SYSCRG(Module, AutoCSR):
-    def __init__(self, platform, ps7, main_clk, clk_sw=None, freq=125e6):
+    def __init__(self, platform, ps7, main_clk, clk_sw=None, clk_sw_status=None, freq=125e6, ext_async_rst=None, ):
         # assumes bootstrap clock is same freq as main and sys output
         self.clock_domains.cd_sys = ClockDomain()
         self.clock_domains.cd_sys4x = ClockDomain(reset_less=True)
@@ -88,7 +88,7 @@ class SYSCRG(Module, AutoCSR):
         else:
             self.comb += self.clk_sw_fsm.i_clk_sw.eq(clk_sw)
 
-        mmcm_locked = Signal()
+        self.mmcm_locked = Signal()
         mmcm_sys = Signal()
         mmcm_sys4x = Signal()
         mmcm_sys5x = Signal()
@@ -96,7 +96,7 @@ class SYSCRG(Module, AutoCSR):
         mmcm_fb_clk = Signal()
         self.specials += [
             Instance("MMCME2_ADV",
-                p_STARTUP_WAIT="FALSE", o_LOCKED=mmcm_locked,
+                p_STARTUP_WAIT="FALSE", o_LOCKED=self.mmcm_locked,
                 p_BANDWIDTH="HIGH",
                 p_REF_JITTER1=0.001,
                 p_CLKIN1_PERIOD=period, i_CLKIN1=main_clk,
@@ -125,10 +125,19 @@ class SYSCRG(Module, AutoCSR):
             Instance("BUFG", i_I=mmcm_sys, o_O=self.cd_sys.clk),
             Instance("BUFG", i_I=mmcm_sys4x, o_O=self.cd_sys4x.clk),
             Instance("BUFG", i_I=mmcm_clk208, o_O=self.cd_clk200.clk),
-            AsyncResetSynchronizer(self.cd_sys, ~mmcm_locked),
-            AsyncResetSynchronizer(self.cd_clk200, ~mmcm_locked),
         ]
 
+        if ext_async_rst is not None:
+            self.specials += [
+                AsyncResetSynchronizer(self.cd_sys, ~self.mmcm_locked | ext_async_rst),
+                AsyncResetSynchronizer(self.cd_clk200, ~self.mmcm_locked | ext_async_rst),
+            ]
+        else:
+            self.specials += [
+                AsyncResetSynchronizer(self.cd_sys, ~self.mmcm_locked),
+                AsyncResetSynchronizer(self.cd_clk200, ~self.mmcm_locked),
+            ]
+
         reset_counter = Signal(4, reset=15)
         ic_reset = Signal(reset=1)
         self.sync.clk200 += \
@@ -139,4 +148,7 @@ class SYSCRG(Module, AutoCSR):
             )
         self.specials += Instance("IDELAYCTRL", i_REFCLK=ClockSignal("clk200"), i_RST=ic_reset)
 
-        self.comb += self.current_clock.status.eq(self.clk_sw_fsm.o_clk_sw)
+        if clk_sw_status is None:
+            self.comb += self.current_clock.status.eq(self.clk_sw_fsm.o_clk_sw)
+        else:
+            self.comb += self.current_clock.status.eq(clk_sw_status)

src/libboard_artiq/Cargo.toml.tpl

@@ -10,6 +10,8 @@ name = "libboard_artiq"
 [features]
 target_zc706 = ["libboard_zynq/target_zc706", "libconfig/target_zc706"]
 target_kasli_soc = ["libboard_zynq/target_kasli_soc", "libconfig/target_kasli_soc"]
+target_ebaz4205 = ["libboard_zynq/target_ebaz4205", "libconfig/target_ebaz4205"]
+calibrate_wrpll_skew = []
 
 [build-dependencies]
 build_zynq = { path = "../libbuild_zynq" }
@@ -18,7 +20,6 @@ build_zynq = { path = "../libbuild_zynq" }
 log = "0.4"
 log_buffer = { version = "1.2" }
 crc = { version = "1.7", default-features = false }
-core_io = { version = "0.1", features = ["collections"] }
 embedded-hal = "0.2"
 nb = "1.0"
 void = { version = "1", default-features = false }

src/libboard_artiq/src/drtioaux.rs

@@ -1,15 +1,15 @@
-use core_io::{Error as IoError, ErrorKind as IoErrorKind};
+use core::slice;
+
 use crc;
 use io::{proto::{ProtoRead, ProtoWrite},
-         Cursor};
+         Cursor, Error as IoError};
 use libboard_zynq::{time::Milliseconds, timer::GlobalTimer};
-use libcortex_a9::asm::dmb;
 
 pub use crate::drtioaux_proto::Packet;
 use crate::{drtioaux_proto::Error as ProtocolError, mem::mem::DRTIOAUX_MEM, pl::csr::DRTIOAUX};
 
 #[derive(Debug)]
-pub enum Error {
+pub enum Error<T> {
     GatewareError,
     CorruptedPacket,
 
@@ -19,17 +19,17 @@ pub enum Error {
 
     RoutingError,
 
-    Protocol(ProtocolError),
+    Protocol(ProtocolError<T>),
 }
 
-impl From<ProtocolError> for Error {
+impl<T> From<ProtocolError<T>> for Error<T> {
-    fn from(value: ProtocolError) -> Error {
+    fn from(value: ProtocolError<T>) -> Error<T> {
         Error::Protocol(value)
     }
 }
 
-impl From<IoError> for Error {
+impl<T> From<IoError<T>> for Error<T> {
-    fn from(value: IoError) -> Error {
+    fn from(value: IoError<T>) -> Error<T> {
         Error::Protocol(ProtocolError::Io(value))
     }
 }
@@ -56,30 +56,14 @@ pub fn has_rx_error(linkno: u8) -> bool {
     }
 }
 
-pub fn copy_work_buffer(src: *mut u32, dst: *mut u32, len: isize) {
+fn receive<F, T>(linkno: u8, f: F) -> Result<Option<T>, Error<!>>
-    // AXI writes must be 4-byte aligned (drtio proto doesn't care for that),
+where F: FnOnce(&[u8]) -> Result<T, Error<!>> {
-    // and AXI burst reads/writes are not implemented yet in gateware
-    // thus the need for a work buffer for transmitting and copying it over
-    unsafe {
-        for i in 0..(len / 4) {
-            *dst.offset(i) = *src.offset(i);
-            //data memory barrier to prevent bursts
-            dmb();
-        }
-    }
-}
-
-fn receive<F, T>(linkno: u8, f: F) -> Result<Option<T>, Error>
-where F: FnOnce(&[u8]) -> Result<T, Error> {
     let linkidx = linkno as usize;
     unsafe {
         if (DRTIOAUX[linkidx].aux_rx_present_read)() == 1 {
-            let ptr = (DRTIOAUX_MEM[linkidx].base + DRTIOAUX_MEM[linkidx].size / 2) as *mut u32;
+            let read_ptr = (DRTIOAUX[linkidx].aux_read_pointer_read)() as usize;
-            let len = (DRTIOAUX[linkidx].aux_rx_length_read)() as usize;
+            let ptr = (DRTIOAUX_MEM[linkidx].base + DRTIOAUX_MEM[linkidx].size / 2 + read_ptr * 0x400) as *mut u32;
-            // work buffer to accomodate axi burst reads
+            let result = f(slice::from_raw_parts(ptr as *mut u8, 0x400 as usize));
-            let mut buf: [u8; 1024] = [0; 1024];
-            copy_work_buffer(ptr, buf.as_mut_ptr() as *mut u32, len as isize);
-            let result = f(&buf[0..len]);
             (DRTIOAUX[linkidx].aux_rx_present_write)(1);
             Ok(Some(result?))
         } else {
@@ -88,31 +72,31 @@ where F: FnOnce(&[u8]) -> Result<T, Error> {
     }
 }
 
-pub fn recv(linkno: u8) -> Result<Option<Packet>, Error> {
+pub fn recv(linkno: u8) -> Result<Option<Packet>, Error<!>> {
     if has_rx_error(linkno) {
         return Err(Error::GatewareError);
     }
 
     receive(linkno, |buffer| {
         if buffer.len() < 8 {
-            return Err(IoError::new(IoErrorKind::UnexpectedEof, "Unexpected end").into());
+            return Err(IoError::UnexpectedEnd.into());
         }
 
         let mut reader = Cursor::new(buffer);
 
-        let checksum_at = buffer.len() - 4;
+        let packet = Packet::read_from(&mut reader)?;
+        let padding = (12 - (reader.position() % 8)) % 8;
+        let checksum_at = reader.position() + padding;
         let checksum = crc::crc32::checksum_ieee(&reader.get_ref()[0..checksum_at]);
         reader.set_position(checksum_at);
         if reader.read_u32()? != checksum {
             return Err(Error::CorruptedPacket);
         }
-        reader.set_position(0);
+        Ok(packet)
-
-        Ok(Packet::read_from(&mut reader)?)
     })
 }
 
-pub fn recv_timeout(linkno: u8, timeout_ms: Option<u64>, timer: GlobalTimer) -> Result<Packet, Error> {
+pub fn recv_timeout(linkno: u8, timeout_ms: Option<u64>, timer: GlobalTimer) -> Result<Packet, Error<!>> {
     let timeout_ms = Milliseconds(timeout_ms.unwrap_or(10));
     let limit = timer.get_time() + timeout_ms;
     while timer.get_time() < limit {
@@ -124,24 +108,20 @@ pub fn recv_timeout(linkno: u8, timeout_ms: Option<u64>, timer: GlobalTimer) ->
     Err(Error::TimedOut)
 }
 
-fn transmit<F>(linkno: u8, f: F) -> Result<(), Error>
+fn transmit<F>(linkno: u8, f: F) -> Result<(), Error<!>>
-where F: FnOnce(&mut [u8]) -> Result<usize, Error> {
+where F: FnOnce(&mut [u8]) -> Result<usize, Error<!>> {
     let linkno = linkno as usize;
     unsafe {
         while (DRTIOAUX[linkno].aux_tx_read)() != 0 {}
         let ptr = DRTIOAUX_MEM[linkno].base as *mut u32;
-        let len = DRTIOAUX_MEM[linkno].size / 2;
+        let len = f(slice::from_raw_parts_mut(ptr as *mut u8, 0x400 as usize))?;
-        // work buffer, works with unaligned mem access
-        let mut buf: [u8; 1024] = [0; 1024];
-        let len = f(&mut buf[0..len])?;
-        copy_work_buffer(buf.as_mut_ptr() as *mut u32, ptr, len as isize);
         (DRTIOAUX[linkno].aux_tx_length_write)(len as u16);
         (DRTIOAUX[linkno].aux_tx_write)(1);
         Ok(())
     }
 }
 
-pub fn send(linkno: u8, packet: &Packet) -> Result<(), Error> {
+pub fn send(linkno: u8, packet: &Packet) -> Result<(), Error<!>> {
     transmit(linkno, |buffer| {
         let mut writer = Cursor::new(buffer);
 

src/libboard_artiq/src/drtioaux_async.rs

@@ -1,14 +1,15 @@
-use core_io::{Error as IoError, ErrorKind as IoErrorKind};
+use core::slice;
+
 use crc;
 use io::{proto::{ProtoRead, ProtoWrite},
-         Cursor};
+         Cursor, Error as IoError};
 use libasync::{block_async, task};
 use libboard_zynq::{time::Milliseconds, timer::GlobalTimer};
 use nb;
 use void::Void;
 
 pub use crate::drtioaux_proto::Packet;
-use crate::{drtioaux::{copy_work_buffer, has_rx_error, Error},
+use crate::{drtioaux::{has_rx_error, Error},
             mem::mem::DRTIOAUX_MEM,
             pl::csr::DRTIOAUX};
 
@@ -33,17 +34,14 @@ fn tx_ready(linkno: usize) -> nb::Result<(), Void> {
     }
 }
 
-async fn receive<F, T>(linkno: u8, f: F) -> Result<Option<T>, Error>
+async fn receive<F, T>(linkno: u8, f: F) -> Result<Option<T>, Error<!>>
-where F: FnOnce(&[u8]) -> Result<T, Error> {
+where F: FnOnce(&[u8]) -> Result<T, Error<!>> {
     let linkidx = linkno as usize;
     unsafe {
         if (DRTIOAUX[linkidx].aux_rx_present_read)() == 1 {
-            let ptr = (DRTIOAUX_MEM[linkidx].base + DRTIOAUX_MEM[linkidx].size / 2) as *mut u32;
+            let read_ptr = (DRTIOAUX[linkidx].aux_read_pointer_read)() as usize;
-            let len = (DRTIOAUX[linkidx].aux_rx_length_read)() as usize;
+            let ptr = (DRTIOAUX_MEM[linkidx].base + DRTIOAUX_MEM[linkidx].size / 2 + read_ptr * 0x400) as *mut u32;
-            // work buffer to accomodate axi burst reads
+            let result = f(slice::from_raw_parts(ptr as *mut u8, 0x400 as usize));
-            let mut buf: [u8; 1024] = [0; 1024];
-            copy_work_buffer(ptr, buf.as_mut_ptr() as *mut u32, len as isize);
-            let result = f(&buf[0..len]);
             (DRTIOAUX[linkidx].aux_rx_present_write)(1);
             Ok(Some(result?))
         } else {
@@ -52,32 +50,32 @@ where F: FnOnce(&[u8]) -> Result<T, Error> {
     }
 }
 
-pub async fn recv(linkno: u8) -> Result<Option<Packet>, Error> {
+pub async fn recv(linkno: u8) -> Result<Option<Packet>, Error<!>> {
     if has_rx_error(linkno) {
         return Err(Error::GatewareError);
     }
 
     receive(linkno, |buffer| {
         if buffer.len() < 8 {
-            return Err(IoError::new(IoErrorKind::UnexpectedEof, "Unexpected end").into());
+            return Err(IoError::UnexpectedEnd.into());
         }
 
         let mut reader = Cursor::new(buffer);
 
-        let checksum_at = buffer.len() - 4;
+        let packet = Packet::read_from(&mut reader)?;
+        let padding = (12 - (reader.position() % 8)) % 8;
+        let checksum_at = reader.position() + padding;
         let checksum = crc::crc32::checksum_ieee(&reader.get_ref()[0..checksum_at]);
         reader.set_position(checksum_at);
         if reader.read_u32()? != checksum {
             return Err(Error::CorruptedPacket);
         }
-        reader.set_position(0);
+        Ok(packet)
-
-        Ok(Packet::read_from(&mut reader)?)
     })
     .await
 }
 
-pub async fn recv_timeout(linkno: u8, timeout_ms: Option<u64>, timer: GlobalTimer) -> Result<Packet, Error> {
+pub async fn recv_timeout(linkno: u8, timeout_ms: Option<u64>, timer: GlobalTimer) -> Result<Packet, Error<!>> {
     let timeout_ms = Milliseconds(timeout_ms.unwrap_or(10));
     let limit = timer.get_time() + timeout_ms;
     let mut would_block = false;
@@ -97,24 +95,20 @@ pub async fn recv_timeout(linkno: u8, timeout_ms: Option<u64>, timer: GlobalTime
     Err(Error::TimedOut)
 }
 
-async fn transmit<F>(linkno: u8, f: F) -> Result<(), Error>
+async fn transmit<F>(linkno: u8, f: F) -> Result<(), Error<!>>
-where F: FnOnce(&mut [u8]) -> Result<usize, Error> {
+where F: FnOnce(&mut [u8]) -> Result<usize, Error<!>> {
     let linkno = linkno as usize;
     unsafe {
         let _ = block_async!(tx_ready(linkno)).await;
         let ptr = DRTIOAUX_MEM[linkno].base as *mut u32;
-        let len = DRTIOAUX_MEM[linkno].size / 2;
+        let len = f(slice::from_raw_parts_mut(ptr as *mut u8, 0x400 as usize))?;
-        // work buffer, works with unaligned mem access
-        let mut buf: [u8; 1024] = [0; 1024];
-        let len = f(&mut buf[0..len])?;
-        copy_work_buffer(buf.as_mut_ptr() as *mut u32, ptr, len as isize);
         (DRTIOAUX[linkno].aux_tx_length_write)(len as u16);
         (DRTIOAUX[linkno].aux_tx_write)(1);
         Ok(())
     }
 }
 
-pub async fn send(linkno: u8, packet: &Packet) -> Result<(), Error> {
+pub async fn send(linkno: u8, packet: &Packet) -> Result<(), Error<!>> {
     transmit(linkno, |buffer| {
         let mut writer = Cursor::new(buffer);
 

src/libboard_artiq/src/drtioaux_proto.rs

@@ -1,24 +1,66 @@
-use core_io::{Error as IoError, Read, Write};
+use io::{Error as IoError, Read, Write};
 use io::proto::{ProtoRead, ProtoWrite};
 
+const MAX_PACKET: usize = 1024;
+
 // maximum size of arbitrary payloads
 // used by satellite -> master analyzer, subkernel exceptions
-pub const SAT_PAYLOAD_MAX_SIZE: usize  = /*max size*/512 - /*CRC*/4 - /*packet ID*/1 - /*last*/1 - /*length*/2;
+pub const SAT_PAYLOAD_MAX_SIZE: usize  = /*max size*/MAX_PACKET - /*CRC*/4 - /*packet ID*/1 - /*last*/1 - /*length*/2;
 // used by DDMA, subkernel program data (need to provide extra ID and destination)
-pub const MASTER_PAYLOAD_MAX_SIZE: usize = SAT_PAYLOAD_MAX_SIZE - /*destination*/1 - /*ID*/4;
+pub const MASTER_PAYLOAD_MAX_SIZE: usize = SAT_PAYLOAD_MAX_SIZE - /*source*/1 - /*destination*/1 - /*ID*/4;
 
 #[derive(Debug)]
-pub enum Error {
+pub enum Error<T> {
     UnknownPacket(u8),
-    Io(IoError),
+    Io(IoError<T>),
 }
 
-impl From<IoError> for Error {
+impl<T> From<IoError<T>> for Error<T> {
-    fn from(value: IoError) -> Error {
+    fn from(value: IoError<T>) -> Error<T> {
         Error::Io(value)
     }
 }
 
+#[derive(PartialEq, Clone, Copy, Debug)]
+#[repr(u8)]
+pub enum PayloadStatus {
+    Middle = 0,
+    First = 1,
+    Last = 2,
+    FirstAndLast = 3,
+}
+
+impl From<u8> for PayloadStatus {
+    fn from(value: u8) -> PayloadStatus {
+        match value {
+            0 => PayloadStatus::Middle,
+            1 => PayloadStatus::First,
+            2 => PayloadStatus::Last,
+            3 => PayloadStatus::FirstAndLast,
+            _ => unreachable!(),
+        }
+    }
+}
+
+impl PayloadStatus {
+    pub fn is_first(self) -> bool {
+        self == PayloadStatus::First || self == PayloadStatus::FirstAndLast
+    }
+
+    pub fn is_last(self) -> bool {
+        self == PayloadStatus::Last || self == PayloadStatus::FirstAndLast
+    }
+
+    pub fn from_status(first: bool, last: bool) -> PayloadStatus {
+        match (first, last) {
+            (true, true) => PayloadStatus::FirstAndLast,
+            (true, false) => PayloadStatus::First,
+            (false, true) => PayloadStatus::Last,
+            (false, false) => PayloadStatus::Middle,
+        }
+    }
+}
+
 #[derive(PartialEq, Debug)]
 pub enum Packet {
     EchoRequest,
@@ -157,31 +199,40 @@ pub enum Packet {
     },
 
     DmaAddTraceRequest {
+        source: u8,
         destination: u8,
         id: u32,
-        last: bool,
+        status: PayloadStatus,
         length: u16,
         trace: [u8; MASTER_PAYLOAD_MAX_SIZE],
     },
     DmaAddTraceReply {
+        source: u8,
+        destination: u8,
+        id: u32,
         succeeded: bool,
     },
     DmaRemoveTraceRequest {
+        source: u8,
         destination: u8,
         id: u32,
     },
     DmaRemoveTraceReply {
+        destination: u8,
         succeeded: bool,
     },
     DmaPlaybackRequest {
+        source: u8,
         destination: u8,
         id: u32,
         timestamp: u64,
     },
     DmaPlaybackReply {
+        destination: u8,
         succeeded: bool,
     },
     DmaPlaybackStatus {
+        source: u8,
         destination: u8,
         id: u32,
         error: u8,
@@ -192,7 +243,7 @@ pub enum Packet {
     SubkernelAddDataRequest {
         destination: u8,
         id: u32,
-        last: bool,
+        status: PayloadStatus,
         length: u16,
         data: [u8; MASTER_PAYLOAD_MAX_SIZE],
     },
@@ -200,35 +251,37 @@ pub enum Packet {
         succeeded: bool,
     },
     SubkernelLoadRunRequest {
+        source: u8,
         destination: u8,
         id: u32,
         run: bool,
+        timestamp: u64,
     },
     SubkernelLoadRunReply {
-        succeeded: bool,
-    },
-    SubkernelStopRequest {
         destination: u8,
-    },
-    SubkernelStopReply {
         succeeded: bool,
     },
     SubkernelFinished {
+        destination: u8,
         id: u32,
         with_exception: bool,
+        exception_src: u8,
     },
     SubkernelExceptionRequest {
+        source: u8,
         destination: u8,
     },
     SubkernelException {
+        destination: u8,
         last: bool,
         length: u16,
-        data: [u8; SAT_PAYLOAD_MAX_SIZE],
+        data: [u8; MASTER_PAYLOAD_MAX_SIZE],
     },
     SubkernelMessage {
+        source: u8,
         destination: u8,
         id: u32,
-        last: bool,
+        status: PayloadStatus,
         length: u16,
         data: [u8; MASTER_PAYLOAD_MAX_SIZE],
     },
@@ -238,7 +291,7 @@ pub enum Packet {
 }
 
 impl Packet {
-    pub fn read_from<R>(reader: &mut R) -> Result<Self, Error>
+    pub fn read_from<R>(reader: &mut R) -> Result<Self, Error<R::ReadError>>
     where R: Read + ?Sized {
         Ok(match reader.read_u8()? {
             0x00 => Packet::EchoRequest,
@@ -389,39 +442,49 @@ impl Packet {
             }
 
             0xb0 => {
+                let source = reader.read_u8()?;
                 let destination = reader.read_u8()?;
                 let id = reader.read_u32()?;
-                let last = reader.read_bool()?;
+                let status = reader.read_u8()?;
                 let length = reader.read_u16()?;
                 let mut trace: [u8; MASTER_PAYLOAD_MAX_SIZE] = [0; MASTER_PAYLOAD_MAX_SIZE];
                 reader.read_exact(&mut trace[0..length as usize])?;
                 Packet::DmaAddTraceRequest {
+                    source: source,
                     destination: destination,
                     id: id,
-                    last: last,
+                    status: PayloadStatus::from(status),
                     length: length as u16,
                     trace: trace,
                 }
             }
             0xb1 => Packet::DmaAddTraceReply {
+                source: reader.read_u8()?,
+                destination: reader.read_u8()?,
+                id: reader.read_u32()?,
                 succeeded: reader.read_bool()?,
             },
             0xb2 => Packet::DmaRemoveTraceRequest {
+                source: reader.read_u8()?,
                 destination: reader.read_u8()?,
                 id: reader.read_u32()?,
             },
             0xb3 => Packet::DmaRemoveTraceReply {
+                destination: reader.read_u8()?,
                 succeeded: reader.read_bool()?,
             },
             0xb4 => Packet::DmaPlaybackRequest {
+                source: reader.read_u8()?,
                 destination: reader.read_u8()?,
                 id: reader.read_u32()?,
                 timestamp: reader.read_u64()?,
             },
             0xb5 => Packet::DmaPlaybackReply {
+                destination: reader.read_u8()?,
                 succeeded: reader.read_bool()?,
             },
             0xb6 => Packet::DmaPlaybackStatus {
+                source: reader.read_u8()?,
                 destination: reader.read_u8()?,
                 id: reader.read_u32()?,
                 error: reader.read_u8()?,
@@ -432,14 +495,14 @@ impl Packet {
             0xc0 => {
                 let destination = reader.read_u8()?;
                 let id = reader.read_u32()?;
-                let last = reader.read_bool()?;
+                let status = PayloadStatus::from(reader.read_u8()?);
                 let length = reader.read_u16()?;
                 let mut data: [u8; MASTER_PAYLOAD_MAX_SIZE] = [0; MASTER_PAYLOAD_MAX_SIZE];
                 reader.read_exact(&mut data[0..length as usize])?;
                 Packet::SubkernelAddDataRequest {
                     destination: destination,
                     id: id,
-                    last: last,
+                    status: status,
                     length: length as u16,
                     data: data,
                 }
@@ -448,48 +511,52 @@ impl Packet {
                 succeeded: reader.read_bool()?,
             },
             0xc4 => Packet::SubkernelLoadRunRequest {
+                source: reader.read_u8()?,
                 destination: reader.read_u8()?,
                 id: reader.read_u32()?,
                 run: reader.read_bool()?,
+                timestamp: reader.read_u64()?,
             },
             0xc5 => Packet::SubkernelLoadRunReply {
-                succeeded: reader.read_bool()?,
-            },
-            0xc6 => Packet::SubkernelStopRequest {
                 destination: reader.read_u8()?,
-            },
-            0xc7 => Packet::SubkernelStopReply {
                 succeeded: reader.read_bool()?,
             },
             0xc8 => Packet::SubkernelFinished {
+                destination: reader.read_u8()?,
                 id: reader.read_u32()?,
                 with_exception: reader.read_bool()?,
+                exception_src: reader.read_u8()?,
             },
             0xc9 => Packet::SubkernelExceptionRequest {
+                source: reader.read_u8()?,
                 destination: reader.read_u8()?,
             },
             0xca => {
+                let destination = reader.read_u8()?;
                 let last = reader.read_bool()?;
                 let length = reader.read_u16()?;
-                let mut data: [u8; SAT_PAYLOAD_MAX_SIZE] = [0; SAT_PAYLOAD_MAX_SIZE];
+                let mut data: [u8; MASTER_PAYLOAD_MAX_SIZE] = [0; MASTER_PAYLOAD_MAX_SIZE];
                 reader.read_exact(&mut data[0..length as usize])?;
                 Packet::SubkernelException {
+                    destination: destination,
                     last: last,
                     length: length,
                     data: data,
                 }
             }
             0xcb => {
+                let source = reader.read_u8()?;
                 let destination = reader.read_u8()?;
                 let id = reader.read_u32()?;
-                let last = reader.read_bool()?;
+                let status = reader.read_u8()?;
                 let length = reader.read_u16()?;
                 let mut data: [u8; MASTER_PAYLOAD_MAX_SIZE] = [0; MASTER_PAYLOAD_MAX_SIZE];
                 reader.read_exact(&mut data[0..length as usize])?;
                 Packet::SubkernelMessage {
+                    source: source,
                     destination: destination,
                     id: id,
-                    last: last,
+                    status: PayloadStatus::from(status),
                     length: length as u16,
                     data: data,
                 }
@@ -502,7 +569,7 @@ impl Packet {
         })
     }
 
-    pub fn write_to<W>(&self, writer: &mut W) -> Result<(), IoError>
+    pub fn write_to<W>(&self, writer: &mut W) -> Result<(), IoError<W::WriteError>>
     where W: Write + ?Sized {
         match *self {
             Packet::EchoRequest => writer.write_u8(0x00)?,
@@ -711,49 +778,69 @@ impl Packet {
             }
 
             Packet::DmaAddTraceRequest {
+                source,
                 destination,
                 id,
-                last,
+                status,
                 trace,
                 length,
             } => {
                 writer.write_u8(0xb0)?;
+                writer.write_u8(source)?;
                 writer.write_u8(destination)?;
                 writer.write_u32(id)?;
-                writer.write_bool(last)?;
+                writer.write_u8(status as u8)?;
                 // trace may be broken down to fit within drtio aux memory limit
                 // will be reconstructed by satellite
                 writer.write_u16(length)?;
                 writer.write_all(&trace[0..length as usize])?;
             }
-            Packet::DmaAddTraceReply { succeeded } => {
+            Packet::DmaAddTraceReply {
+                source,
+                destination,
+                id,
+                succeeded,
+            } => {
                 writer.write_u8(0xb1)?;
+                writer.write_u8(source)?;
+                writer.write_u8(destination)?;
+                writer.write_u32(id)?;
                 writer.write_bool(succeeded)?;
             }
-            Packet::DmaRemoveTraceRequest { destination, id } => {
+            Packet::DmaRemoveTraceRequest {
+                source,
+                destination,
+                id,
+            } => {
                 writer.write_u8(0xb2)?;
+                writer.write_u8(source)?;
                 writer.write_u8(destination)?;
                 writer.write_u32(id)?;
             }
-            Packet::DmaRemoveTraceReply { succeeded } => {
+            Packet::DmaRemoveTraceReply { destination, succeeded } => {
                 writer.write_u8(0xb3)?;
+                writer.write_u8(destination)?;
                 writer.write_bool(succeeded)?;
             }
             Packet::DmaPlaybackRequest {
+                source,
                 destination,
                 id,
                 timestamp,
             } => {
                 writer.write_u8(0xb4)?;
+                writer.write_u8(source)?;
                 writer.write_u8(destination)?;
                 writer.write_u32(id)?;
                 writer.write_u64(timestamp)?;
             }
-            Packet::DmaPlaybackReply { succeeded } => {
+            Packet::DmaPlaybackReply { destination, succeeded } => {
                 writer.write_u8(0xb5)?;
+                writer.write_u8(destination)?;
                 writer.write_bool(succeeded)?;
             }
             Packet::DmaPlaybackStatus {
+                source,
                 destination,
                 id,
                 error,
@@ -761,6 +848,7 @@ impl Packet {
                 timestamp,
             } => {
                 writer.write_u8(0xb6)?;
+                writer.write_u8(source)?;
                 writer.write_u8(destination)?;
                 writer.write_u32(id)?;
                 writer.write_u8(error)?;
@@ -771,14 +859,14 @@ impl Packet {
             Packet::SubkernelAddDataRequest {
                 destination,
                 id,
-                last,
+                status,
                 data,
                 length,
             } => {
                 writer.write_u8(0xc0)?;
                 writer.write_u8(destination)?;
                 writer.write_u32(id)?;
-                writer.write_bool(last)?;
+                writer.write_u8(status as u8)?;
                 writer.write_u16(length)?;
                 writer.write_all(&data[0..length as usize])?;
             }
@@ -786,50 +874,67 @@ impl Packet {
                 writer.write_u8(0xc1)?;
                 writer.write_bool(succeeded)?;
             }
-            Packet::SubkernelLoadRunRequest { destination, id, run } => {
+            Packet::SubkernelLoadRunRequest {
+                source,
+                destination,
+                id,
+                run,
+                timestamp,
+            } => {
                 writer.write_u8(0xc4)?;
+                writer.write_u8(source)?;
                 writer.write_u8(destination)?;
                 writer.write_u32(id)?;
                 writer.write_bool(run)?;
+                writer.write_u64(timestamp)?;
             }
-            Packet::SubkernelLoadRunReply { succeeded } => {
+            Packet::SubkernelLoadRunReply { destination, succeeded } => {
                 writer.write_u8(0xc5)?;
-                writer.write_bool(succeeded)?;
-            }
-            Packet::SubkernelStopRequest { destination } => {
-                writer.write_u8(0xc6)?;
                 writer.write_u8(destination)?;
-            }
-            Packet::SubkernelStopReply { succeeded } => {
-                writer.write_u8(0xc7)?;
                 writer.write_bool(succeeded)?;
             }
-            Packet::SubkernelFinished { id, with_exception } => {
+            Packet::SubkernelFinished {
+                destination,
+                id,
+                with_exception,
+                exception_src,
+            } => {
                 writer.write_u8(0xc8)?;
+                writer.write_u8(destination)?;
                 writer.write_u32(id)?;
                 writer.write_bool(with_exception)?;
+                writer.write_u8(exception_src)?;
             }
-            Packet::SubkernelExceptionRequest { destination } => {
+            Packet::SubkernelExceptionRequest { source, destination } => {
                 writer.write_u8(0xc9)?;
+                writer.write_u8(source)?;
                 writer.write_u8(destination)?;
             }
-            Packet::SubkernelException { last, length, data } => {
+            Packet::SubkernelException {
+                destination,
+                last,
+                length,
+                data,
+            } => {
                 writer.write_u8(0xca)?;
+                writer.write_u8(destination)?;
                 writer.write_bool(last)?;
                 writer.write_u16(length)?;
                 writer.write_all(&data[0..length as usize])?;
             }
             Packet::SubkernelMessage {
+                source,
                 destination,
                 id,
-                last,
+                status,
                 data,
                 length,
             } => {
                 writer.write_u8(0xcb)?;
+                writer.write_u8(source)?;
                 writer.write_u8(destination)?;
                 writer.write_u32(id)?;
-                writer.write_bool(last)?;
+                writer.write_u8(status as u8)?;
                 writer.write_u16(length)?;
                 writer.write_all(&data[0..length as usize])?;
             }
@@ -840,4 +945,41 @@ impl Packet {
         }
         Ok(())
     }
+
+    pub fn routable_destination(&self) -> Option<u8> {
+        // only for packets that could be re-routed, not only forwarded
+        match self {
+            Packet::DmaAddTraceRequest { destination, .. } => Some(*destination),
+            Packet::DmaAddTraceReply { destination, .. } => Some(*destination),
+            Packet::DmaRemoveTraceRequest { destination, .. } => Some(*destination),
+            Packet::DmaRemoveTraceReply { destination, .. } => Some(*destination),
+            Packet::DmaPlaybackRequest { destination, .. } => Some(*destination),
+            Packet::DmaPlaybackReply { destination, .. } => Some(*destination),
+            Packet::SubkernelLoadRunRequest { destination, .. } => Some(*destination),
+            Packet::SubkernelLoadRunReply { destination, .. } => Some(*destination),
+            Packet::SubkernelMessage { destination, .. } => Some(*destination),
+            Packet::SubkernelMessageAck { destination } => Some(*destination),
+            Packet::SubkernelExceptionRequest { destination, .. } => Some(*destination),
+            Packet::SubkernelException { destination, .. } => Some(*destination),
+            Packet::DmaPlaybackStatus { destination, .. } => Some(*destination),
+            Packet::SubkernelFinished { destination, .. } => Some(*destination),
+            _ => None,
+        }
+    }
+
+    pub fn expects_response(&self) -> bool {
+        // returns true if the routable packet should elicit a response
+        // e.g. reply, ACK packets end a conversation,
+        // and firmware should not wait for response
+        match self {
+            Packet::DmaAddTraceReply { .. }
+            | Packet::DmaRemoveTraceReply { .. }
+            | Packet::DmaPlaybackReply { .. }
+            | Packet::SubkernelLoadRunReply { .. }
+            | Packet::SubkernelMessageAck { .. }
+            | Packet::DmaPlaybackStatus { .. }
+            | Packet::SubkernelFinished { .. } => false,
+            _ => true,
+        }
+    }
 }

src/libboard_artiq/src/fiq.rs

@@ -0,0 +1,23 @@
+use libboard_zynq::{println, stdio};
+use libcortex_a9::{interrupt_handler, regs::MPIDR};
+use core::arch::asm;
+use libregister::RegisterR;
+
+#[cfg(has_si549)]
+use crate::si549;
+
+interrupt_handler!(FIQ, fiq, __irq_stack0_start, __irq_stack1_start, {
+    match MPIDR.read().cpu_id() {
+        0 => {
+            // nFIQ is driven directly and bypass GIC
+            #[cfg(has_si549)]
+            si549::wrpll::interrupt_handler();
+            return;
+        }
+        _ => {}
+    };
+
+    stdio::drop_uart();
+    println!("FIQ");
+    loop {}
+});

src/libboard_artiq/src/io_expander.rs

@@ -1,6 +1,7 @@
 use libboard_zynq::i2c;
 use log::info;
 
+#[cfg(has_virtual_leds)]
 use crate::pl::csr;
 
 // Only the bare minimum registers. Bits/IO connections equivalent between IC types.
@@ -19,11 +20,15 @@ const IODIR_OUT_SFP_LED: u8 = 0x40;
 const IODIR_OUT_SFP0_LED: u8 = 0x40;
 #[cfg(hw_rev = "v1.1")]
 const IODIR_OUT_SFP0_LED: u8 = 0x80;
+#[cfg(has_si549)]
+const IODIR_CLK_SEL: u8 = 0x80; // out
+#[cfg(has_si5324)]
+const IODIR_CLK_SEL: u8 = 0x00; // in
 
 //IO expander port direction
 const IODIR0: [u8; 2] = [
     0xFF & !IODIR_OUT_SFP_TX_DISABLE & !IODIR_OUT_SFP0_LED,
-    0xFF & !IODIR_OUT_SFP_TX_DISABLE & !IODIR_OUT_SFP_LED,
+    0xFF & !IODIR_OUT_SFP_TX_DISABLE & !IODIR_OUT_SFP_LED & !IODIR_CLK_SEL,
 ];
 
 const IODIR1: [u8; 2] = [
@@ -33,6 +38,7 @@ const IODIR1: [u8; 2] = [
 
 pub struct IoExpander {
     address: u8,
+    #[cfg(has_virtual_leds)]
     virtual_led_mapping: &'static [(u8, u8, u8)],
     iodir: [u8; 2],
     out_current: [u8; 2],
@@ -42,17 +48,18 @@ pub struct IoExpander {
 
 impl IoExpander {
     pub fn new(i2c: &mut i2c::I2c, index: u8) -> Result<Self, &'static str> {
-        #[cfg(hw_rev = "v1.0")]
+        #[cfg(all(hw_rev = "v1.0", has_virtual_leds))]
         const VIRTUAL_LED_MAPPING0: [(u8, u8, u8); 2] = [(0, 0, 6), (1, 1, 6)];
-        #[cfg(hw_rev = "v1.1")]
+        #[cfg(all(hw_rev = "v1.1", has_virtual_leds))]
         const VIRTUAL_LED_MAPPING0: [(u8, u8, u8); 2] = [(0, 0, 7), (1, 1, 6)];
-
+        #[cfg(has_virtual_leds)]
         const VIRTUAL_LED_MAPPING1: [(u8, u8, u8); 2] = [(2, 0, 6), (3, 1, 6)];
 
         // Both expanders on SHARED I2C bus
         let mut io_expander = match index {
             0 => IoExpander {
                 address: 0x40,
+                #[cfg(has_virtual_leds)]
                 virtual_led_mapping: &VIRTUAL_LED_MAPPING0,
                 iodir: IODIR0,
                 out_current: [0; 2],
@@ -66,6 +73,7 @@ impl IoExpander {
             },
             1 => IoExpander {
                 address: 0x42,
+                #[cfg(has_virtual_leds)]
                 virtual_led_mapping: &VIRTUAL_LED_MAPPING1,
                 iodir: IODIR1,
                 out_current: [0; 2],

src/libboard_artiq/src/lib.rs

@@ -1,7 +1,8 @@
 #![no_std]
 #![feature(never_type)]
+#![feature(naked_functions)]
+#![allow(unexpected_cfgs)]
 
-extern crate core_io;
 extern crate crc;
 extern crate embedded_hal;
 extern crate io;
@@ -19,6 +20,7 @@ pub mod drtioaux;
 #[cfg(has_drtio)]
 pub mod drtioaux_async;
 pub mod drtioaux_proto;
+pub mod fiq;
 #[cfg(feature = "target_kasli_soc")]
 pub mod io_expander;
 pub mod logger;
@@ -35,7 +37,8 @@ pub mod drtio_eem;
 pub mod grabber;
 #[cfg(has_si5324)]
 pub mod si5324;
-
+#[cfg(has_si549)]
+pub mod si549;
 use core::{cmp, str};
 
 pub fn identifier_read(buf: &mut [u8]) -> &str {

src/libboard_artiq/src/si549.rs

@@ -0,0 +1,854 @@
+use embedded_hal::prelude::_embedded_hal_blocking_delay_DelayUs;
+use libboard_zynq::timer::GlobalTimer;
+use log::info;
+
+use crate::pl::csr;
+
+#[cfg(feature = "target_kasli_soc")]
+const ADDRESS: u8 = 0x67;
+
+const ADPLL_MAX: i32 = (950.0 / 0.0001164) as i32;
+
+pub struct DividerConfig {
+    pub hsdiv: u16,
+    pub lsdiv: u8,
+    pub fbdiv: u64,
+}
+
+pub struct FrequencySetting {
+    pub main: DividerConfig,
+    pub helper: DividerConfig,
+}
+
+mod i2c {
+    use super::*;
+
+    #[derive(Clone, Copy)]
+    pub enum DCXO {
+        Main,
+        Helper,
+    }
+
+    fn half_period(timer: &mut GlobalTimer) {
+        timer.delay_us(1)
+    }
+
+    fn sda_i(dcxo: DCXO) -> bool {
+        match dcxo {
+            DCXO::Main => unsafe { csr::wrpll::main_dcxo_sda_in_read() == 1 },
+            DCXO::Helper => unsafe { csr::wrpll::helper_dcxo_sda_in_read() == 1 },
+        }
+    }
+
+    fn sda_oe(dcxo: DCXO, oe: bool) {
+        let val = if oe { 1 } else { 0 };
+        match dcxo {
+            DCXO::Main => unsafe { csr::wrpll::main_dcxo_sda_oe_write(val) },
+            DCXO::Helper => unsafe { csr::wrpll::helper_dcxo_sda_oe_write(val) },
+        };
+    }
+
+    fn sda_o(dcxo: DCXO, o: bool) {
+        let val = if o { 1 } else { 0 };
+        match dcxo {
+            DCXO::Main => unsafe { csr::wrpll::main_dcxo_sda_out_write(val) },
+            DCXO::Helper => unsafe { csr::wrpll::helper_dcxo_sda_out_write(val) },
+        };
+    }
+
+    fn scl_oe(dcxo: DCXO, oe: bool) {
+        let val = if oe { 1 } else { 0 };
+        match dcxo {
+            DCXO::Main => unsafe { csr::wrpll::main_dcxo_scl_oe_write(val) },
+            DCXO::Helper => unsafe { csr::wrpll::helper_dcxo_scl_oe_write(val) },
+        };
+    }
+
+    fn scl_o(dcxo: DCXO, o: bool) {
+        let val = if o { 1 } else { 0 };
+        match dcxo {
+            DCXO::Main => unsafe { csr::wrpll::main_dcxo_scl_out_write(val) },
+            DCXO::Helper => unsafe { csr::wrpll::helper_dcxo_scl_out_write(val) },
+        };
+    }
+
+    pub fn init(dcxo: DCXO, timer: &mut GlobalTimer) -> Result<(), &'static str> {
+        // Set SCL as output, and high level
+        scl_o(dcxo, true);
+        scl_oe(dcxo, true);
+        // Prepare a zero level on SDA so that sda_oe pulls it down
+        sda_o(dcxo, false);
+        // Release SDA
+        sda_oe(dcxo, false);
+
+        // Check the I2C bus is ready
+        half_period(timer);
+        half_period(timer);
+        if !sda_i(dcxo) {
+            // Try toggling SCL a few times
+            for _bit in 0..8 {
+                scl_o(dcxo, false);
+                half_period(timer);
+                scl_o(dcxo, true);
+                half_period(timer);
+            }
+        }
+
+        if !sda_i(dcxo) {
+            return Err("SDA is stuck low and doesn't get unstuck");
+        }
+        Ok(())
+    }
+
+    pub fn start(dcxo: DCXO, timer: &mut GlobalTimer) {
+        // Set SCL high then SDA low
+        scl_o(dcxo, true);
+        half_period(timer);
+        sda_oe(dcxo, true);
+        half_period(timer);
+    }
+
+    pub fn stop(dcxo: DCXO, timer: &mut GlobalTimer) {
+        // First, make sure SCL is low, so that the target releases the SDA line
+        scl_o(dcxo, false);
+        half_period(timer);
+        // Set SCL high then SDA high
+        sda_oe(dcxo, true);
+        scl_o(dcxo, true);
+        half_period(timer);
+        sda_oe(dcxo, false);
+        half_period(timer);
+    }
+
+    pub fn write(dcxo: DCXO, data: u8, timer: &mut GlobalTimer) -> bool {
+        // MSB first
+        for bit in (0..8).rev() {
+            // Set SCL low and set our bit on SDA
+            scl_o(dcxo, false);
+            sda_oe(dcxo, data & (1 << bit) == 0);
+            half_period(timer);
+            // Set SCL high ; data is shifted on the rising edge of SCL
+            scl_o(dcxo, true);
+            half_period(timer);
+        }
+        // Check ack
+        // Set SCL low, then release SDA so that the I2C target can respond
+        scl_o(dcxo, false);
+        half_period(timer);
+        sda_oe(dcxo, false);
+        // Set SCL high and check for ack
+        scl_o(dcxo, true);
+        half_period(timer);
+        // returns true if acked (I2C target pulled SDA low)
+        !sda_i(dcxo)
+    }
+
+    pub fn read(dcxo: DCXO, ack: bool, timer: &mut GlobalTimer) -> u8 {
+        // Set SCL low first, otherwise setting SDA as input may cause a transition
+        // on SDA with SCL high which will be interpreted as START/STOP condition.
+        scl_o(dcxo, false);
+        half_period(timer); // make sure SCL has settled low
+        sda_oe(dcxo, false);
+
+        let mut data: u8 = 0;
+
+        // MSB first
+        for bit in (0..8).rev() {
+            scl_o(dcxo, false);
+            half_period(timer);
+            // Set SCL high and shift data
+            scl_o(dcxo, true);
+            half_period(timer);
+            if sda_i(dcxo) {
+                data |= 1 << bit
+            }
+        }
+        // Send ack
+        // Set SCL low and pull SDA low when acking
+        scl_o(dcxo, false);
+        if ack {
+            sda_oe(dcxo, true)
+        }
+        half_period(timer);
+        // then set SCL high
+        scl_o(dcxo, true);
+        half_period(timer);
+
+        data
+    }
+}
+
+fn write(dcxo: i2c::DCXO, reg: u8, val: u8, timer: &mut GlobalTimer) -> Result<(), &'static str> {
+    i2c::start(dcxo, timer);
+    if !i2c::write(dcxo, ADDRESS << 1, timer) {
+        return Err("Si549 failed to ack write address");
+    }
+    if !i2c::write(dcxo, reg, timer) {
+        return Err("Si549 failed to ack register");
+    }
+    if !i2c::write(dcxo, val, timer) {
+        return Err("Si549 failed to ack value");
+    }
+    i2c::stop(dcxo, timer);
+    Ok(())
+}
+
+fn read(dcxo: i2c::DCXO, reg: u8, timer: &mut GlobalTimer) -> Result<u8, &'static str> {
+    i2c::start(dcxo, timer);
+    if !i2c::write(dcxo, ADDRESS << 1, timer) {
+        return Err("Si549 failed to ack write address");
+    }
+    if !i2c::write(dcxo, reg, timer) {
+        return Err("Si549 failed to ack register");
+    }
+    i2c::stop(dcxo, timer);
+
+    i2c::start(dcxo, timer);
+    if !i2c::write(dcxo, (ADDRESS << 1) | 1, timer) {
+        return Err("Si549 failed to ack read address");
+    }
+    let val = i2c::read(dcxo, false, timer);
+    i2c::stop(dcxo, timer);
+    Ok(val)
+}
+
+fn setup(dcxo: i2c::DCXO, config: &DividerConfig, timer: &mut GlobalTimer) -> Result<(), &'static str> {
+    i2c::init(dcxo, timer)?;
+
+    write(dcxo, 255, 0x00, timer)?; // PAGE
+    write(dcxo, 69, 0x00, timer)?; // Disable FCAL override.
+    write(dcxo, 17, 0x00, timer)?; // Synchronously disable output
+
+    // The Si549 has no ID register, so we check that it responds correctly
+    // by writing values to a RAM-like register and reading them back.
+    for test_value in 0..255 {
+        write(dcxo, 23, test_value, timer)?;
+        let readback = read(dcxo, 23, timer)?;
+        if readback != test_value {
+            return Err("Si549 detection failed");
+        }
+    }
+
+    write(dcxo, 23, config.hsdiv as u8, timer)?;
+    write(dcxo, 24, (config.hsdiv >> 8) as u8 | (config.lsdiv << 4), timer)?;
+    write(dcxo, 26, config.fbdiv as u8, timer)?;
+    write(dcxo, 27, (config.fbdiv >> 8) as u8, timer)?;
+    write(dcxo, 28, (config.fbdiv >> 16) as u8, timer)?;
+    write(dcxo, 29, (config.fbdiv >> 24) as u8, timer)?;
+    write(dcxo, 30, (config.fbdiv >> 32) as u8, timer)?;
+    write(dcxo, 31, (config.fbdiv >> 40) as u8, timer)?;
+
+    write(dcxo, 7, 0x08, timer)?; // Start FCAL
+    timer.delay_us(30_000); // Internal FCAL VCO calibration
+    write(dcxo, 17, 0x01, timer)?; // Synchronously enable output
+
+    Ok(())
+}
+
+pub fn main_setup(timer: &mut GlobalTimer, settings: &FrequencySetting) -> Result<(), &'static str> {
+    unsafe {
+        csr::wrpll::main_dcxo_bitbang_enable_write(1);
+        csr::wrpll::main_dcxo_i2c_address_write(ADDRESS);
+    }
+
+    setup(i2c::DCXO::Main, &settings.main, timer)?;
+
+    // Si549 maximum settling time for large frequency change.
+    timer.delay_us(40_000);
+
+    unsafe {
+        csr::wrpll::main_dcxo_bitbang_enable_write(0);
+    }
+
+    info!("Main Si549 started");
+    Ok(())
+}
+
+pub fn helper_setup(timer: &mut GlobalTimer, settings: &FrequencySetting) -> Result<(), &'static str> {
+    unsafe {
+        csr::wrpll::helper_reset_write(1);
+        csr::wrpll::helper_dcxo_bitbang_enable_write(1);
+        csr::wrpll::helper_dcxo_i2c_address_write(ADDRESS);
+    }
+
+    setup(i2c::DCXO::Helper, &settings.helper, timer)?;
+
+    // Si549 maximum settling time for large frequency change.
+    timer.delay_us(40_000);
+
+    unsafe {
+        csr::wrpll::helper_reset_write(0);
+        csr::wrpll::helper_dcxo_bitbang_enable_write(0);
+    }
+    info!("Helper Si549 started");
+    Ok(())
+}
+
+fn set_adpll(dcxo: i2c::DCXO, adpll: i32) -> Result<(), &'static str> {
+    if adpll.abs() > ADPLL_MAX {
+        return Err("adpll is too large");
+    }
+
+    match dcxo {
+        i2c::DCXO::Main => unsafe {
+            if csr::wrpll::main_dcxo_bitbang_enable_read() == 1 {
+                return Err("Main si549 bitbang mode is active when using gateware i2c");
+            }
+
+            while csr::wrpll::main_dcxo_adpll_busy_read() == 1 {}
+            if csr::wrpll::main_dcxo_nack_read() == 1 {
+                return Err("Main si549 failed to ack adpll write");
+            }
+
+            csr::wrpll::main_dcxo_i2c_address_write(ADDRESS);
+            csr::wrpll::main_dcxo_adpll_write(adpll as u32);
+
+            csr::wrpll::main_dcxo_adpll_stb_write(1);
+        },
+        i2c::DCXO::Helper => unsafe {
+            if csr::wrpll::helper_dcxo_bitbang_enable_read() == 1 {
+                return Err("Helper si549 bitbang mode is active when using gateware i2c");
+            }
+
+            while csr::wrpll::helper_dcxo_adpll_busy_read() == 1 {}
+            if csr::wrpll::helper_dcxo_nack_read() == 1 {
+                return Err("Helper si549 failed to ack adpll write");
+            }
+
+            csr::wrpll::helper_dcxo_i2c_address_write(ADDRESS);
+            csr::wrpll::helper_dcxo_adpll_write(adpll as u32);
+
+            csr::wrpll::helper_dcxo_adpll_stb_write(1);
+        },
+    };
+
+    Ok(())
+}
+
+#[cfg(has_wrpll)]
+pub mod wrpll {
+
+    use super::*;
+
+    const BEATING_PERIOD: i32 = 0x8000;
+    const BEATING_HALFPERIOD: i32 = 0x4000;
+    const COUNTER_WIDTH: u32 = 24;
+    const DIV_WIDTH: u32 = 2;
+
+    // y[n] = b0*x[n] + b1*x[n-1] + b2*x[n-2] - a1*y[n-1] - a2*y[n-2]
+    struct FilterParameters {
+        pub b0: f64,
+        pub b1: f64,
+        pub b2: f64,
+        pub a1: f64,
+        pub a2: f64,
+    }
+
+    #[cfg(rtio_frequency = "100.0")]
+    const LPF: FilterParameters = FilterParameters {
+        b0: 0.03967479060647884,
+        b1: 0.07934958121295768,
+        b2: 0.03967479060647884,
+        a1: -1.3865593741228928,
+        a2: 0.5452585365488082,
+    };
+
+    #[cfg(rtio_frequency = "125.0")]
+    const LPF: FilterParameters = FilterParameters {
+        b0: 0.07209205036273991,
+        b1: 0.14418410072547982,
+        b2: 0.07209205036273991,
+        a1: -0.6114078511562919,
+        a2: -0.10022394739274834,
+    };
+
+    static mut H_ADPLL1: i32 = 0;
+    static mut H_ADPLL2: i32 = 0;
+    static mut PERIOD_ERR1: i32 = 0;
+    static mut PERIOD_ERR2: i32 = 0;
+
+    static mut M_ADPLL1: i32 = 0;
+    static mut M_ADPLL2: i32 = 0;
+    static mut PHASE_ERR1: i32 = 0;
+    static mut PHASE_ERR2: i32 = 0;
+
+    static mut BASE_ADPLL: i32 = 0;
+
+    #[derive(Clone, Copy)]
+    pub enum ISR {
+        RefTag,
+        MainTag,
+    }
+
+    mod tag_collector {
+        use super::*;
+
+        #[cfg(wrpll_ref_clk = "GT_CDR")]
+        static mut TAG_OFFSET: u32 = 8382;
+        #[cfg(wrpll_ref_clk = "SMA_CLKIN")]
+        static mut TAG_OFFSET: u32 = 0;
+        static mut REF_TAG: u32 = 0;
+        static mut REF_TAG_READY: bool = false;
+        static mut MAIN_TAG: u32 = 0;
+        static mut MAIN_TAG_READY: bool = false;
+
+        pub fn reset() {
+            clear_phase_diff_ready();
+            unsafe {
+                REF_TAG = 0;
+                MAIN_TAG = 0;
+            }
+        }
+
+        pub fn clear_phase_diff_ready() {
+            unsafe {
+                REF_TAG_READY = false;
+                MAIN_TAG_READY = false;
+            }
+        }
+
+        pub fn collect_tags(interrupt: ISR) {
+            match interrupt {
+                ISR::RefTag => unsafe {
+                    REF_TAG = csr::wrpll::ref_tag_read();
+                    REF_TAG_READY = true;
+                },
+                ISR::MainTag => unsafe {
+                    MAIN_TAG = csr::wrpll::main_tag_read();
+                    MAIN_TAG_READY = true;
+                },
+            }
+        }
+
+        pub fn phase_diff_ready() -> bool {
+            unsafe { REF_TAG_READY && MAIN_TAG_READY }
+        }
+
+        #[cfg(feature = "calibrate_wrpll_skew")]
+        pub fn set_tag_offset(offset: u32) {
+            unsafe {
+                TAG_OFFSET = offset;
+            }
+        }
+
+        #[cfg(feature = "calibrate_wrpll_skew")]
+        pub fn get_tag_offset() -> u32 {
+            unsafe { TAG_OFFSET }
+        }
+
+        pub fn get_period_error() -> i32 {
+            // n * BEATING_PERIOD - REF_TAG(n) mod BEATING_PERIOD
+            let mut period_error = unsafe { REF_TAG.overflowing_neg().0.rem_euclid(BEATING_PERIOD as u32) as i32 };
+            // mapping tags from [0, 2π] -> [-π, π]
+            if period_error > BEATING_HALFPERIOD {
+                period_error -= BEATING_PERIOD
+            }
+            period_error
+        }
+
+        pub fn get_phase_error() -> i32 {
+            // MAIN_TAG(n) - REF_TAG(n) - TAG_OFFSET mod BEATING_PERIOD
+            let mut phase_error = unsafe {
+                MAIN_TAG
+                    .overflowing_sub(REF_TAG + TAG_OFFSET)
+                    .0
+                    .rem_euclid(BEATING_PERIOD as u32) as i32
+            };
+
+            // mapping tags from [0, 2π] -> [-π, π]
+            if phase_error > BEATING_HALFPERIOD {
+                phase_error -= BEATING_PERIOD
+            }
+            phase_error
+        }
+    }
+
+    fn set_isr(en: bool) {
+        let val = if en { 1 } else { 0 };
+        unsafe {
+            csr::wrpll::ref_tag_ev_enable_write(val);
+            csr::wrpll::main_tag_ev_enable_write(val);
+        }
+    }
+
+    fn set_base_adpll() -> Result<(), &'static str> {
+        let count2adpll =
+            |error: i32| ((error as f64 * 1e6) / (0.0001164 * (1 << (COUNTER_WIDTH - DIV_WIDTH)) as f64)) as i32;
+
+        let (ref_count, main_count) = get_freq_counts();
+        unsafe {
+            BASE_ADPLL = count2adpll(ref_count as i32 - main_count as i32);
+            set_adpll(i2c::DCXO::Main, BASE_ADPLL)?;
+            set_adpll(i2c::DCXO::Helper, BASE_ADPLL)?;
+        }
+        Ok(())
+    }
+
+    fn get_freq_counts() -> (u32, u32) {
+        unsafe {
+            csr::wrpll::frequency_counter_update_write(1);
+            while csr::wrpll::frequency_counter_busy_read() == 1 {}
+            #[cfg(wrpll_ref_clk = "GT_CDR")]
+            let ref_count = csr::wrpll::frequency_counter_counter_rtio_rx0_read();
+            #[cfg(wrpll_ref_clk = "SMA_CLKIN")]
+            let ref_count = csr::wrpll::frequency_counter_counter_ref_read();
+            let main_count = csr::wrpll::frequency_counter_counter_sys_read();
+
+            (ref_count, main_count)
+        }
+    }
+
+    fn reset_plls(timer: &mut GlobalTimer) -> Result<(), &'static str> {
+        unsafe {
+            H_ADPLL1 = 0;
+            H_ADPLL2 = 0;
+            PERIOD_ERR1 = 0;
+            PERIOD_ERR2 = 0;
+            M_ADPLL1 = 0;
+            M_ADPLL2 = 0;
+            PHASE_ERR1 = 0;
+            PHASE_ERR2 = 0;
+        }
+        set_adpll(i2c::DCXO::Main, 0)?;
+        set_adpll(i2c::DCXO::Helper, 0)?;
+        // wait for adpll to transfer and DCXO to settle
+        timer.delay_us(200);
+        Ok(())
+    }
+
+    fn clear_pending(interrupt: ISR) {
+        match interrupt {
+            ISR::RefTag => unsafe { csr::wrpll::ref_tag_ev_pending_write(1) },
+            ISR::MainTag => unsafe { csr::wrpll::main_tag_ev_pending_write(1) },
+        };
+    }
+
+    fn is_pending(interrupt: ISR) -> bool {
+        match interrupt {
+            ISR::RefTag => unsafe { csr::wrpll::ref_tag_ev_pending_read() == 1 },
+            ISR::MainTag => unsafe { csr::wrpll::main_tag_ev_pending_read() == 1 },
+        }
+    }
+
+    pub fn interrupt_handler() {
+        if is_pending(ISR::RefTag) {
+            tag_collector::collect_tags(ISR::RefTag);
+            clear_pending(ISR::RefTag);
+            helper_pll().expect("failed to run helper DCXO PLL");
+        }
+
+        if is_pending(ISR::MainTag) {
+            tag_collector::collect_tags(ISR::MainTag);
+            clear_pending(ISR::MainTag);
+        }
+
+        if tag_collector::phase_diff_ready() {
+            main_pll().expect("failed to run main DCXO PLL");
+            tag_collector::clear_phase_diff_ready();
+        }
+    }
+
+    fn helper_pll() -> Result<(), &'static str> {
+        let period_err = tag_collector::get_period_error();
+        unsafe {
+            let adpll = ((LPF.b0 * period_err as f64) + (LPF.b1 * PERIOD_ERR1 as f64) + (LPF.b2 * PERIOD_ERR2 as f64)
+                - (LPF.a1 * H_ADPLL1 as f64)
+                - (LPF.a2 * H_ADPLL2 as f64)) as i32;
+            set_adpll(i2c::DCXO::Helper, BASE_ADPLL + adpll)?;
+            H_ADPLL2 = H_ADPLL1;
+            PERIOD_ERR2 = PERIOD_ERR1;
+            H_ADPLL1 = adpll;
+            PERIOD_ERR1 = period_err;
+        };
+        Ok(())
+    }
+
+    fn main_pll() -> Result<(), &'static str> {
+        let phase_err = tag_collector::get_phase_error();
+        unsafe {
+            let adpll = ((LPF.b0 * phase_err as f64) + (LPF.b1 * PHASE_ERR1 as f64) + (LPF.b2 * PHASE_ERR2 as f64)
+                - (LPF.a1 * M_ADPLL1 as f64)
+                - (LPF.a2 * M_ADPLL2 as f64)) as i32;
+            set_adpll(i2c::DCXO::Main, BASE_ADPLL + adpll)?;
+            M_ADPLL2 = M_ADPLL1;
+            PHASE_ERR2 = PHASE_ERR1;
+            M_ADPLL1 = adpll;
+            PHASE_ERR1 = phase_err;
+        };
+        Ok(())
+    }
+
+    #[cfg(wrpll_ref_clk = "GT_CDR")]
+    fn test_skew(timer: &mut GlobalTimer) -> Result<(), &'static str> {
+        // wait for PLL to stabilize
+        timer.delay_us(20_000);
+
+        info!("testing the skew of SYS CLK...");
+        if has_timing_error(timer) {
+            return Err("the skew cannot satisfy setup/hold time constraint of RX synchronizer");
+        }
+        info!("the skew of SYS CLK met the timing constraint");
+        Ok(())
+    }
+
+    #[cfg(wrpll_ref_clk = "GT_CDR")]
+    fn has_timing_error(timer: &mut GlobalTimer) -> bool {
+        unsafe {
+            csr::wrpll_skewtester::error_write(1);
+        }
+        timer.delay_us(5_000);
+        unsafe { csr::wrpll_skewtester::error_read() == 1 }
+    }
+
+    #[cfg(feature = "calibrate_wrpll_skew")]
+    fn find_edge(target: bool, timer: &mut GlobalTimer) -> Result<u32, &'static str> {
+        const STEP: u32 = 8;
+        const STABLE_THRESHOLD: u32 = 10;
+
+        enum FSM {
+            Init,
+            WaitEdge,
+            GotEdge,
+        }
+
+        let mut state: FSM = FSM::Init;
+        let mut offset: u32 = tag_collector::get_tag_offset();
+        let mut median_edge: u32 = 0;
+        let mut stable_counter: u32 = 0;
+
+        for _ in 0..(BEATING_PERIOD as u32 / STEP) as usize {
+            tag_collector::set_tag_offset(offset);
+            offset += STEP;
+            // wait for PLL to stabilize
+            timer.delay_us(20_000);
+
+            let error = has_timing_error(timer);
+            // A median edge deglitcher
+            match state {
+                FSM::Init => {
+                    if error != target {
+                        stable_counter += 1;
+                    } else {
+                        stable_counter = 0;
+                    }
+
+                    if stable_counter >= STABLE_THRESHOLD {
+                        state = FSM::WaitEdge;
+                        stable_counter = 0;
+                    }
+                }
+                FSM::WaitEdge => {
+                    if error == target {
+                        state = FSM::GotEdge;
+                        median_edge = offset;
+                    }
+                }
+                FSM::GotEdge => {
+                    if error != target {
+                        median_edge += STEP;
+                        stable_counter = 0;
+                    } else {
+                        stable_counter += 1;
+                    }
+
+                    if stable_counter >= STABLE_THRESHOLD {
+                        return Ok(median_edge);
+                    }
+                }
+            }
+        }
+        return Err("failed to find timing error edge");
+    }
+
+    #[cfg(feature = "calibrate_wrpll_skew")]
+    fn calibrate_skew(timer: &mut GlobalTimer) -> Result<(), &'static str> {
+        info!("calibrating skew to meet timing constraint...");
+
+        // clear calibrated value
+        tag_collector::set_tag_offset(0);
+        let rising = find_edge(true, timer)? as i32;
+        let falling = find_edge(false, timer)? as i32;
+
+        let width = BEATING_PERIOD - (falling - rising);
+        let result = falling + width / 2;
+        tag_collector::set_tag_offset(result as u32);
+
+        info!(
+            "calibration successful, error zone: {} -> {}, width: {} ({}deg), middle of working region: {}",
+            rising,
+            falling,
+            width,
+            360 * width / BEATING_PERIOD,
+            result,
+        );
+
+        Ok(())
+    }
+
+    pub fn select_recovered_clock(rc: bool, timer: &mut GlobalTimer) {
+        set_isr(false);
+
+        if rc {
+            tag_collector::reset();
+            reset_plls(timer).expect("failed to reset main and helper PLL");
+
+            // get within capture range
+            set_base_adpll().expect("failed to set base adpll");
+
+            // clear gateware pending flag
+            clear_pending(ISR::RefTag);
+            clear_pending(ISR::MainTag);
+
+            // use nFIQ to avoid IRQ being disabled by mutex lock and mess up PLL
+            set_isr(true);
+            info!("WRPLL interrupt enabled");
+
+            #[cfg(feature = "calibrate_wrpll_skew")]
+            calibrate_skew(timer).expect("failed to set the correct skew");
+
+            #[cfg(wrpll_ref_clk = "GT_CDR")]
+            test_skew(timer).expect("skew test failed");
+        }
+    }
+}
+
+#[cfg(has_wrpll_refclk)]
+pub mod wrpll_refclk {
+    use super::*;
+
+    pub struct MmcmSetting {
+        pub clkout0_reg1: u16,  //0x08
+        pub clkout0_reg2: u16,  //0x09
+        pub clkfbout_reg1: u16, //0x14
+        pub clkfbout_reg2: u16, //0x15
+        pub div_reg: u16,       //0x16
+        pub lock_reg1: u16,     //0x18
+        pub lock_reg2: u16,     //0x19
+        pub lock_reg3: u16,     //0x1A
+        pub power_reg: u16,     //0x28
+        pub filt_reg1: u16,     //0x4E
+        pub filt_reg2: u16,     //0x4F
+    }
+
+    fn one_clock_cycle() {
+        unsafe {
+            csr::wrpll_refclk::mmcm_dclk_write(1);
+            csr::wrpll_refclk::mmcm_dclk_write(0);
+        }
+    }
+
+    fn set_addr(address: u8) {
+        unsafe {
+            csr::wrpll_refclk::mmcm_daddr_write(address);
+        }
+    }
+
+    fn set_data(value: u16) {
+        unsafe {
+            csr::wrpll_refclk::mmcm_din_write(value);
+        }
+    }
+
+    fn set_enable(en: bool) {
+        unsafe {
+            let val = if en { 1 } else { 0 };
+            csr::wrpll_refclk::mmcm_den_write(val);
+        }
+    }
+
+    fn set_write_enable(en: bool) {
+        unsafe {
+            let val = if en { 1 } else { 0 };
+            csr::wrpll_refclk::mmcm_dwen_write(val);
+        }
+    }
+
+    fn get_data() -> u16 {
+        unsafe { csr::wrpll_refclk::mmcm_dout_read() }
+    }
+
+    fn drp_ready() -> bool {
+        unsafe { csr::wrpll_refclk::mmcm_dready_read() == 1 }
+    }
+
+    #[allow(dead_code)]
+    fn read(address: u8) -> u16 {
+        set_addr(address);
+        set_enable(true);
+        // Set DADDR on the mmcm and assert DEN for one clock cycle
+        one_clock_cycle();
+
+        set_enable(false);
+        while !drp_ready() {
+            // keep the clock signal until data is ready
+            one_clock_cycle();
+        }
+        get_data()
+    }
+
+    fn write(address: u8, value: u16) {
+        set_addr(address);
+        set_data(value);
+        set_write_enable(true);
+        set_enable(true);
+        // Set DADDR, DI on the mmcm and assert DWE, DEN for one clock cycle
+        one_clock_cycle();
+
+        set_write_enable(false);
+        set_enable(false);
+        while !drp_ready() {
+            // keep the clock signal until write is finished
+            one_clock_cycle();
+        }
+    }
+
+    fn reset(rst: bool) {
+        unsafe {
+            let val = if rst { 1 } else { 0 };
+            csr::wrpll_refclk::mmcm_reset_write(val)
+        }
+    }
+
+    pub fn setup(timer: &mut GlobalTimer, settings: MmcmSetting, mmcm_bypass: bool) -> Result<(), &'static str> {
+        unsafe {
+            csr::wrpll_refclk::refclk_reset_write(1);
+        }
+
+        if mmcm_bypass {
+            info!("Bypassing mmcm");
+            unsafe {
+                csr::wrpll_refclk::mmcm_bypass_write(1);
+            }
+        } else {
+            // Based on "DRP State Machine" from XAPP888
+            // hold reset HIGH during mmcm config
+            reset(true);
+            write(0x08, settings.clkout0_reg1);
+            write(0x09, settings.clkout0_reg2);
+            write(0x14, settings.clkfbout_reg1);
+            write(0x15, settings.clkfbout_reg2);
+            write(0x16, settings.div_reg);
+            write(0x18, settings.lock_reg1);
+            write(0x19, settings.lock_reg2);
+            write(0x1A, settings.lock_reg3);
+            write(0x28, settings.power_reg);
+            write(0x4E, settings.filt_reg1);
+            write(0x4F, settings.filt_reg2);
+            reset(false);
+
+            // wait for the mmcm to lock
+            timer.delay_us(100);
+
+            let locked = unsafe { csr::wrpll_refclk::mmcm_locked_read() == 1 };
+            if !locked {
+                return Err("mmcm failed to generate 125MHz ref clock from SMA CLKIN");
+            }
+        }
+
+        unsafe {
+            csr::wrpll_refclk::refclk_reset_write(0);
+        }
+
+        Ok(())
+    }
+}

src/libbuild_zynq/lib.rs

@@ -1,3 +1,5 @@
+#![allow(unexpected_cfgs)]
+
 use std::{env,
           fs::File,
           io::{BufRead, BufReader, Write},

src/libbuild_zynq/link.x

@@ -10,7 +10,9 @@ SECTIONS
     __text_start = .;
     .text :
     {
+        __exceptions_start = .;
         KEEP(*(.text.exceptions));
+        __exceptions_end = .;
         *(.text.boot);
         *(.text .text.*);
     } > SDRAM

src/libio/Cargo.toml.tpl

@@ -8,7 +8,6 @@ name = "io"
 path = "lib.rs"
 
 [dependencies]
-core_io = { version = "0.1", features = ["collections"] }
 byteorder = { version = "1.0", default-features = false, optional = true }
 
 libsupport_zynq = { path = "@@ZYNQ_RS@@/libsupport_zynq", default-features = false, features = ["alloc_core"] }

src/libio/cursor.rs

@@ -1,7 +1,7 @@
 #[cfg(feature = "alloc")]
 use alloc::vec::Vec;
 
-use core_io::{Error as IoError, Read, Write};
+use crate::{Read, Write};
 
 #[derive(Debug, Clone)]
 pub struct Cursor<T> {
@@ -42,40 +42,52 @@ impl<T> Cursor<T> {
 }
 
 impl<T: AsRef<[u8]>> Read for Cursor<T> {
-    fn read(&mut self, buf: &mut [u8]) -> Result<usize, IoError> {
+    type ReadError = !;
+
+    fn read(&mut self, buf: &mut [u8]) -> Result<usize, Self::ReadError> {
         let data = &self.inner.as_ref()[self.pos..];
         let len = buf.len().min(data.len());
-        buf[..len].copy_from_slice(&data[..len]);
+        // ``copy_from_slice`` generates AXI bursts, use a regular loop instead
+        for i in 0..len {
+            buf[i] = data[i];
+        }
         self.pos += len;
         Ok(len)
     }
 }
 
 impl Write for Cursor<&mut [u8]> {
-    fn write(&mut self, buf: &[u8]) -> Result<usize, IoError> {
+    type WriteError = !;
+    type FlushError = !;
+
+    fn write(&mut self, buf: &[u8]) -> Result<usize, Self::WriteError> {
         let data = &mut self.inner[self.pos..];
         let len = buf.len().min(data.len());
-        data[..len].copy_from_slice(&buf[..len]);
+        for i in 0..len {
+            data[i] = buf[i];
+        }
         self.pos += len;
         Ok(len)
     }
 
     #[inline]
-    fn flush(&mut self) -> Result<(), IoError> {
+    fn flush(&mut self) -> Result<(), Self::FlushError> {
         Ok(())
     }
 }
 
 #[cfg(feature = "alloc")]
 impl Write for Cursor<Vec<u8>> {
-    #[inline]
+    type WriteError = !;
-    fn write(&mut self, buf: &[u8]) -> Result<usize, IoError> {
+    type FlushError = !;
+
+    fn write(&mut self, buf: &[u8]) -> Result<usize, Self::WriteError> {
         self.inner.extend_from_slice(buf);
         Ok(buf.len())
     }
 
     #[inline]
-    fn flush(&mut self) -> Result<(), IoError> {
+    fn flush(&mut self) -> Result<(), Self::FlushError> {
         Ok(())
     }
 }

src/libio/lib.rs

@@ -1,9 +1,9 @@
 #![no_std]
 #![feature(never_type)]
+#![allow(unexpected_cfgs)]
 
 #[cfg(feature = "alloc")]
 extern crate alloc;
-extern crate core_io;
 
 #[cfg(feature = "byteorder")]
 extern crate byteorder;
@@ -12,8 +12,142 @@ pub mod cursor;
 #[cfg(feature = "byteorder")]
 pub mod proto;
 
+use core::fmt::{Display, Formatter};
+
 pub use cursor::Cursor;
 #[cfg(all(feature = "byteorder", feature = "alloc"))]
 pub use proto::ReadStringError;
 #[cfg(feature = "byteorder")]
 pub use proto::{ProtoRead, ProtoWrite};
+
+#[derive(Debug, Clone, PartialEq)]
+pub enum Error<T> {
+    UnexpectedEnd,
+    Other(T),
+}
+
+impl<T> From<T> for Error<T> {
+    fn from(value: T) -> Error<T> {
+        Error::Other(value)
+    }
+}
+
+impl<T: Display> Display for Error<T> {
+    fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), core::fmt::Error> {
+        match self {
+            Error::UnexpectedEnd => write!(f, "unexpected end of stream"),
+            Error::Other(other) => write!(f, "other IO error: {}", other),
+        }
+    }
+}
+
+pub trait Read {
+    type ReadError;
+
+    /// Pull some bytes from this source into the specified buffer, returning
+    /// how many bytes were read.
+    fn read(&mut self, buf: &mut [u8]) -> Result<usize, Self::ReadError>;
+
+    /// Read the exact number of bytes required to fill `buf`.
+    fn read_exact(&mut self, mut buf: &mut [u8]) -> Result<(), Error<Self::ReadError>> {
+        while !buf.is_empty() {
+            let read_bytes = self.read(buf)?;
+            if read_bytes == 0 {
+                return Err(Error::UnexpectedEnd);
+            }
+
+            buf = &mut { buf }[read_bytes..];
+        }
+
+        Ok(())
+    }
+}
+
+impl<'a, T: Read> Read for &'a mut T {
+    type ReadError = T::ReadError;
+
+    fn read(&mut self, buf: &mut [u8]) -> Result<usize, Self::ReadError> {
+        T::read(self, buf)
+    }
+}
+
+pub trait Write {
+    type WriteError;
+    type FlushError;
+
+    /// Write a buffer into this object, returning how many bytes were written.
+    fn write(&mut self, buf: &[u8]) -> Result<usize, Self::WriteError>;
+
+    /// Flush this output stream, ensuring that all intermediately buffered contents
+    /// reach their destination.
+    fn flush(&mut self) -> Result<(), Self::FlushError>;
+
+    /// Attempts to write an entire buffer into `self`.
+    fn write_all(&mut self, mut buf: &[u8]) -> Result<(), Error<Self::WriteError>> {
+        while buf.len() > 0 {
+            let written_bytes = self.write(buf)?;
+            if written_bytes == 0 {
+                return Err(Error::UnexpectedEnd);
+            }
+
+            buf = &buf[written_bytes..];
+        }
+
+        Ok(())
+    }
+
+    /// Hints the writer how much bytes will be written after call to this function.
+    ///
+    /// At least `min` bytes should be written after the call to this function and
+    /// if `max` is `Some(x)` than at most `x` bytes should be written.
+    fn size_hint(&mut self, _min: usize, _max: Option<usize>) {}
+}
+
+impl<'a, T: Write> Write for &'a mut T {
+    type WriteError = T::WriteError;
+    type FlushError = T::FlushError;
+
+    fn write(&mut self, buf: &[u8]) -> Result<usize, Self::WriteError> {
+        T::write(self, buf)
+    }
+
+    fn flush(&mut self) -> Result<(), Self::FlushError> {
+        T::flush(self)
+    }
+
+    fn size_hint(&mut self, min: usize, max: Option<usize>) {
+        T::size_hint(self, min, max)
+    }
+}
+
+impl<'a> Write for &'a mut [u8] {
+    type WriteError = !;
+    type FlushError = !;
+
+    fn write(&mut self, buf: &[u8]) -> Result<usize, Self::WriteError> {
+        let len = buf.len().min(self.len());
+        self[..len].copy_from_slice(&buf[..len]);
+        Ok(len)
+    }
+
+    #[inline]
+    fn flush(&mut self) -> Result<(), Self::FlushError> {
+        Ok(())
+    }
+}
+
+#[cfg(feature = "alloc")]
+impl<'a> Write for alloc::vec::Vec<u8> {
+    type WriteError = !;
+    type FlushError = !;
+
+    fn write(&mut self, buf: &[u8]) -> Result<usize, Self::WriteError> {
+        self.extend_from_slice(buf);
+        Ok(buf.len())
+    }
+
+    #[inline]
+    fn flush(&mut self) -> Result<(), Self::FlushError> {
+        Ok(())
+    }
+}

src/libio/proto.rs

@@ -3,7 +3,9 @@ use alloc::{string::String, vec};
 use core::str::Utf8Error;
 
 use byteorder::{ByteOrder, NativeEndian};
-use core_io::{Error as IoError, Read, Write};
+use Error as IoError;
+use Read;
+use Write;
 
 #[allow(dead_code)]
 #[derive(Debug, Clone, PartialEq)]
@@ -147,7 +149,7 @@ pub trait ProtoWrite {
 impl<T> ProtoRead for T
 where T: Read + ?Sized
 {
-    type ReadError = IoError;
+    type ReadError = IoError<T::ReadError>;
 
     fn read_exact(&mut self, buf: &mut [u8]) -> Result<(), Self::ReadError> {
         T::read_exact(self, buf)
@@ -157,7 +159,7 @@ where T: Read + ?Sized
 impl<T> ProtoWrite for T
 where T: Write + ?Sized
 {
-    type WriteError = IoError;
+    type WriteError = IoError<T::WriteError>;
 
     fn write_all(&mut self, buf: &[u8]) -> Result<(), Self::WriteError> {
         T::write_all(self, buf)

src/libksupport/Cargo.toml.tpl

@@ -12,12 +12,12 @@ build_zynq = { path = "../libbuild_zynq" }
 cslice = "0.3"
 log = "0.4"
 nb = "0.1"
-core_io = { version = "0.1", features = ["collections"] }
 byteorder = { version = "1.3", default-features = false }
 void = { version = "1", default-features = false }
 log_buffer = { version = "1.2" }
 libm = { version = "0.2", features = ["unstable"] }
 vcell = "0.1"
+nalgebra = { version = "0.33.0", default-features = false, features = ["libm", "alloc"]}
 
 libboard_zynq = { path = "@@ZYNQ_RS@@/libboard_zynq", features = ["ipv6"]}
 libsupport_zynq = { path = "@@ZYNQ_RS@@/libsupport_zynq", default-features = false, features = ["alloc_core"] }

src/libksupport/src/eh_artiq.rs

@@ -14,8 +14,9 @@
 
 use core::mem;
 
-use cslice::CSlice;
+use cslice::{AsCSlice, CSlice};
 use dwarf::eh::{self, EHAction, EHContext};
+use io::{Cursor, Error as ReadError, ProtoRead, Read};
 use libc::{c_int, c_void, uintptr_t};
 use log::{error, trace};
 use unwind as uw;
@@ -94,30 +95,30 @@ struct ExceptionBuffer {
 }
 
 static mut EXCEPTION_BUFFER: ExceptionBuffer = ExceptionBuffer {
-    uw_exceptions: [uw::_Unwind_Exception {
+    uw_exceptions: [ const { uw::_Unwind_Exception {
         exception_class: EXCEPTION_CLASS,
         exception_cleanup: cleanup,
         private: [0; uw::unwinder_private_data_size],
-    }; MAX_INFLIGHT_EXCEPTIONS],
+    } }; MAX_INFLIGHT_EXCEPTIONS],
     exceptions: [None; MAX_INFLIGHT_EXCEPTIONS + 1],
     exception_stack: [-1; MAX_INFLIGHT_EXCEPTIONS + 1],
     backtrace: [(0, 0); MAX_BACKTRACE_SIZE],
     backtrace_size: 0,
-    stack_pointers: [StackPointerBacktrace {
+    stack_pointers: [ const { StackPointerBacktrace {
         stack_pointer: 0,
         initial_backtrace_size: 0,
         current_backtrace_size: 0,
-    }; MAX_INFLIGHT_EXCEPTIONS + 1],
+    } }; MAX_INFLIGHT_EXCEPTIONS + 1],
     exception_count: 0,
 };
 
 pub unsafe extern "C" fn reset_exception_buffer() {
     trace!("reset exception buffer");
-    EXCEPTION_BUFFER.uw_exceptions = [uw::_Unwind_Exception {
+    EXCEPTION_BUFFER.uw_exceptions = [ const { uw::_Unwind_Exception {
         exception_class: EXCEPTION_CLASS,
         exception_cleanup: cleanup,
         private: [0; uw::unwinder_private_data_size],
-    }; MAX_INFLIGHT_EXCEPTIONS];
+    } }; MAX_INFLIGHT_EXCEPTIONS];
     EXCEPTION_BUFFER.exceptions = [None; MAX_INFLIGHT_EXCEPTIONS + 1];
     EXCEPTION_BUFFER.exception_stack = [-1; MAX_INFLIGHT_EXCEPTIONS + 1];
     EXCEPTION_BUFFER.backtrace_size = 0;
@@ -220,8 +221,6 @@ pub unsafe fn artiq_personality(
 }
 
 pub unsafe extern "C" fn raise(exception: *const Exception) -> ! {
-    use cslice::AsCSlice;
-
     let count = EXCEPTION_BUFFER.exception_count;
     let stack = &mut EXCEPTION_BUFFER.exception_stack;
     let diff = exception as isize - EXCEPTION_BUFFER.exceptions.as_ptr() as isize;
@@ -295,6 +294,62 @@ pub unsafe extern "C" fn raise(exception: *const Exception) -> ! {
     unreachable!();
 }
 
+fn read_exception_string<'a, 'b>(
+    reader: &mut Cursor<&'b [u8]>,
+) -> Result<CSlice<'a, u8>, ReadError<<Cursor<&'b [u8]> as Read>::ReadError>> {
+    let len = reader.read_u32()? as usize;
+    if len == usize::MAX {
+        let data = reader.read_u32()?;
+        Ok(unsafe { CSlice::new(data as *const u8, len) })
+    } else {
+        let pos = reader.position();
+        let slice = unsafe {
+            let ptr = reader.get_ref().as_ptr().offset(pos as isize);
+            CSlice::new(ptr, len)
+        };
+        reader.set_position(pos + len);
+        Ok(slice)
+    }
+}
+
+fn read_exception(raw_exception: &[u8]) -> Result<Exception, ReadError<<Cursor<&[u8]> as Read>::ReadError>> {
+    let mut reader = Cursor::new(raw_exception);
+
+    let mut byte = reader.read_u8()?;
+    // to sync
+    while byte != 0x5a {
+        byte = reader.read_u8()?;
+    }
+    // skip sync bytes, 0x09 indicates exception
+    while byte != 0x09 {
+        byte = reader.read_u8()?;
+    }
+    let _len = reader.read_u32()?;
+    // ignore the remaining exceptions, stack traces etc. - unwinding from another device would be unwise anyway
+    Ok(Exception {
+        id: reader.read_u32()?,
+        message: read_exception_string(&mut reader)?,
+        param: [
+            reader.read_u64()? as i64,
+            reader.read_u64()? as i64,
+            reader.read_u64()? as i64,
+        ],
+        file: read_exception_string(&mut reader)?,
+        line: reader.read_u32()?,
+        column: reader.read_u32()?,
+        function: read_exception_string(&mut reader)?,
+    })
+}
+
+pub fn raise_raw(raw_exception: &[u8]) -> ! {
+    use crate::artiq_raise;
+    if let Ok(exception) = read_exception(raw_exception) {
+        unsafe { raise(&exception) };
+    } else {
+        artiq_raise!("SubkernelError", "Error passing exception");
+    }
+}
+
 pub unsafe extern "C" fn resume() -> ! {
     trace!("resume");
     assert!(EXCEPTION_BUFFER.exception_count != 0);
@@ -421,20 +476,30 @@ extern "C" fn stop_fn(
     }
 }
 
-// Must be kept in sync with preallocate_runtime_exception_names() in artiq/language/embedding_map.py
+// Must be kept in sync with preallocate_runtime_exception_names() in `artiq.compiler.embedding`
-static EXCEPTION_ID_LOOKUP: [(&str, u32); 12] = [
+static EXCEPTION_ID_LOOKUP: [(&str, u32); 22] = [
-    ("RuntimeError", 0),
+    ("RTIOUnderflow", 0),
-    ("RTIOUnderflow", 1),
+    ("RTIOOverflow", 1),
-    ("RTIOOverflow", 2),
+    ("RTIODestinationUnreachable", 2),
-    ("RTIODestinationUnreachable", 3),
+    ("DMAError", 3),
-    ("DMAError", 4),
+    ("I2CError", 4),
-    ("I2CError", 5),
+    ("CacheError", 5),
-    ("CacheError", 6),
+    ("SPIError", 6),
-    ("SPIError", 7),
+    ("SubkernelError", 7),
-    ("ZeroDivisionError", 8),
+    ("AssertionError", 8),
-    ("IndexError", 9),
+    ("AttributeError", 9),
-    ("UnwrapNoneError", 10),
+    ("IndexError", 10),
-    ("SubkernelError", 11),
+    ("IOError", 11),
+    ("KeyError", 12),
+    ("NotImplementedError", 13),
+    ("OverflowError", 14),
+    ("RuntimeError", 15),
+    ("TimeoutError", 16),
+    ("TypeError", 17),
+    ("ValueError", 18),
+    ("ZeroDivisionError", 19),
+    ("LinAlgError", 20),
+    ("UnwrapNoneError", 21),
 ];
 
 pub fn get_exception_id(name: &str) -> u32 {
@@ -469,3 +534,29 @@ macro_rules! artiq_raise {
     }};
     ($name:expr, $message:expr) => {{ artiq_raise!($name, $message, 0, 0, 0) }};
 }
+
+/// Takes as input exception id from host
+/// Generates a new exception with:
+///   * `id` set to `exn_id`
+///   * `message` set to corresponding exception name from `EXCEPTION_ID_LOOKUP`
+///
+/// The message is matched on host to ensure correct exception is being referred
+/// This test checks the synchronization of exception ids for runtime errors
+#[no_mangle]
+pub extern "C" fn test_exception_id_sync(exn_id: u32) {
+    let message = EXCEPTION_ID_LOOKUP
+        .iter()
+        .find_map(|&(name, id)| if id == exn_id { Some(name) } else { None })
+        .unwrap_or("unallocated internal exception id");
+
+    let exn = Exception {
+        id: exn_id,
+        file: file!().as_c_slice(),
+        line: 0,
+        column: 0,
+        function: "test_exception_id_sync".as_c_slice(),
+        message: message.as_c_slice(),
+        param: [0, 0, 0],
+    };
+    unsafe { raise(&exn) };
+}

src/libksupport/src/eh_rust.rs

@@ -0,0 +1,149 @@
+// Copyright 2015 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+// This is the Rust personality function, adapted for use in ARTIQ. We never actually panic
+// from Rust or recover from Rust exceptions (there's nothing to catch the panics), but we
+// need a personality function to step back through Rust frames in order to make a backtrace.
+//
+// By design, this personality function is only ever called in the search phase, although
+// to keep things simple and close to upstream, it is not modified
+use unwind as uw;
+use libc::c_int;
+
+use dwarf::eh::{EHAction, EHContext};
+
+// Register ids were lifted from LLVM's TargetLowering::getExceptionPointerRegister()
+// and TargetLowering::getExceptionSelectorRegister() for each architecture,
+// then mapped to DWARF register numbers via register definition tables
+// (typically <arch>RegisterInfo.td, search for "DwarfRegNum").
+// See also http://llvm.org/docs/WritingAnLLVMBackend.html#defining-a-register.
+
+#[cfg(target_arch = "x86")]
+const UNWIND_DATA_REG: (i32, i32) = (0, 2); // EAX, EDX
+
+#[cfg(target_arch = "x86_64")]
+const UNWIND_DATA_REG: (i32, i32) = (0, 1); // RAX, RDX
+
+#[cfg(any(target_arch = "riscv32"))]
+const UNWIND_DATA_REG: (i32, i32) = (10, 11); // X10, X11
+
+#[cfg(any(target_arch = "arm", target_arch = "aarch64"))]
+const UNWIND_DATA_REG: (i32, i32) = (0, 1); // R0, R1 / X0, X1
+
+// The following code is based on GCC's C and C++ personality routines.  For reference, see:
+// https://github.com/gcc-mirror/gcc/blob/master/libstdc++-v3/libsupc++/eh_personality.cc
+// https://github.com/gcc-mirror/gcc/blob/trunk/libgcc/unwind-c.c
+#[lang = "eh_personality"]
+#[no_mangle]
+unsafe extern "C" fn rust_eh_personality(
+    state: uw::_Unwind_State,
+    exception_object: *mut uw::_Unwind_Exception,
+    context: *mut uw::_Unwind_Context,
+) -> uw::_Unwind_Reason_Code {
+    let state = state as c_int;
+    let action = state & uw::_US_ACTION_MASK as c_int;
+    let search_phase = if action == uw::_US_VIRTUAL_UNWIND_FRAME as c_int {
+        // Backtraces on ARM will call the personality routine with
+        // state == _US_VIRTUAL_UNWIND_FRAME | _US_FORCE_UNWIND. In those cases
+        // we want to continue unwinding the stack, otherwise all our backtraces
+        // would end at __rust_try
+        if state & uw::_US_FORCE_UNWIND as c_int != 0 {
+            return continue_unwind(exception_object, context);
+        }
+        true
+    } else if action == uw::_US_UNWIND_FRAME_STARTING as c_int {
+        false
+    } else if action == uw::_US_UNWIND_FRAME_RESUME as c_int {
+        return continue_unwind(exception_object, context);
+    } else {
+        return uw::_URC_FAILURE;
+    };
+
+    // The DWARF unwinder assumes that _Unwind_Context holds things like the function
+    // and LSDA pointers, however ARM EHABI places them into the exception object.
+    // To preserve signatures of functions like _Unwind_GetLanguageSpecificData(), which
+    // take only the context pointer, GCC personality routines stash a pointer to
+    // exception_object in the context, using location reserved for ARM's
+    // "scratch register" (r12).
+    uw::_Unwind_SetGR(context, uw::UNWIND_POINTER_REG, exception_object as uw::_Unwind_Ptr);
+    // ...A more principled approach would be to provide the full definition of ARM's
+    // _Unwind_Context in our libunwind bindings and fetch the required data from there
+    // directly, bypassing DWARF compatibility functions.
+
+    let eh_action = match find_eh_action(context) {
+        Ok(action) => action,
+        Err(_) => return uw::_URC_FAILURE,
+    };
+    if search_phase {
+        match eh_action {
+            EHAction::None | EHAction::Cleanup(_) => {
+                return continue_unwind(exception_object, context);
+            }
+            EHAction::Catch(_) => {
+                // EHABI requires the personality routine to update the
+                // SP value in the barrier cache of the exception object.
+                (*exception_object).private[5] =
+                    uw::_Unwind_GetGR(context, uw::UNWIND_SP_REG);
+                return uw::_URC_HANDLER_FOUND;
+            }
+            EHAction::Terminate => return uw::_URC_FAILURE,
+        }
+    } else {
+        match eh_action {
+            EHAction::None => return continue_unwind(exception_object, context),
+            EHAction::Cleanup(lpad) | EHAction::Catch(lpad) => {
+                uw::_Unwind_SetGR(
+                    context,
+                    UNWIND_DATA_REG.0,
+                    exception_object as uw::_Unwind_Ptr,
+                );
+                uw::_Unwind_SetGR(context, UNWIND_DATA_REG.1, 0);
+                uw::_Unwind_SetIP(context, lpad);
+                return uw::_URC_INSTALL_CONTEXT;
+            }
+            EHAction::Terminate => return uw::_URC_FAILURE,
+        }
+    }
+
+    // On ARM EHABI the personality routine is responsible for actually
+    // unwinding a single stack frame before returning (ARM EHABI Sec. 6.1).
+    unsafe fn continue_unwind(
+        exception_object: *mut uw::_Unwind_Exception,
+        context: *mut uw::_Unwind_Context,
+    ) -> uw::_Unwind_Reason_Code {
+        if __gnu_unwind_frame(exception_object, context) == uw::_URC_NO_REASON {
+            uw::_URC_CONTINUE_UNWIND
+        } else {
+            uw::_URC_FAILURE
+        }
+    }
+    // defined in libgcc
+    extern "C" {
+        fn __gnu_unwind_frame(
+            exception_object: *mut uw::_Unwind_Exception,
+            context: *mut uw::_Unwind_Context,
+        ) -> uw::_Unwind_Reason_Code;
+    }
+}
+
+unsafe fn find_eh_action(context: *mut uw::_Unwind_Context) -> Result<EHAction, ()> {
+    let lsda = uw::_Unwind_GetLanguageSpecificData(context) as *const u8;
+    let mut ip_before_instr: c_int = 0;
+    let ip = uw::_Unwind_GetIPInfo(context, &mut ip_before_instr);
+    let eh_context = EHContext {
+        // The return address points 1 byte past the call instruction,
+        // which could be in the next IP range in LSDA range table.
+        ip: if ip_before_instr != 0 { ip } else { ip - 1 },
+        func_start: uw::_Unwind_GetRegionStart(context),
+        get_text_start: &|| uw::_Unwind_GetTextRelBase(context),
+        get_data_start: &|| uw::_Unwind_GetDataRelBase(context),
+    };
+    dwarf::eh::find_eh_action(lsda, &eh_context, false,0)
+}

src/libksupport/src/irq.rs

@@ -1,5 +1,5 @@
 use core::sync::atomic::{AtomicBool, Ordering};
-
+use core::arch::asm;
 use libboard_zynq::{gic, mpcore, println, stdio};
 use libcortex_a9::{asm, interrupt_handler, notify_spin_lock, regs::MPIDR, spin_lock_yield};
 use libregister::RegisterR;

src/libksupport/src/kernel/api.rs

@@ -9,7 +9,7 @@ use log::{info, warn};
 use super::subkernel;
 use super::{cache,
             core1::rtio_get_destination_status,
-            dma,
+            dma, linalg,
             rpc::{rpc_recv, rpc_send, rpc_send_async}};
 use crate::{eh_artiq, i2c, rtio};
 
@@ -303,6 +303,7 @@ pub fn resolve(required: &[u8]) -> Option<u32> {
         api_libm_f64f64f64!(nextafter),
         api_libm_f64f64f64!(pow),
         api_libm_f64f64!(round),
+        api_libm_f64f64!(rint),
         api_libm_f64f64!(sin),
         api_libm_f64f64!(sinh),
         api_libm_f64f64!(sqrt),
@@ -318,6 +319,26 @@ pub fn resolve(required: &[u8]) -> Option<u32> {
             }
             api!(yn = yn)
         },
+
+        // linalg
+        api!(np_linalg_cholesky = linalg::np_linalg_cholesky),
+        api!(np_linalg_qr = linalg::np_linalg_qr),
+        api!(np_linalg_svd = linalg::np_linalg_svd),
+        api!(np_linalg_inv = linalg::np_linalg_inv),
+        api!(np_linalg_pinv = linalg::np_linalg_pinv),
+        api!(np_linalg_matrix_power = linalg::np_linalg_matrix_power),
+        api!(np_linalg_det = linalg::np_linalg_det),
+        api!(sp_linalg_lu = linalg::sp_linalg_lu),
+        api!(sp_linalg_schur = linalg::sp_linalg_schur),
+        api!(sp_linalg_hessenberg = linalg::sp_linalg_hessenberg),
+
+        /*
+         * syscall for unit tests
+         * Used in `artiq.tests.coredevice.test_exceptions.ExceptionTest.test_raise_exceptions_kernel`
+         * This syscall checks that the exception IDs used in the Python `EmbeddingMap` (in `artiq.language.embedding`)
+         * match the `EXCEPTION_ID_LOOKUP` defined in the firmware (`libksupport::src::eh_artiq`)
+         */
+        api!(test_exception_id_sync = eh_artiq::test_exception_id_sync)
     ];
     api.iter()
         .find(|&&(exported, _)| exported.as_bytes() == required)

src/libksupport/src/kernel/core1.rs

@@ -25,12 +25,14 @@ extern "C" {
 }
 
 unsafe fn attribute_writeback(typeinfo: *const ()) {
+    #[repr(C)]
     struct Attr {
         offset: usize,
         tag: CSlice<'static, u8>,
         name: CSlice<'static, u8>,
     }
 
+    #[repr(C)]
     struct Type {
         attributes: *const *const Attr,
         objects: *const *const (),

src/libksupport/src/kernel/dma.rs

@@ -170,6 +170,7 @@ pub extern "C" fn dma_playback(timestamp: i64, ptr: i32, _uses_ddma: bool) {
         csr::rtio_dma::base_address_write(ptr as u32);
         csr::rtio_dma::time_offset_write(timestamp as u64);
 
+        let old_cri_master = csr::cri_con::selected_read();
         csr::cri_con::selected_write(1);
         csr::rtio_dma::enable_write(1);
         #[cfg(has_drtio)]
@@ -183,7 +184,7 @@ pub extern "C" fn dma_playback(timestamp: i64, ptr: i32, _uses_ddma: bool) {
                 });
         }
         while csr::rtio_dma::enable_read() != 0 {}
-        csr::cri_con::selected_write(0);
+        csr::cri_con::selected_write(old_cri_master);
 
         let error = csr::rtio_dma::error_read();
         if error != 0 {

src/libksupport/src/kernel/linalg.rs

@@ -0,0 +1,440 @@
+// Uses `nalgebra` crate to invoke `np_linalg` and `sp_linalg` functions
+// When converting between `nalgebra::Matrix` and `NDArray` following considerations are necessary
+//
+// * Both `nalgebra::Matrix` and `NDArray` require their content to be stored in row-major order
+// * `NDArray` data pointer can be directly read and converted to `nalgebra::Matrix` (row and column number must be known)
+// * `nalgebra::Matrix::as_slice` returns the content of matrix in column-major order and initial data needs to be transposed before storing it in `NDArray` data pointer
+
+use alloc::vec::Vec;
+use core::slice;
+
+use nalgebra::DMatrix;
+
+use crate::artiq_raise;
+
+pub struct InputMatrix {
+    pub ndims: usize,
+    pub dims: *const usize,
+    pub data: *mut f64,
+}
+
+impl InputMatrix {
+    fn get_dims(&mut self) -> Vec<usize> {
+        let dims = unsafe { slice::from_raw_parts(self.dims, self.ndims) };
+        dims.to_vec()
+    }
+}
+
+/// # Safety
+///
+/// `mat1` should point to a valid 2DArray of `f64` floats in row-major order
+#[no_mangle]
+pub unsafe extern "C" fn np_linalg_cholesky(mat1: *mut InputMatrix, out: *mut InputMatrix) {
+    let mat1 = mat1.as_mut().unwrap();
+    let out = out.as_mut().unwrap();
+
+    if mat1.ndims != 2 {
+        artiq_raise!(
+            "ValueError",
+            "expected 2D Vector Input, but received {1}D input)",
+            0,
+            mat1.ndims as i64,
+            0
+        );
+    }
+
+    let dim1 = (*mat1).get_dims();
+    if dim1[0] != dim1[1] {
+        artiq_raise!(
+            "ValueError",
+            "last 2 dimensions of the array must be square: {1} != {2}",
+            0,
+            dim1[0] as i64,
+            dim1[1] as i64
+        );
+    }
+
+    let outdim = out.get_dims();
+    let out_slice = slice::from_raw_parts_mut(out.data, outdim[0] * outdim[1]);
+    let data_slice1 = slice::from_raw_parts_mut(mat1.data, dim1[0] * dim1[1]);
+
+    let matrix1 = DMatrix::from_row_slice(dim1[0], dim1[1], data_slice1);
+    let result = matrix1.cholesky();
+    match result {
+        Some(res) => {
+            out_slice.copy_from_slice(res.unpack().transpose().as_slice());
+        }
+        None => {
+            artiq_raise!("LinAlgError", "Matrix is not positive definite");
+        }
+    };
+}
+
+/// # Safety
+///
+/// `mat1` should point to a valid 2DArray of `f64` floats in row-major order
+#[no_mangle]
+pub unsafe extern "C" fn np_linalg_qr(mat1: *mut InputMatrix, out_q: *mut InputMatrix, out_r: *mut InputMatrix) {
+    let mat1 = mat1.as_mut().unwrap();
+    let out_q = out_q.as_mut().unwrap();
+    let out_r = out_r.as_mut().unwrap();
+
+    if mat1.ndims != 2 {
+        artiq_raise!(
+            "ValueError",
+            "expected 2D Vector Input, but received {1}D input)",
+            0,
+            mat1.ndims as i64,
+            0
+        );
+    }
+
+    let dim1 = (*mat1).get_dims();
+    let outq_dim = (*out_q).get_dims();
+    let outr_dim = (*out_r).get_dims();
+
+    let data_slice1 = slice::from_raw_parts_mut(mat1.data, dim1[0] * dim1[1]);
+    let out_q_slice = slice::from_raw_parts_mut(out_q.data, outq_dim[0] * outq_dim[1]);
+    let out_r_slice = slice::from_raw_parts_mut(out_r.data, outr_dim[0] * outr_dim[1]);
+
+    // Refer to https://github.com/dimforge/nalgebra/issues/735
+    let matrix1 = DMatrix::from_row_slice(dim1[0], dim1[1], data_slice1);
+
+    let res = matrix1.qr();
+    let (q, r) = res.unpack();
+
+    // Uses different algo need to match numpy
+    out_q_slice.copy_from_slice(q.transpose().as_slice());
+    out_r_slice.copy_from_slice(r.transpose().as_slice());
+}
+
+/// # Safety
+///
+/// `mat1` should point to a valid 2DArray of `f64` floats in row-major order
+#[no_mangle]
+pub unsafe extern "C" fn np_linalg_svd(
+    mat1: *mut InputMatrix,
+    outu: *mut InputMatrix,
+    outs: *mut InputMatrix,
+    outvh: *mut InputMatrix,
+) {
+    let mat1 = mat1.as_mut().unwrap();
+    let outu = outu.as_mut().unwrap();
+    let outs = outs.as_mut().unwrap();
+    let outvh = outvh.as_mut().unwrap();
+
+    if mat1.ndims != 2 {
+        artiq_raise!(
+            "ValueError",
+            "expected 2D Vector Input, but received {1}D input)",
+            0,
+            mat1.ndims as i64,
+            0
+        );
+    }
+
+    let dim1 = (*mat1).get_dims();
+    let outu_dim = (*outu).get_dims();
+    let outs_dim = (*outs).get_dims();
+    let outvh_dim = (*outvh).get_dims();
+
+    let data_slice1 = slice::from_raw_parts_mut(mat1.data, dim1[0] * dim1[1]);
+    let out_u_slice = slice::from_raw_parts_mut(outu.data, outu_dim[0] * outu_dim[1]);
+    let out_s_slice = slice::from_raw_parts_mut(outs.data, outs_dim[0]);
+    let out_vh_slice = slice::from_raw_parts_mut(outvh.data, outvh_dim[0] * outvh_dim[1]);
+
+    let matrix = DMatrix::from_row_slice(dim1[0], dim1[1], data_slice1);
+    let result = matrix.svd(true, true);
+    out_u_slice.copy_from_slice(result.u.unwrap().transpose().as_slice());
+    out_s_slice.copy_from_slice(result.singular_values.as_slice());
+    out_vh_slice.copy_from_slice(result.v_t.unwrap().transpose().as_slice());
+}
+
+/// # Safety
+///
+/// `mat1` should point to a valid 2DArray of `f64` floats in row-major order
+#[no_mangle]
+pub unsafe extern "C" fn np_linalg_inv(mat1: *mut InputMatrix, out: *mut InputMatrix) {
+    let mat1 = mat1.as_mut().unwrap();
+    let out = out.as_mut().unwrap();
+
+    if mat1.ndims != 2 {
+        artiq_raise!(
+            "ValueError",
+            "expected 2D Vector Input, but received {1}D input)",
+            0,
+            mat1.ndims as i64,
+            0
+        );
+    }
+    let dim1 = (*mat1).get_dims();
+
+    if dim1[0] != dim1[1] {
+        artiq_raise!(
+            "ValueError",
+            "last 2 dimensions of the array must be square: {1} != {2}",
+            0,
+            dim1[0] as i64,
+            dim1[1] as i64
+        );
+    }
+
+    let outdim = out.get_dims();
+    let out_slice = slice::from_raw_parts_mut(out.data, outdim[0] * outdim[1]);
+    let data_slice1 = slice::from_raw_parts_mut(mat1.data, dim1[0] * dim1[1]);
+
+    let matrix = DMatrix::from_row_slice(dim1[0], dim1[1], data_slice1);
+    if !matrix.is_invertible() {
+        artiq_raise!("LinAlgError", "no inverse for Singular Matrix");
+    }
+    let inv = matrix.try_inverse().unwrap();
+    out_slice.copy_from_slice(inv.transpose().as_slice());
+}
+
+/// # Safety
+///
+/// `mat1` should point to a valid 2DArray of `f64` floats in row-major order
+#[no_mangle]
+pub unsafe extern "C" fn np_linalg_pinv(mat1: *mut InputMatrix, out: *mut InputMatrix) {
+    let mat1 = mat1.as_mut().unwrap();
+    let out = out.as_mut().unwrap();
+
+    if mat1.ndims != 2 {
+        artiq_raise!(
+            "ValueError",
+            "expected 2D Vector Input, but received {1}D input)",
+            0,
+            mat1.ndims as i64,
+            0
+        );
+    }
+    let dim1 = (*mat1).get_dims();
+    let outdim = out.get_dims();
+    let out_slice = slice::from_raw_parts_mut(out.data, outdim[0] * outdim[1]);
+    let data_slice1 = slice::from_raw_parts_mut(mat1.data, dim1[0] * dim1[1]);
+
+    let matrix = DMatrix::from_row_slice(dim1[0], dim1[1], data_slice1);
+    let svd = matrix.svd(true, true);
+    let inv = svd.pseudo_inverse(1e-15);
+
+    match inv {
+        Ok(m) => {
+            out_slice.copy_from_slice(m.transpose().as_slice());
+        }
+        Err(_) => {
+            artiq_raise!("LinAlgError", "SVD computation does not converge");
+        }
+    }
+}
+
+/// # Safety
+///
+/// `mat1` should point to a valid 2DArray of `f64` floats in row-major order
+#[no_mangle]
+pub unsafe extern "C" fn np_linalg_matrix_power(mat1: *mut InputMatrix, mat2: *mut InputMatrix, out: *mut InputMatrix) {
+    let mat1 = mat1.as_mut().unwrap();
+    let mat2 = mat2.as_mut().unwrap();
+    let out = out.as_mut().unwrap();
+
+    if mat1.ndims != 2 {
+        artiq_raise!(
+            "ValueError",
+            "expected 2D Vector Input, but received {1}D input)",
+            0,
+            mat1.ndims as i64,
+            0
+        );
+    }
+
+    let dim1 = (*mat1).get_dims();
+    let power = slice::from_raw_parts_mut(mat2.data, 1);
+    let power = power[0];
+    let outdim = out.get_dims();
+    let out_slice = slice::from_raw_parts_mut(out.data, outdim[0] * outdim[1]);
+    let data_slice1 = slice::from_raw_parts_mut(mat1.data, dim1[0] * dim1[1]);
+    let mut abs_power = power;
+    if abs_power < 0.0 {
+        abs_power = abs_power * -1.0;
+    }
+    let matrix1 = DMatrix::from_row_slice(dim1[0], dim1[1], data_slice1);
+    if !matrix1.is_square() {
+        artiq_raise!(
+            "ValueError",
+            "last 2 dimensions of the array must be square: {1} != {2}",
+            0,
+            dim1[0] as i64,
+            dim1[1] as i64
+        );
+    }
+    let mut result = matrix1.pow(abs_power as u32);
+
+    if power < 0.0 {
+        if !matrix1.is_invertible() {
+            artiq_raise!("LinAlgError", "no inverse for Singular Matrix");
+        }
+        result = result.try_inverse().unwrap();
+    }
+    out_slice.copy_from_slice(result.transpose().as_slice());
+}
+
+/// # Safety
+///
+/// `mat1` should point to a valid 2DArray of `f64` floats in row-major order
+#[no_mangle]
+pub unsafe extern "C" fn np_linalg_det(mat1: *mut InputMatrix, out: *mut InputMatrix) {
+    let mat1 = mat1.as_mut().unwrap();
+    let out = out.as_mut().unwrap();
+
+    if mat1.ndims != 2 {
+        artiq_raise!(
+            "ValueError",
+            "expected 2D Vector Input, but received {1}D input)",
+            0,
+            mat1.ndims as i64,
+            0
+        );
+    }
+    let dim1 = (*mat1).get_dims();
+    let out_slice = slice::from_raw_parts_mut(out.data, 1);
+    let data_slice1 = slice::from_raw_parts_mut(mat1.data, dim1[0] * dim1[1]);
+
+    let matrix = DMatrix::from_row_slice(dim1[0], dim1[1], data_slice1);
+    if !matrix.is_square() {
+        artiq_raise!(
+            "ValueError",
+            "last 2 dimensions of the array must be square: {1} != {2}",
+            0,
+            dim1[0] as i64,
+            dim1[1] as i64
+        );
+    }
+    out_slice[0] = matrix.determinant();
+}
+
+/// # Safety
+///
+/// `mat1` should point to a valid 2DArray of `f64` floats in row-major order
+#[no_mangle]
+pub unsafe extern "C" fn sp_linalg_lu(mat1: *mut InputMatrix, out_l: *mut InputMatrix, out_u: *mut InputMatrix) {
+    let mat1 = mat1.as_mut().unwrap();
+    let out_l = out_l.as_mut().unwrap();
+    let out_u = out_u.as_mut().unwrap();
+
+    if mat1.ndims != 2 {
+        artiq_raise!(
+            "ValueError",
+            "expected 2D Vector Input, but received {1}D input)",
+            0,
+            mat1.ndims as i64,
+            0
+        );
+    }
+
+    let dim1 = (*mat1).get_dims();
+    let outl_dim = (*out_l).get_dims();
+    let outu_dim = (*out_u).get_dims();
+
+    let data_slice1 = slice::from_raw_parts_mut(mat1.data, dim1[0] * dim1[1]);
+    let out_l_slice = slice::from_raw_parts_mut(out_l.data, outl_dim[0] * outl_dim[1]);
+    let out_u_slice = slice::from_raw_parts_mut(out_u.data, outu_dim[0] * outu_dim[1]);
+
+    let matrix = DMatrix::from_row_slice(dim1[0], dim1[1], data_slice1);
+    let (_, l, u) = matrix.lu().unpack();
+
+    out_l_slice.copy_from_slice(l.transpose().as_slice());
+    out_u_slice.copy_from_slice(u.transpose().as_slice());
+}
+
+/// # Safety
+///
+/// `mat1` should point to a valid 2DArray of `f64` floats in row-major order
+#[no_mangle]
+pub unsafe extern "C" fn sp_linalg_schur(mat1: *mut InputMatrix, out_t: *mut InputMatrix, out_z: *mut InputMatrix) {
+    let mat1 = mat1.as_mut().unwrap();
+    let out_t = out_t.as_mut().unwrap();
+    let out_z = out_z.as_mut().unwrap();
+
+    if mat1.ndims != 2 {
+        artiq_raise!(
+            "ValueError",
+            "expected 2D Vector Input, but received {1}D input)",
+            0,
+            mat1.ndims as i64,
+            0
+        );
+    }
+
+    let dim1 = (*mat1).get_dims();
+
+    if dim1[0] != dim1[1] {
+        artiq_raise!(
+            "ValueError",
+            "last 2 dimensions of the array must be square: {1} != {2}",
+            0,
+            dim1[0] as i64,
+            dim1[1] as i64
+        );
+    }
+
+    let out_t_dim = (*out_t).get_dims();
+    let out_z_dim = (*out_z).get_dims();
+
+    let data_slice1 = slice::from_raw_parts_mut(mat1.data, dim1[0] * dim1[1]);
+    let out_t_slice = slice::from_raw_parts_mut(out_t.data, out_t_dim[0] * out_t_dim[1]);
+    let out_z_slice = slice::from_raw_parts_mut(out_z.data, out_z_dim[0] * out_z_dim[1]);
+
+    let matrix = DMatrix::from_row_slice(dim1[0], dim1[1], data_slice1);
+    let (z, t) = matrix.schur().unpack();
+
+    out_t_slice.copy_from_slice(t.transpose().as_slice());
+    out_z_slice.copy_from_slice(z.transpose().as_slice());
+}
+
+/// # Safety
+///
+/// `mat1` should point to a valid 2DArray of `f64` floats in row-major order
+#[no_mangle]
+pub unsafe extern "C" fn sp_linalg_hessenberg(
+    mat1: *mut InputMatrix,
+    out_h: *mut InputMatrix,
+    out_q: *mut InputMatrix,
+) {
+    let mat1 = mat1.as_mut().unwrap();
+    let out_h = out_h.as_mut().unwrap();
+    let out_q = out_q.as_mut().unwrap();
+
+    if mat1.ndims != 2 {
+        artiq_raise!(
+            "ValueError",
+            "expected 2D Vector Input, but received {1}D input)",
+            0,
+            mat1.ndims as i64,
+            0
+        );
+    }
+
+    let dim1 = (*mat1).get_dims();
+
+    if dim1[0] != dim1[1] {
+        artiq_raise!(
+            "ValueError",
+            "last 2 dimensions of the array must be square: {1} != {2}",
+            0,
+            dim1[0] as i64,
+            dim1[1] as i64
+        );
+    }
+
+    let out_h_dim = (*out_h).get_dims();
+    let out_q_dim = (*out_q).get_dims();
+
+    let data_slice1 = slice::from_raw_parts_mut(mat1.data, dim1[0] * dim1[1]);
+    let out_h_slice = slice::from_raw_parts_mut(out_h.data, out_h_dim[0] * out_h_dim[1]);
+    let out_q_slice = slice::from_raw_parts_mut(out_q.data, out_q_dim[0] * out_q_dim[1]);
+
+    let matrix = DMatrix::from_row_slice(dim1[0], dim1[1], data_slice1);
+    let (q, h) = matrix.hessenberg().unpack();
+
+    out_h_slice.copy_from_slice(h.transpose().as_slice());
+    out_q_slice.copy_from_slice(q.transpose().as_slice());
+}

src/libksupport/src/kernel/mod.rs

@@ -13,6 +13,7 @@ mod dma;
 mod rpc;
 pub use dma::DmaRecorder;
 mod cache;
+mod linalg;
 #[cfg(has_drtio)]
 mod subkernel;
 
@@ -23,6 +24,7 @@ pub enum SubkernelStatus {
     Timeout,
     IncorrectState,
     CommLost,
+    Exception(Vec<u8>),
     OtherError,
 }
 
@@ -77,7 +79,9 @@ pub enum Message {
     #[cfg(has_drtio)]
     SubkernelLoadRunRequest {
         id: u32,
+        destination: u8,
         run: bool,
+        timestamp: u64,
     },
     #[cfg(has_drtio)]
     SubkernelLoadRunReply {
@@ -86,30 +90,30 @@ pub enum Message {
     #[cfg(has_drtio)]
     SubkernelAwaitFinishRequest {
         id: u32,
-        timeout: u64,
+        timeout: i64,
     },
     #[cfg(has_drtio)]
-    SubkernelAwaitFinishReply {
+    SubkernelAwaitFinishReply,
-        status: SubkernelStatus,
-    },
     #[cfg(has_drtio)]
     SubkernelMsgSend {
         id: u32,
+        destination: Option<u8>,
         data: Vec<u8>,
     },
     #[cfg(has_drtio)]
     SubkernelMsgSent,
     #[cfg(has_drtio)]
     SubkernelMsgRecvRequest {
-        id: u32,
+        id: i32,
-        timeout: u64,
+        timeout: i64,
         tags: Vec<u8>,
     },
     #[cfg(has_drtio)]
     SubkernelMsgRecvReply {
-        status: SubkernelStatus,
         count: u8,
     },
+    #[cfg(has_drtio)]
+    SubkernelError(SubkernelStatus),
 }
 
 static CHANNEL_0TO1: Mutex<Option<sync_channel::Sender<'static, Message>>> = Mutex::new(None);

src/libksupport/src/kernel/subkernel.rs

@@ -3,14 +3,19 @@ use alloc::vec::Vec;
 use cslice::CSlice;
 
 use super::{Message, SubkernelStatus, KERNEL_CHANNEL_0TO1, KERNEL_CHANNEL_1TO0};
-use crate::{artiq_raise, rpc::send_args};
+use crate::{artiq_raise, eh_artiq, rpc::send_args, rtio::now_mu};
 
-pub extern "C" fn load_run(id: u32, run: bool) {
+pub extern "C" fn load_run(id: u32, destination: u8, run: bool) {
     unsafe {
         KERNEL_CHANNEL_1TO0
             .as_mut()
             .unwrap()
-            .send(Message::SubkernelLoadRunRequest { id: id, run: run });
+            .send(Message::SubkernelLoadRunRequest {
+                id: id,
+                destination: destination,
+                run: run,
+                timestamp: now_mu() as u64,
+            });
     }
     match unsafe { KERNEL_CHANNEL_0TO1.as_mut().unwrap() }.recv() {
         Message::SubkernelLoadRunReply { succeeded: true } => (),
@@ -21,7 +26,7 @@ pub extern "C" fn load_run(id: u32, run: bool) {
     }
 }
 
-pub extern "C" fn await_finish(id: u32, timeout: u64) {
+pub extern "C" fn await_finish(id: u32, timeout: i64) {
     unsafe {
         KERNEL_CHANNEL_1TO0
             .as_mut()
@@ -32,26 +37,30 @@ pub extern "C" fn await_finish(id: u32, timeout: u64) {
             });
     }
     match unsafe { KERNEL_CHANNEL_0TO1.as_mut().unwrap() }.recv() {
-        Message::SubkernelAwaitFinishReply {
+        Message::SubkernelAwaitFinishReply => (),
-            status: SubkernelStatus::NoError,
+        Message::SubkernelError(SubkernelStatus::IncorrectState) => {
-        } => (),
+            artiq_raise!("SubkernelError", "Subkernel not running")
-        Message::SubkernelAwaitFinishReply {
+        }
-            status: SubkernelStatus::IncorrectState,
+        Message::SubkernelError(SubkernelStatus::Timeout) => artiq_raise!("SubkernelError", "Subkernel timed out"),
-        } => artiq_raise!("SubkernelError", "Subkernel not running"),
+        Message::SubkernelError(SubkernelStatus::CommLost) => {
-        Message::SubkernelAwaitFinishReply {
+            artiq_raise!("SubkernelError", "Lost communication with satellite")
-            status: SubkernelStatus::Timeout,
+        }
-        } => artiq_raise!("SubkernelError", "Subkernel timed out"),
+        Message::SubkernelError(SubkernelStatus::OtherError) => {
-        Message::SubkernelAwaitFinishReply {
+            artiq_raise!("SubkernelError", "An error occurred during subkernel operation")
-            status: SubkernelStatus::CommLost,
+        }
-        } => artiq_raise!("SubkernelError", "Lost communication with satellite"),
+        Message::SubkernelError(SubkernelStatus::Exception(raw_exception)) => eh_artiq::raise_raw(&raw_exception),
-        Message::SubkernelAwaitFinishReply {
-            status: SubkernelStatus::OtherError,
-        } => artiq_raise!("SubkernelError", "An error occurred during subkernel operation"),
         _ => panic!("expected SubkernelAwaitFinishReply after SubkernelAwaitFinishRequest"),
     }
 }
 
-pub extern "C" fn send_message(id: u32, count: u8, tag: &CSlice<u8>, data: *const *const ()) {
+pub extern "C" fn send_message(
+    id: u32,
+    is_return: bool,
+    destination: u8,
+    count: u8,
+    tag: &CSlice<u8>,
+    data: *const *const (),
+) {
     let mut buffer = Vec::<u8>::new();
     send_args(&mut buffer, 0, tag.as_ref(), data, false).expect("RPC encoding failed");
     // overwrite service tag, include how many tags are in the message
@@ -59,6 +68,7 @@ pub extern "C" fn send_message(id: u32, count: u8, tag: &CSlice<u8>, data: *cons
     unsafe {
         KERNEL_CHANNEL_1TO0.as_mut().unwrap().send(Message::SubkernelMsgSend {
             id: id,
+            destination: if is_return { None } else { Some(destination) },
             data: buffer[3..].to_vec(),
         });
     }
@@ -68,7 +78,7 @@ pub extern "C" fn send_message(id: u32, count: u8, tag: &CSlice<u8>, data: *cons
     }
 }
 
-pub extern "C" fn await_message(id: u32, timeout: u64, tags: &CSlice<u8>, min: u8, max: u8) {
+pub extern "C" fn await_message(id: i32, timeout: i64, tags: &CSlice<u8>, min: u8, max: u8) {
     unsafe {
         KERNEL_CHANNEL_1TO0
             .as_mut()
@@ -80,30 +90,22 @@ pub extern "C" fn await_message(id: u32, timeout: u64, tags: &CSlice<u8>, min: u
             });
     }
     match unsafe { KERNEL_CHANNEL_0TO1.as_mut().unwrap() }.recv() {
-        Message::SubkernelMsgRecvReply {
+        Message::SubkernelMsgRecvReply { count } => {
-            status: SubkernelStatus::NoError,
-            count,
-        } => {
             if min > count || count > max {
                 artiq_raise!("SubkernelError", "Received more or less arguments than required")
             }
         }
-        Message::SubkernelMsgRecvReply {
+        Message::SubkernelError(SubkernelStatus::IncorrectState) => {
-            status: SubkernelStatus::IncorrectState,
+            artiq_raise!("SubkernelError", "Subkernel not running")
-            ..
+        }
-        } => artiq_raise!("SubkernelError", "Subkernel not running"),
+        Message::SubkernelError(SubkernelStatus::Timeout) => artiq_raise!("SubkernelError", "Subkernel timed out"),
-        Message::SubkernelMsgRecvReply {
+        Message::SubkernelError(SubkernelStatus::CommLost) => {
-            status: SubkernelStatus::Timeout,
+            artiq_raise!("SubkernelError", "Lost communication with satellite")
-            ..
+        }
-        } => artiq_raise!("SubkernelError", "Subkernel timed out"),
+        Message::SubkernelError(SubkernelStatus::OtherError) => {
-        Message::SubkernelMsgRecvReply {
+            artiq_raise!("SubkernelError", "An error occurred during subkernel operation")
-            status: SubkernelStatus::CommLost,
+        }
-            ..
+        Message::SubkernelError(SubkernelStatus::Exception(raw_exception)) => eh_artiq::raise_raw(&raw_exception),
-        } => artiq_raise!("SubkernelError", "Lost communication with satellite"),
-        Message::SubkernelMsgRecvReply {
-            status: SubkernelStatus::OtherError,
-            ..
-        } => artiq_raise!("SubkernelError", "An error occurred during subkernel operation"),
         _ => panic!("expected SubkernelMsgRecvReply after SubkernelMsgRecvRequest"),
     }
     // RpcRecvRequest should be called after this to receive message data

src/libksupport/src/lib.rs

@@ -1,9 +1,10 @@
 #![no_std]
 #![feature(c_variadic)]
-#![feature(const_btree_new)]
+#![feature(lang_items)]
-#![feature(const_in_array_repeat_expressions)]
+#![feature(const_btree_len)]
+#![feature(generic_const_exprs)]
 #![feature(naked_functions)]
-#![feature(asm)]
+#![allow(unexpected_cfgs)]
 
 #[macro_use]
 extern crate alloc;
@@ -21,6 +22,7 @@ pub use pl::csr::rtio_core;
 use void::Void;
 
 pub mod eh_artiq;
+pub mod eh_rust;
 pub mod i2c;
 pub mod irq;
 pub mod kernel;

src/libksupport/src/rpc.rs

@@ -1,9 +1,8 @@
 use core::str;
 
 use byteorder::{ByteOrder, NativeEndian};
-use core_io::{Error, Read, Write};
 use cslice::{CMutSlice, CSlice};
-use io::{ProtoRead, ProtoWrite};
+use io::{Error, ProtoRead, ProtoWrite, Read, Write};
 use log::trace;
 
 use self::tag::{split_tag, Tag, TagIterator};
@@ -37,16 +36,17 @@ pub unsafe fn align_ptr_mut<T>(ptr: *mut ()) -> *mut T {
 
 // versions for reader rather than TcpStream
 // they will be made into sync for satellite subkernels later
-unsafe fn recv_elements<F, R>(
+unsafe fn recv_elements<F, R, E>(
     reader: &mut R,
     elt_tag: Tag,
     length: usize,
     storage: *mut (),
     alloc: &mut F,
-) -> Result<(), Error>
+) -> Result<(), E>
 where
-    F: FnMut(usize) -> *mut (),
+    F: FnMut(usize) -> Result<*mut (), E>,
     R: Read + ?Sized,
+    E: From<Error<R::ReadError>>,
 {
     match elt_tag {
         Tag::Bool => {
@@ -57,7 +57,6 @@ where
             let ptr = storage as *mut u32;
             let dest = core::slice::from_raw_parts_mut(ptr as *mut u8, length * 4);
             reader.read_exact(dest)?;
-            drop(dest);
             let dest = core::slice::from_raw_parts_mut(ptr, length);
             NativeEndian::from_slice_u32(dest);
         }
@@ -65,7 +64,6 @@ where
             let ptr = storage as *mut u64;
             let dest = core::slice::from_raw_parts_mut(ptr as *mut u8, length * 8);
             reader.read_exact(dest)?;
-            drop(dest);
             let dest = core::slice::from_raw_parts_mut(ptr, length);
             NativeEndian::from_slice_u64(dest);
         }
@@ -79,10 +77,11 @@ where
     Ok(())
 }
 
-unsafe fn recv_value<F, R>(reader: &mut R, tag: Tag, data: &mut *mut (), alloc: &mut F) -> Result<(), Error>
+unsafe fn recv_value<F, R, E>(reader: &mut R, tag: Tag, data: &mut *mut (), alloc: &mut F) -> Result<(), E>
 where
-    F: FnMut(usize) -> *mut (),
+    F: FnMut(usize) -> Result<*mut (), E>,
     R: Read + ?Sized,
+    E: From<Error<R::ReadError>>,
 {
     macro_rules! consume_value {
         ($ty:ty, | $ptr:ident | $map:expr) => {{
@@ -109,7 +108,7 @@ where
         Tag::String | Tag::Bytes | Tag::ByteArray => {
             consume_value!(CMutSlice<u8>, |ptr| {
                 let length = reader.read_u32()? as usize;
-                *ptr = CMutSlice::new(alloc(length) as *mut u8, length);
+                *ptr = CMutSlice::new(alloc(length)? as *mut u8, length);
                 reader.read_exact((*ptr).as_mut())?;
                 Ok(())
             })
@@ -139,7 +138,7 @@ where
                 let storage_offset = round_up(list_size, tag.alignment());
                 let storage_size = tag.size() * length;
 
-                let allocation = alloc(storage_offset + storage_size) as *mut u8;
+                let allocation = alloc(storage_offset + storage_size)? as *mut u8;
                 *ptr_to_list = allocation as *mut List;
                 let storage = allocation.offset(storage_offset as isize) as *mut ();
 
@@ -158,7 +157,7 @@ where
                 }
 
                 let elt_tag = it.clone().next().expect("truncated tag");
-                *buffer = alloc(elt_tag.size() * total_len);
+                *buffer = alloc(elt_tag.size() * total_len)?;
                 recv_elements(reader, elt_tag, total_len, *buffer, alloc)
             })
         }
@@ -175,15 +174,16 @@ where
     }
 }
 
-pub fn recv_return<'a, F, R>(
+pub fn recv_return<'a, F, R, E>(
     reader: &mut R,
     tag_bytes: &'a [u8],
     data: *mut (),
     alloc: &mut F,
-) -> Result<&'a [u8], Error>
+) -> Result<&'a [u8], E>
 where
-    F: FnMut(usize) -> *mut (),
+    F: FnMut(usize) -> Result<*mut (), E>,
     R: Read + ?Sized,
+    E: From<Error<R::ReadError>>,
 {
     let mut it = TagIterator::new(tag_bytes);
     trace!("recv ...->{}", it);
@@ -201,7 +201,7 @@ unsafe fn send_elements<W>(
     length: usize,
     data: *const (),
     write_tags: bool,
-) -> Result<(), Error>
+) -> Result<(), Error<W::WriteError>>
 where
     W: Write + ?Sized,
 {
@@ -233,8 +233,15 @@ where
     Ok(())
 }
 
-unsafe fn send_value<W>(writer: &mut W, tag: Tag, data: &mut *const (), write_tags: bool) -> Result<(), Error>
+unsafe fn send_value<W>(
-where W: Write + ?Sized {
+    writer: &mut W,
+    tag: Tag,
+    data: &mut *const (),
+    write_tags: bool,
+) -> Result<(), Error<W::WriteError>>
+where
+    W: Write + ?Sized,
+{
     macro_rules! consume_value {
         ($ty:ty, | $ptr:ident | $map:expr) => {{
             let $ptr = align_ptr::<$ty>(*data);
@@ -337,7 +344,7 @@ pub fn send_args<W>(
     tag_bytes: &[u8],
     data: *const *const (),
     write_tags: bool,
-) -> Result<(), Error>
+) -> Result<(), Error<W::WriteError>>
 where
     W: Write + ?Sized,
 {

src/libunwind/lib.rs

@@ -1,8 +1,5 @@
 #![no_std]
 #![feature(link_cfg)]
-#![feature(nll)]
-#![feature(unwind_attributes)]
-#![feature(static_nobundle)]
 #![cfg_attr(not(target_env = "msvc"), feature(libc))]
 
 cfg_if::cfg_if! {
@@ -17,17 +14,3 @@ cfg_if::cfg_if! {
         pub use backtrace::backtrace;
     }
 }
-
-#[cfg(target_env = "musl")]
-#[link(name = "unwind", kind = "static", cfg(target_feature = "crt-static"))]
-#[link(name = "gcc_s", cfg(not(target_feature = "crt-static")))]
-extern "C" {}
-
-#[cfg(target_os = "redox")]
-#[link(name = "gcc_eh", kind = "static-nobundle", cfg(target_feature = "crt-static"))]
-#[link(name = "gcc_s", cfg(not(target_feature = "crt-static")))]
-extern "C" {}
-
-#[cfg(all(target_vendor = "fortanix", target_env = "sgx"))]
-#[link(name = "unwind", kind = "static-nobundle")]
-extern "C" {}

src/libunwind/libunwind.rs

@@ -77,8 +77,7 @@ pub type _Unwind_Exception_Cleanup_Fn =
     all(feature = "llvm-libunwind", any(target_os = "fuchsia", target_os = "linux")),
     link(name = "unwind", kind = "static")
 )]
-extern "C" {
+extern "C-unwind" {
-    #[unwind(allowed)]
     pub fn _Unwind_Resume(exception: *mut _Unwind_Exception) -> !;
     pub fn _Unwind_DeleteException(exception: *mut _Unwind_Exception);
     pub fn _Unwind_GetLanguageSpecificData(ctx: *mut _Unwind_Context) -> *mut c_void;
@@ -226,8 +225,7 @@ if #[cfg(not(all(target_os = "ios", target_arch = "arm")))] {
     #[cfg_attr(all(feature = "llvm-libunwind",
                    any(target_os = "fuchsia", target_os = "linux")),
                link(name = "unwind", kind = "static"))]
-    extern "C" {
+    extern "C-unwind" {
-        #[unwind(allowed)]
         pub fn _Unwind_RaiseException(exception: *mut _Unwind_Exception) -> _Unwind_Reason_Code;
         pub fn _Unwind_Backtrace(trace: _Unwind_Trace_Fn,
                                  trace_argument: *mut c_void)
@@ -238,8 +236,7 @@ if #[cfg(not(all(target_os = "ios", target_arch = "arm")))] {
     #[cfg_attr(all(feature = "llvm-libunwind",
                    any(target_os = "fuchsia", target_os = "linux")),
                link(name = "unwind", kind = "static"))]
-    extern "C" {
+    extern "C-unwind" {
-        #[unwind(allowed)]
         pub fn _Unwind_SjLj_RaiseException(e: *mut _Unwind_Exception) -> _Unwind_Reason_Code;
     }
 

src/runtime/Cargo.toml.tpl

@@ -8,6 +8,7 @@ edition = "2018"
 [features]
 target_zc706 = ["libboard_zynq/target_zc706", "libsupport_zynq/target_zc706", "libconfig/target_zc706", "libboard_artiq/target_zc706"]
 target_kasli_soc = ["libboard_zynq/target_kasli_soc", "libsupport_zynq/target_kasli_soc", "libconfig/target_kasli_soc", "libboard_artiq/target_kasli_soc"]
+target_ebaz4205 = ["libboard_zynq/target_ebaz4205", "libsupport_zynq/target_ebaz4205", "libconfig/target_ebaz4205", "libboard_artiq/target_ebaz4205"]
 default = ["target_zc706"]
 
 [build-dependencies]
@@ -19,7 +20,6 @@ num-derive = "0.3"
 cslice = "0.3"
 log = "0.4"
 embedded-hal = "0.2"
-core_io = { version = "0.1", features = ["collections"] }
 byteorder = { version = "1.3", default-features = false }
 void = { version = "1", default-features = false }
 futures = { version = "0.3", default-features = false, features = ["async-await"] }
@@ -40,4 +40,8 @@ unwind = { path = "../libunwind" }
 libc = { path = "../libc" }
 io = { path = "../libio", features = ["alloc"] }
 ksupport = { path = "../libksupport" }
-libboard_artiq = { path = "../libboard_artiq" }
+libboard_artiq = { path = "../libboard_artiq" }
+
+[dependencies.tar-no-std]
+git = "https://git.m-labs.hk/M-Labs/tar-no-std"
+rev = "2ab6dc5"

src/runtime/build.rs

@@ -1,3 +1,4 @@
+#![allow(unexpected_cfgs)]
 extern crate build_zynq;
 
 fn main() {

src/runtime/src/analyzer.rs

@@ -60,7 +60,7 @@ pub mod remote_analyzer {
         routing_table: &drtio_routing::RoutingTable,
         up_destinations: &Rc<RefCell<[bool; drtio_routing::DEST_COUNT]>>,
         timer: GlobalTimer,
-    ) -> Result<RemoteBuffer, &'static str> {
+    ) -> Result<RemoteBuffer, drtio::Error> {
         // gets data from satellites and returns consolidated data
         let mut remote_data: Vec<u8> = Vec::new();
         let mut remote_error = false;

src/runtime/src/comms.rs

@@ -1,19 +1,24 @@
 use alloc::{collections::BTreeMap, rc::Rc, string::String, vec, vec::Vec};
 use core::{cell::RefCell, fmt, slice, str};
 
-use core_io::Error as IoError;
 use cslice::CSlice;
+use dyld::elf;
 use futures::{future::FutureExt, select_biased};
 #[cfg(has_drtio)]
 use io::Cursor;
+use io::Error as IoError;
 #[cfg(has_drtio)]
 use ksupport::rpc;
 use ksupport::{kernel, resolve_channel_name};
+#[cfg(has_drtio)]
+use libasync::delay;
 use libasync::{smoltcp::{Sockets, TcpStream},
                task};
 use libboard_artiq::drtio_routing;
 #[cfg(feature = "target_kasli_soc")]
 use libboard_zynq::error_led::ErrorLED;
+#[cfg(has_drtio)]
+use libboard_zynq::time::Milliseconds;
 use libboard_zynq::{self as zynq,
                     smoltcp::{self,
                               iface::{EthernetInterfaceBuilder, NeighborCache},
@@ -27,6 +32,8 @@ use libcortex_a9::{mutex::Mutex,
 use log::{error, info, warn};
 use num_derive::{FromPrimitive, ToPrimitive};
 use num_traits::{FromPrimitive, ToPrimitive};
+#[cfg(has_drtio)]
+use tar_no_std::TarArchiveRef;
 
 #[cfg(has_drtio)]
 use crate::pl;
@@ -43,6 +50,8 @@ pub enum Error {
     BufferExhausted,
     #[cfg(has_drtio)]
     SubkernelError(subkernel::Error),
+    #[cfg(has_drtio)]
+    DestinationDown,
 }
 
 pub type Result<T> = core::result::Result<T, Error>;
@@ -57,6 +66,8 @@ impl fmt::Display for Error {
             Error::BufferExhausted => write!(f, "buffer exhausted"),
             #[cfg(has_drtio)]
             Error::SubkernelError(error) => write!(f, "subkernel error: {:?}", error),
+            #[cfg(has_drtio)]
+            Error::DestinationDown => write!(f, "subkernel destination down"),
         }
     }
 }
@@ -67,8 +78,8 @@ impl From<smoltcp::Error> for Error {
     }
 }
 
-impl From<IoError> for Error {
+impl<T> From<IoError<T>> for Error {
-    fn from(_error: IoError) -> Self {
+    fn from(_error: IoError<T>) -> Self {
         Error::IoError
     }
 }
@@ -390,8 +401,13 @@ async fn handle_run_kernel(
                 control.borrow_mut().tx.async_send(reply).await;
             }
             #[cfg(has_drtio)]
-            kernel::Message::SubkernelLoadRunRequest { id, run } => {
+            kernel::Message::SubkernelLoadRunRequest {
-                let succeeded = match subkernel::load(aux_mutex, routing_table, timer, id, run).await {
+                id,
+                destination: _,
+                run,
+                timestamp,
+            } => {
+                let succeeded = match subkernel::load(aux_mutex, routing_table, timer, id, run, timestamp).await {
                     Ok(()) => true,
                     Err(e) => {
                         error!("Error loading subkernel: {:?}", e);
@@ -407,37 +423,28 @@ async fn handle_run_kernel(
             #[cfg(has_drtio)]
             kernel::Message::SubkernelAwaitFinishRequest { id, timeout } => {
                 let res = subkernel::await_finish(aux_mutex, routing_table, timer, id, timeout).await;
-                let status = match res {
+                let response = match res {
-                    Ok(ref res) => {
+                    Ok(res) => {
                         if res.status == subkernel::FinishStatus::CommLost {
-                            kernel::SubkernelStatus::CommLost
+                            kernel::Message::SubkernelError(kernel::SubkernelStatus::CommLost)
-                        } else if let Some(exception) = &res.exception {
+                        } else if let Some(exception) = res.exception {
-                            error!("Exception in subkernel");
+                            kernel::Message::SubkernelError(kernel::SubkernelStatus::Exception(exception))
-                            match stream {
-                                None => (),
-                                Some(stream) => {
-                                    write_chunk(stream, exception).await?;
-                                }
-                            }
-                            // will not be called after exception is served
-                            kernel::SubkernelStatus::OtherError
                         } else {
-                            kernel::SubkernelStatus::NoError
+                            kernel::Message::SubkernelAwaitFinishReply
                         }
                     }
-                    Err(SubkernelError::Timeout) => kernel::SubkernelStatus::Timeout,
+                    Err(SubkernelError::Timeout) => kernel::Message::SubkernelError(kernel::SubkernelStatus::Timeout),
-                    Err(SubkernelError::IncorrectState) => kernel::SubkernelStatus::IncorrectState,
+                    Err(SubkernelError::IncorrectState) => {
-                    Err(_) => kernel::SubkernelStatus::OtherError,
+                        kernel::Message::SubkernelError(kernel::SubkernelStatus::IncorrectState)
+                    }
+                    Err(_) => kernel::Message::SubkernelError(kernel::SubkernelStatus::OtherError),
                 };
-                control
+                control.borrow_mut().tx.async_send(response).await;
-                    .borrow_mut()
-                    .tx
-                    .async_send(kernel::Message::SubkernelAwaitFinishReply { status: status })
-                    .await;
             }
             #[cfg(has_drtio)]
-            kernel::Message::SubkernelMsgSend { id, data } => {
+            kernel::Message::SubkernelMsgSend { id, destination, data } => {
-                let res = subkernel::message_send(aux_mutex, routing_table, timer, id, data).await;
+                let res =
+                    subkernel::message_send(aux_mutex, routing_table, timer, id, destination.unwrap(), data).await;
                 match res {
                     Ok(_) => (),
                     Err(e) => {
@@ -452,22 +459,24 @@ async fn handle_run_kernel(
             }
             #[cfg(has_drtio)]
             kernel::Message::SubkernelMsgRecvRequest { id, timeout, tags } => {
-                let message_received = subkernel::message_await(id, timeout, timer).await;
+                let message_received = subkernel::message_await(id as u32, timeout, timer).await;
-                let (status, count) = match message_received {
+                let response = match message_received {
-                    Ok(ref message) => (kernel::SubkernelStatus::NoError, message.count),
+                    Ok(ref message) => kernel::Message::SubkernelMsgRecvReply { count: message.count },
-                    Err(SubkernelError::Timeout) => (kernel::SubkernelStatus::Timeout, 0),
+                    Err(SubkernelError::Timeout) => kernel::Message::SubkernelError(kernel::SubkernelStatus::Timeout),
-                    Err(SubkernelError::IncorrectState) => (kernel::SubkernelStatus::IncorrectState, 0),
+                    Err(SubkernelError::IncorrectState) => {
-                    Err(SubkernelError::CommLost) => (kernel::SubkernelStatus::CommLost, 0),
+                        kernel::Message::SubkernelError(kernel::SubkernelStatus::IncorrectState)
-                    Err(_) => (kernel::SubkernelStatus::OtherError, 0),
+                    }
+                    Err(SubkernelError::CommLost) => kernel::Message::SubkernelError(kernel::SubkernelStatus::CommLost),
+                    Err(SubkernelError::SubkernelException) => {
+                        // just retrieve the exception
+                        let status = subkernel::await_finish(aux_mutex, routing_table, timer, id as u32, timeout)
+                            .await
+                            .unwrap();
+                        kernel::Message::SubkernelError(kernel::SubkernelStatus::Exception(status.exception.unwrap()))
+                    }
+                    Err(_) => kernel::Message::SubkernelError(kernel::SubkernelStatus::OtherError),
                 };
-                control
+                control.borrow_mut().tx.async_send(response).await;
-                    .borrow_mut()
-                    .tx
-                    .async_send(kernel::Message::SubkernelMsgRecvReply {
-                        status: status,
-                        count: count,
-                    })
-                    .await;
                 if let Ok(message) = message_received {
                     // receive code almost identical to RPC recv, except we are not reading from a stream
                     let mut reader = Cursor::new(message.data);
@@ -479,14 +488,14 @@ async fn handle_run_kernel(
                             kernel::Message::RpcRecvRequest(slot) => slot,
                             other => panic!("expected root value slot from core1, not {:?}", other),
                         };
-                        let remaining_tags = rpc::recv_return(&mut reader, &current_tags, slot, &mut |size| {
+                        let remaining_tags = rpc::recv_return(&mut reader, &current_tags, slot, &mut |size| -> Result<*mut ()> {
                             if size == 0 {
-                                0 as *mut ()
+                                Ok(0 as *mut ())
                             } else {
                                 let mut control = control.borrow_mut();
                                 control.tx.send(kernel::Message::RpcRecvReply(Ok(size)));
                                 match control.rx.recv() {
-                                    kernel::Message::RpcRecvRequest(slot) => slot,
+                                    kernel::Message::RpcRecvRequest(slot) => Ok(slot),
                                     other => {
                                         panic!("expected nested value slot from kernel CPU, not {:?}", other)
                                     }
@@ -499,7 +508,7 @@ async fn handle_run_kernel(
                             .async_send(kernel::Message::RpcRecvReply(Ok(0)))
                             .await;
                         i += 1;
-                        if i < count {
+                        if i < message.count {
                             current_tags = remaining_tags;
                         } else {
                             break;
@@ -524,6 +533,56 @@ async fn handle_run_kernel(
     Ok(())
 }
 
+async fn handle_flash_kernel(
+    buffer: &Vec<u8>,
+    control: &Rc<RefCell<kernel::Control>>,
+    _up_destinations: &Rc<RefCell<[bool; drtio_routing::DEST_COUNT]>>,
+    _aux_mutex: &Rc<Mutex<bool>>,
+    _routing_table: &drtio_routing::RoutingTable,
+    _timer: GlobalTimer,
+) -> Result<()> {
+    if buffer[0] == elf::ELFMAG0 && buffer[1] == elf::ELFMAG1 && buffer[2] == elf::ELFMAG2 && buffer[3] == elf::ELFMAG3
+    {
+        // assume ELF file, proceed as before
+        load_kernel(buffer, control, None).await
+    } else {
+        #[cfg(has_drtio)]
+        {
+            let archive = TarArchiveRef::new(buffer.as_ref());
+            let entries = archive.entries();
+            let mut main_lib: Vec<u8> = Vec::new();
+            for entry in entries {
+                if entry.filename().as_str() == "main.elf" {
+                    main_lib = entry.data().to_vec();
+                } else {
+                    // subkernel filename must be in format:
+                    // "<subkernel id> <destination>.elf"
+                    let filename = entry.filename();
+                    let mut iter = filename.as_str().split_whitespace();
+                    let sid: u32 = iter.next().unwrap().parse().unwrap();
+                    let dest: u8 = iter.next().unwrap().strip_suffix(".elf").unwrap().parse().unwrap();
+                    let up = _up_destinations.borrow()[dest as usize];
+                    if up {
+                        let subkernel_lib = entry.data().to_vec();
+                        subkernel::add_subkernel(sid, dest, subkernel_lib).await;
+                        match subkernel::upload(_aux_mutex, _routing_table, _timer, sid).await {
+                            Ok(_) => (),
+                            Err(_) => return Err(Error::UnexpectedPattern),
+                        }
+                    } else {
+                        return Err(Error::DestinationDown);
+                    }
+                }
+            }
+            load_kernel(&main_lib, control, None).await
+        }
+        #[cfg(not(has_drtio))]
+        {
+            panic!("multi-kernel libraries are not supported in standalone systems");
+        }
+    }
+}
+
 async fn load_kernel(
     buffer: &Vec<u8>,
     control: &Rc<RefCell<kernel::Control>>,
@@ -679,7 +738,6 @@ pub fn main(timer: GlobalTimer, cfg: Config) {
 
     Sockets::init(32);
 
-    // before, mutex was on io, but now that io isn't used...?
     let aux_mutex: Rc<Mutex<bool>> = Rc::new(Mutex::new(false));
     #[cfg(has_drtio)]
     let drtio_routing_table = Rc::new(RefCell::new(drtio_routing::config_routing_table(
@@ -692,19 +750,25 @@ pub fn main(timer: GlobalTimer, cfg: Config) {
     #[cfg(has_drtio_routing)]
     drtio_routing::interconnect_disable_all();
 
-    rtio_mgt::startup(&aux_mutex, &drtio_routing_table, &up_destinations, timer);
+    rtio_mgt::startup(&aux_mutex, &drtio_routing_table, &up_destinations, &cfg, timer);
     ksupport::setup_device_map(&cfg);
 
     analyzer::start(&aux_mutex, &drtio_routing_table, &up_destinations, timer);
     moninj::start(timer, &aux_mutex, &drtio_routing_table);
 
     let control: Rc<RefCell<kernel::Control>> = Rc::new(RefCell::new(kernel::Control::start()));
-    let idle_kernel = Rc::new(cfg.read("idle_kernel").ok());
     if let Ok(buffer) = cfg.read("startup_kernel") {
         info!("Loading startup kernel...");
-        if let Ok(()) = task::block_on(load_kernel(&buffer, &control, None)) {
+        let routing_table = drtio_routing_table.borrow();
+        if let Ok(()) = task::block_on(handle_flash_kernel(
+            &buffer,
+            &control,
+            &up_destinations,
+            &aux_mutex,
+            &routing_table,
+            timer,
+        )) {
             info!("Starting startup kernel...");
-            let routing_table = drtio_routing_table.borrow();
             let _ = task::block_on(handle_run_kernel(
                 None,
                 &control,
@@ -719,13 +783,26 @@ pub fn main(timer: GlobalTimer, cfg: Config) {
         }
     }
 
-    mgmt::start(cfg);
+    let cfg = Rc::new(cfg);
+    let restart_idle = Rc::new(Semaphore::new(1, 1));
+    mgmt::start(cfg.clone(), restart_idle.clone());
 
     task::spawn(async move {
         let connection = Rc::new(Semaphore::new(1, 1));
         let terminate = Rc::new(Semaphore::new(0, 1));
+        let can_restart_idle = Rc::new(Semaphore::new(1, 1));
+        let restart_idle = restart_idle.clone();
         loop {
-            let mut stream = TcpStream::accept(1381, 0x10_000, 0x10_000).await.unwrap();
+            let control = control.clone();
+            let mut maybe_stream = select_biased! {
+                s = (async {
+                        TcpStream::accept(1381, 0x10_000, 0x10_000).await.unwrap()
+                    }).fuse() => Some(s),
+                _ = (async {
+                        restart_idle.async_wait().await;
+                        can_restart_idle.async_wait().await;
+                    }).fuse() => None
+            };
 
             if connection.try_wait().is_none() {
                 // there is an existing connection
@@ -733,38 +810,58 @@ pub fn main(timer: GlobalTimer, cfg: Config) {
                 connection.async_wait().await;
             }
 
+            let maybe_idle_kernel = cfg.read("idle_kernel").ok();
+            if maybe_idle_kernel.is_none() && maybe_stream.is_none() {
+                control.borrow_mut().restart(); // terminate idle kernel if running
+            }
+
             let control = control.clone();
-            let idle_kernel = idle_kernel.clone();
             let connection = connection.clone();
             let terminate = terminate.clone();
+            let can_restart_idle = can_restart_idle.clone();
             let up_destinations = up_destinations.clone();
             let aux_mutex = aux_mutex.clone();
             let routing_table = drtio_routing_table.clone();
 
             // we make sure the value of terminate is 0 before we start
             let _ = terminate.try_wait();
+            let _ = can_restart_idle.try_wait();
             task::spawn(async move {
                 let routing_table = routing_table.borrow();
                 select_biased! {
                     _ = (async {
-                        let _ = handle_connection(&mut stream, control.clone(), &up_destinations, &aux_mutex, &routing_table, timer)
+                        if let Some(stream) = &mut maybe_stream {
-                            .await
+                            let _ = handle_connection(stream, control.clone(), &up_destinations, &aux_mutex, &routing_table, timer)
-                            .map_err(|e| warn!("connection terminated: {}", e));
+                                .await
-                        if let Some(buffer) = &*idle_kernel {
+                                .map_err(|e| warn!("connection terminated: {}", e));
-                            info!("Loading idle kernel");
+                        }
-                            let _ = load_kernel(&buffer, &control, None)
+                        can_restart_idle.signal();
-                                .await.map_err(|_| warn!("error loading idle kernel"));
+                        match maybe_idle_kernel {
-                            info!("Running idle kernel");
+                            Some(buffer) => {
-                            let _ = handle_run_kernel(None, &control, &up_destinations, &aux_mutex, &routing_table, timer)
+                                loop {
-                                .await.map_err(|_| warn!("error running idle kernel"));
+                                    info!("loading idle kernel");
-                            info!("Idle kernel terminated");
+                                    match handle_flash_kernel(&buffer, &control, &up_destinations, &aux_mutex, &routing_table, timer).await {
+                                        Ok(_) => {
+                                            info!("running idle kernel");
+                                            match handle_run_kernel(None, &control, &up_destinations, &aux_mutex, &routing_table, timer).await {
+                                                Ok(_) => info!("idle kernel finished"),
+                                                Err(_) => warn!("idle kernel running error")
+                                            }
+                                        },
+                                        Err(_) => warn!("idle kernel loading error")
+                                    }
+                                }
+                            },
+                            None => info!("no idle kernel found")
                         }
                     }).fuse() => (),
                     _ = terminate.async_wait().fuse() => ()
                 }
                 connection.signal();
-                let _ = stream.flush().await;
+                if let Some(stream) = maybe_stream {
-                let _ = stream.abort().await;
+                    let _ = stream.flush().await;
+                    let _ = stream.abort().await;
+                }
             });
         }
     });
@@ -813,7 +910,8 @@ pub fn soft_panic_main(timer: GlobalTimer, cfg: Config) -> ! {
 
     Sockets::init(32);
 
-    mgmt::start(cfg);
+    let dummy = Rc::new(Semaphore::new(0, 1));
+    mgmt::start(Rc::new(cfg), dummy);
 
     // getting eth settings disables the LED as it resets GPIO
     // need to re-enable it here

src/runtime/src/main.rs

@@ -1,14 +1,14 @@
 #![no_std]
 #![no_main]
 #![recursion_limit = "1024"] // for futures_util::select!
+#![allow(unexpected_cfgs)]
 #![feature(alloc_error_handler)]
-#![feature(const_btree_new)]
+#![feature(const_btree_len)]
-#![feature(panic_info_message)]
 
 #[macro_use]
 extern crate alloc;
 
-#[cfg(feature = "target_kasli_soc")]
+#[cfg(all(feature = "target_kasli_soc", has_virtual_leds))]
 use core::cell::RefCell;
 
 use ksupport;
@@ -21,7 +21,7 @@ use libboard_artiq::{identifier_read, logger, pl};
 use libboard_zynq::{gic, mpcore, timer::GlobalTimer};
 use libconfig::Config;
 use libcortex_a9::l2c::enable_l2_cache;
-use libsupport_zynq::ram;
+use libsupport_zynq::{exception_vectors, ram};
 use log::{info, warn};
 
 mod analyzer;
@@ -38,7 +38,12 @@ mod rtio_mgt;
 #[cfg(has_drtio)]
 mod subkernel;
 
-#[cfg(feature = "target_kasli_soc")]
+// linker symbols
+extern "C" {
+    static __exceptions_start: u32;
+}
+
+#[cfg(all(feature = "target_kasli_soc", has_virtual_leds))]
 async fn io_expanders_service(
     i2c_bus: RefCell<&mut libboard_zynq::i2c::I2c>,
     io_expander0: RefCell<io_expander::IoExpander>,
@@ -77,6 +82,9 @@ static mut LOG_BUFFER: [u8; 1 << 17] = [0; 1 << 17];
 
 #[no_mangle]
 pub fn main_core0() {
+    unsafe {
+        exception_vectors::set_vector_table(&__exceptions_start as *const u32 as u32);
+    }
     enable_l2_cache(0x8);
     let mut timer = GlobalTimer::start();
 
@@ -94,20 +102,21 @@ pub fn main_core0() {
 
     ksupport::i2c::init();
     #[cfg(feature = "target_kasli_soc")]
-    let i2c_bus = unsafe { (ksupport::i2c::I2C_BUS).as_mut().unwrap() };
-
-    #[cfg(feature = "target_kasli_soc")]
-    let (mut io_expander0, mut io_expander1);
-    #[cfg(feature = "target_kasli_soc")]
     {
-        io_expander0 = io_expander::IoExpander::new(i2c_bus, 0).unwrap();
+        let i2c_bus = unsafe { (ksupport::i2c::I2C_BUS).as_mut().unwrap() };
-        io_expander1 = io_expander::IoExpander::new(i2c_bus, 1).unwrap();
+        let mut io_expander0 = io_expander::IoExpander::new(i2c_bus, 0).unwrap();
+        let mut io_expander1 = io_expander::IoExpander::new(i2c_bus, 1).unwrap();
         io_expander0
             .init(i2c_bus)
             .expect("I2C I/O expander #0 initialization failed");
         io_expander1
             .init(i2c_bus)
             .expect("I2C I/O expander #1 initialization failed");
+
+        // Drive CLK_SEL to true
+        #[cfg(has_si549)]
+        io_expander0.set(1, 7, true);
+
         // Drive TX_DISABLE to false on SFP0..3
         io_expander0.set(0, 1, false);
         io_expander1.set(0, 1, false);
@@ -115,6 +124,12 @@ pub fn main_core0() {
         io_expander1.set(1, 1, false);
         io_expander0.service(i2c_bus).unwrap();
         io_expander1.service(i2c_bus).unwrap();
+        #[cfg(has_virtual_leds)]
+        task::spawn(io_expanders_service(
+            RefCell::new(i2c_bus),
+            RefCell::new(io_expander0),
+            RefCell::new(io_expander1),
+        ));
     }
 
     let cfg = match Config::new() {
@@ -135,11 +150,5 @@ pub fn main_core0() {
 
     task::spawn(ksupport::report_async_rtio_errors());
 
-    #[cfg(feature = "target_kasli_soc")]
-    task::spawn(io_expanders_service(
-        RefCell::new(i2c_bus),
-        RefCell::new(io_expander0),
-        RefCell::new(io_expander1),
-    ));
     comms::main(timer, cfg);
 }

src/runtime/src/mgmt.rs

@@ -6,6 +6,7 @@ use libasync::{smoltcp::TcpStream, task};
 use libboard_artiq::logger::{BufferLogger, LogBufferRef};
 use libboard_zynq::{slcr, smoltcp};
 use libconfig::Config;
+use libcortex_a9::semaphore::Semaphore;
 use log::{self, debug, error, info, warn, LevelFilter};
 use num_derive::FromPrimitive;
 use num_traits::FromPrimitive;
@@ -111,7 +112,12 @@ async fn read_key(stream: &mut TcpStream) -> Result<String> {
     Ok(String::from_utf8(buffer).unwrap())
 }
 
-async fn handle_connection(stream: &mut TcpStream, pull_id: Rc<RefCell<u32>>, cfg: Rc<Config>) -> Result<()> {
+async fn handle_connection(
+    stream: &mut TcpStream,
+    pull_id: Rc<RefCell<u32>>,
+    cfg: Rc<Config>,
+    restart_idle: Rc<Semaphore>,
+) -> Result<()> {
     if !expect(&stream, b"ARTIQ management\n").await? {
         return Err(Error::UnexpectedPattern);
     }
@@ -200,6 +206,9 @@ async fn handle_connection(stream: &mut TcpStream, pull_id: Rc<RefCell<u32>>, cf
                 let value = cfg.write(&key, buffer);
                 if value.is_ok() {
                     debug!("write success");
+                    if key == "idle_kernel" {
+                        restart_idle.signal();
+                    }
                     write_i8(stream, Reply::Success as i8).await?;
                 } else {
                     // this is an error because we do not expect write to fail
@@ -213,6 +222,9 @@ async fn handle_connection(stream: &mut TcpStream, pull_id: Rc<RefCell<u32>>, cf
                 let value = cfg.remove(&key);
                 if value.is_ok() {
                     debug!("erase success");
+                    if key == "idle_kernel" {
+                        restart_idle.signal();
+                    }
                     write_i8(stream, Reply::Success as i8).await?;
                 } else {
                     warn!("erase failed");
@@ -229,17 +241,17 @@ async fn handle_connection(stream: &mut TcpStream, pull_id: Rc<RefCell<u32>>, cf
     }
 }
 
-pub fn start(cfg: Config) {
+pub fn start(cfg: Rc<Config>, restart_idle: Rc<Semaphore>) {
     task::spawn(async move {
         let pull_id = Rc::new(RefCell::new(0u32));
-        let cfg = Rc::new(cfg);
         loop {
             let mut stream = TcpStream::accept(1380, 2048, 2048).await.unwrap();
             let pull_id = pull_id.clone();
             let cfg = cfg.clone();
+            let restart_idle = restart_idle.clone();
             task::spawn(async move {
                 info!("received connection");
-                let _ = handle_connection(&mut stream, pull_id, cfg)
+                let _ = handle_connection(&mut stream, pull_id, cfg, restart_idle)
                     .await
                     .map_err(|e| warn!("connection terminated: {:?}", e));
                 let _ = stream.flush().await;

src/runtime/src/moninj.rs

@@ -58,10 +58,11 @@ mod remote_moninj {
     use log::error;
 
     use super::*;
-    use crate::rtio_mgt::drtio;
+    use crate::rtio_mgt::{drtio, drtio::Error as DrtioError};
 
     pub async fn read_probe(
         aux_mutex: &Rc<Mutex<bool>>,
+        routing_table: &drtio_routing::RoutingTable,
         timer: GlobalTimer,
         linkno: u8,
         destination: u8,
@@ -71,6 +72,7 @@ mod remote_moninj {
         let reply = drtio::aux_transact(
             aux_mutex,
             linkno,
+            routing_table,
             &drtioaux_async::Packet::MonitorRequest {
                 destination: destination,
                 channel: channel as _,
@@ -82,8 +84,8 @@ mod remote_moninj {
         match reply {
             Ok(drtioaux_async::Packet::MonitorReply { value }) => return value as i64,
             Ok(packet) => error!("received unexpected aux packet: {:?}", packet),
-            Err("link went down") => {
+            Err(DrtioError::LinkDown) => {
-                debug!("link is down");
+                warn!("link is down");
             }
             Err(e) => error!("aux packet error ({})", e),
         }
@@ -92,6 +94,7 @@ mod remote_moninj {
 
     pub async fn inject(
         aux_mutex: &Rc<Mutex<bool>>,
+        _routing_table: &drtio_routing::RoutingTable,
         _timer: GlobalTimer,
         linkno: u8,
         destination: u8,
@@ -99,7 +102,7 @@ mod remote_moninj {
         overrd: i8,
         value: i8,
     ) {
-        let _lock = aux_mutex.lock();
+        let _lock = aux_mutex.async_lock().await;
         drtioaux_async::send(
             linkno,
             &drtioaux_async::Packet::InjectionRequest {
@@ -115,6 +118,7 @@ mod remote_moninj {
 
     pub async fn read_injection_status(
         aux_mutex: &Rc<Mutex<bool>>,
+        routing_table: &drtio_routing::RoutingTable,
         timer: GlobalTimer,
         linkno: u8,
         destination: u8,
@@ -124,6 +128,7 @@ mod remote_moninj {
         let reply = drtio::aux_transact(
             aux_mutex,
             linkno,
+            routing_table,
             &drtioaux_async::Packet::InjectionStatusRequest {
                 destination: destination,
                 channel: channel as _,
@@ -135,8 +140,8 @@ mod remote_moninj {
         match reply {
             Ok(drtioaux_async::Packet::InjectionStatusReply { value }) => return value as i8,
             Ok(packet) => error!("received unexpected aux packet: {:?}", packet),
-            Err("link went down") => {
+            Err(DrtioError::LinkDown) => {
-                debug!("link is down");
+                warn!("link is down");
             }
             Err(e) => error!("aux packet error ({})", e),
         }
@@ -183,7 +188,7 @@ macro_rules! dispatch {
             local_moninj::$func(channel.into(), $($param, )*)
         } else {
             let linkno = hop - 1 as u8;
-            remote_moninj::$func($aux_mutex, $timer, linkno, destination, channel, $($param, )*).await
+            remote_moninj::$func($aux_mutex, $routing_table, $timer, linkno, destination, channel, $($param, )*).await
         }
     }}
 }

src/runtime/src/panic.rs

@@ -22,7 +22,7 @@ fn panic(info: &core::panic::PanicInfo) -> ! {
     } else {
         print!("unknown location");
     }
-    if let Some(message) = info.message() {
+    if let Some(message) = info.message().as_str() {
         println!(": {}", message);
     } else {
         println!("");
@@ -61,7 +61,7 @@ fn soft_panic(info: &core::panic::PanicInfo) -> ! {
     } else {
         error!("panic at unknown location");
     }
-    if let Some(message) = info.message() {
+    if let Some(message) = info.message().as_str() {
         error!("panic message: {}", message);
     }
     let timer = GlobalTimer::start();

src/runtime/src/rtio_clocking.rs

@@ -1,14 +1,20 @@
+#[cfg(not(feature = "target_ebaz4205"))]
 use embedded_hal::blocking::delay::DelayMs;
 #[cfg(has_si5324)]
 use ksupport::i2c;
+#[cfg(not(feature = "target_ebaz4205"))]
 use libboard_artiq::pl;
 #[cfg(has_si5324)]
 use libboard_artiq::si5324;
+#[cfg(has_si549)]
+use libboard_artiq::si549;
 #[cfg(has_si5324)]
 use libboard_zynq::i2c::I2c;
 use libboard_zynq::timer::GlobalTimer;
 use libconfig::Config;
-use log::{info, warn};
+#[cfg(not(feature = "target_ebaz4205"))]
+use log::info;
+use log::warn;
 
 #[derive(Debug, PartialEq, Copy, Clone)]
 #[allow(non_camel_case_types)]
@@ -67,7 +73,7 @@ fn get_rtio_clock_cfg(cfg: &Config) -> RtioClock {
     res
 }
 
-#[cfg(not(has_drtio))]
+#[cfg(not(any(has_drtio, feature = "target_ebaz4205")))]
 fn init_rtio(timer: &mut GlobalTimer) {
     info!("Switching SYS clocks...");
     unsafe {
@@ -260,6 +266,150 @@ fn setup_si5324(i2c: &mut I2c, timer: &mut GlobalTimer, clk: RtioClock) {
     si5324::setup(i2c, &si5324_settings, si5324_ref_input, timer).expect("cannot initialize Si5324");
 }
 
+#[cfg(all(has_si549, has_wrpll))]
+fn wrpll_setup(timer: &mut GlobalTimer, clk: RtioClock, si549_settings: &si549::FrequencySetting) {
+    // register values are directly copied from preconfigured mmcm
+    let (mmcm_setting, mmcm_bypass) = match clk {
+        RtioClock::Ext0_Synth0_10to125 => (
+            si549::wrpll_refclk::MmcmSetting {
+                // CLKFBOUT_MULT = 62.5, DIVCLK_DIVIDE = 1 , CLKOUT0_DIVIDE = 5
+                clkout0_reg1: 0x1083,
+                clkout0_reg2: 0x0080,
+                clkfbout_reg1: 0x179e,
+                clkfbout_reg2: 0x4c00,
+                div_reg: 0x1041,
+                lock_reg1: 0x00fa,
+                lock_reg2: 0x7c01,
+                lock_reg3: 0xffe9,
+                power_reg: 0x9900,
+                filt_reg1: 0x1008,
+                filt_reg2: 0x8800,
+            },
+            false,
+        ),
+        RtioClock::Ext0_Synth0_80to125 => (
+            si549::wrpll_refclk::MmcmSetting {
+                // CLKFBOUT_MULT = 15.625, DIVCLK_DIVIDE = 1 , CLKOUT0_DIVIDE = 10
+                clkout0_reg1: 0x1145,
+                clkout0_reg2: 0x0000,
+                clkfbout_reg1: 0x11c7,
+                clkfbout_reg2: 0x5880,
+                div_reg: 0x1041,
+                lock_reg1: 0x028a,
+                lock_reg2: 0x7c01,
+                lock_reg3: 0xffe9,
+                power_reg: 0x9900,
+                filt_reg1: 0x9908,
+                filt_reg2: 0x8100,
+            },
+            false,
+        ),
+        RtioClock::Ext0_Synth0_100to125 => (
+            si549::wrpll_refclk::MmcmSetting {
+                // CLKFBOUT_MULT = 12.5, DIVCLK_DIVIDE = 1 , CLKOUT0_DIVIDE = 10
+                clkout0_reg1: 0x1145,
+                clkout0_reg2: 0x0000,
+                clkfbout_reg1: 0x1145,
+                clkfbout_reg2: 0x4c00,
+                div_reg: 0x1041,
+                lock_reg1: 0x0339,
+                lock_reg2: 0x7c01,
+                lock_reg3: 0xffe9,
+                power_reg: 0x9900,
+                filt_reg1: 0x9108,
+                filt_reg2: 0x0100,
+            },
+            false,
+        ),
+        RtioClock::Ext0_Synth0_125to125 => (
+            si549::wrpll_refclk::MmcmSetting {
+                // CLKFBOUT_MULT = 10, DIVCLK_DIVIDE = 1 , CLKOUT0_DIVIDE = 10
+                clkout0_reg1: 0x1145,
+                clkout0_reg2: 0x0000,
+                clkfbout_reg1: 0x1145,
+                clkfbout_reg2: 0x0000,
+                div_reg: 0x1041,
+                lock_reg1: 0x03e8,
+                lock_reg2: 0x7001,
+                lock_reg3: 0xf3e9,
+                power_reg: 0x0100,
+                filt_reg1: 0x9908,
+                filt_reg2: 0x1100,
+            },
+            true,
+        ),
+        _ => unreachable!(),
+    };
+
+    si549::helper_setup(timer, &si549_settings).expect("cannot initialize helper Si549");
+    si549::wrpll_refclk::setup(timer, mmcm_setting, mmcm_bypass).expect("cannot initialize ref clk for wrpll");
+    si549::wrpll::select_recovered_clock(true, timer);
+}
+
+#[cfg(has_si549)]
+fn get_si549_setting(clk: RtioClock) -> si549::FrequencySetting {
+    match clk {
+        RtioClock::Ext0_Synth0_10to125 => {
+            info!("using 10MHz reference to make 125MHz RTIO clock with WRPLL");
+        }
+        RtioClock::Ext0_Synth0_80to125 => {
+            info!("using 80MHz reference to make 125MHz RTIO clock with WRPLL");
+        }
+        RtioClock::Ext0_Synth0_100to125 => {
+            info!("using 100MHz reference to make 125MHz RTIO clock with WRPLL");
+        }
+        RtioClock::Ext0_Synth0_125to125 => {
+            info!("using 125MHz reference to make 125MHz RTIO clock with WRPLL");
+        }
+        RtioClock::Int_100 => {
+            info!("using internal 100MHz RTIO clock");
+        }
+        RtioClock::Int_125 => {
+            info!("using internal 125MHz RTIO clock");
+        }
+        _ => {
+            warn!(
+                "rtio_clock setting '{:?}' is unsupported. Falling back to default internal 125MHz RTIO clock.",
+                clk
+            );
+        }
+    };
+
+    match clk {
+        RtioClock::Int_100 => {
+            si549::FrequencySetting {
+                main: si549::DividerConfig {
+                    hsdiv: 0x06C,
+                    lsdiv: 0,
+                    fbdiv: 0x046C5F49797,
+                },
+                helper: si549::DividerConfig {
+                    // 100MHz*32767/32768
+                    hsdiv: 0x06C,
+                    lsdiv: 0,
+                    fbdiv: 0x046C5670BBD,
+                },
+            }
+        }
+        _ => {
+            // Everything else use 125MHz
+            si549::FrequencySetting {
+                main: si549::DividerConfig {
+                    hsdiv: 0x058,
+                    lsdiv: 0,
+                    fbdiv: 0x04815791F25,
+                },
+                helper: si549::DividerConfig {
+                    // 125MHz*32767/32768
+                    hsdiv: 0x058,
+                    lsdiv: 0,
+                    fbdiv: 0x04814E8F442,
+                },
+            }
+        }
+    }
+}
+
 pub fn init(timer: &mut GlobalTimer, cfg: &Config) {
     let clk = get_rtio_clock_cfg(cfg);
     #[cfg(has_si5324)]
@@ -274,9 +424,29 @@ pub fn init(timer: &mut GlobalTimer, cfg: &Config) {
         }
     }
 
+    #[cfg(has_si549)]
+    let si549_settings = get_si549_setting(clk);
+
+    #[cfg(has_si549)]
+    si549::main_setup(timer, &si549_settings).expect("cannot initialize main Si549");
+
     #[cfg(has_drtio)]
     init_drtio(timer);
 
-    #[cfg(not(has_drtio))]
+    #[cfg(not(any(has_drtio, feature = "target_ebaz4205")))]
     init_rtio(timer);
+
+    #[cfg(all(has_si549, has_wrpll))]
+    {
+        // SYS CLK switch will reset CSRs that are used by WRPLL
+        match clk {
+            RtioClock::Ext0_Synth0_10to125
+            | RtioClock::Ext0_Synth0_80to125
+            | RtioClock::Ext0_Synth0_100to125
+            | RtioClock::Ext0_Synth0_125to125 => {
+                wrpll_setup(timer, clk, &si549_settings);
+            }
+            _ => {}
+        }
+    }
 }

src/runtime/src/rtio_dma.rs

@@ -142,9 +142,9 @@ pub mod remote_dma {
             }
         }
 
-        pub async fn playback_done(&mut self, destination: u8, error: u8, channel: u32, timestamp: u64) {
+        pub async fn playback_done(&mut self, source: u8, error: u8, channel: u32, timestamp: u64) {
             let mut traces_locked = self.traces.async_lock().await;
-            let mut trace = traces_locked.get_mut(&destination).unwrap();
+            let mut trace = traces_locked.get_mut(&source).unwrap();
             trace.state = RemoteState::PlaybackEnded {
                 error: error,
                 channel: channel,

src/runtime/src/rtio_mgt.rs

@@ -1,29 +1,69 @@
 use alloc::rc::Rc;
 use core::cell::RefCell;
 
-use libboard_artiq::{drtio_routing, pl::csr};
+use libboard_artiq::{drtio_routing, drtio_routing::RoutingTable, pl::csr};
 use libboard_zynq::timer::GlobalTimer;
+use libconfig::Config;
 use libcortex_a9::mutex::Mutex;
+use log::{info, warn};
 
 #[cfg(has_drtio)]
 pub mod drtio {
     use alloc::vec::Vec;
+    use core::fmt;
 
     use embedded_hal::blocking::delay::DelayMs;
     use ksupport::{resolve_channel_name, ASYNC_ERROR_BUSY, ASYNC_ERROR_COLLISION, ASYNC_ERROR_SEQUENCE_ERROR,
                    SEEN_ASYNC_ERRORS};
     use libasync::{delay, task};
-    use libboard_artiq::{drtioaux::Error, drtioaux_async, drtioaux_async::Packet,
+    use libboard_artiq::{drtioaux::Error as DrtioError,
-                         drtioaux_proto::MASTER_PAYLOAD_MAX_SIZE};
+                         drtioaux_async,
+                         drtioaux_async::Packet,
+                         drtioaux_proto::{PayloadStatus, MASTER_PAYLOAD_MAX_SIZE}};
     use libboard_zynq::time::Milliseconds;
     use log::{error, info, warn};
 
     use super::*;
     use crate::{analyzer::remote_analyzer::RemoteBuffer, rtio_dma::remote_dma, subkernel};
 
+    #[derive(Debug, PartialEq, Eq, Clone, Copy)]
+    pub enum Error {
+        Timeout,
+        AuxError,
+        LinkDown,
+        UnexpectedReply,
+        DmaAddTraceFail(u8),
+        DmaEraseFail(u8),
+        DmaPlaybackFail(u8),
+        SubkernelAddFail(u8),
+        SubkernelRunFail(u8),
+    }
+
+    impl fmt::Display for Error {
+        fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+            match self {
+                Error::Timeout => write!(f, "timed out"),
+                Error::AuxError => write!(f, "aux packet error"),
+                Error::LinkDown => write!(f, "link down"),
+                Error::UnexpectedReply => write!(f, "unexpected reply"),
+                Error::DmaAddTraceFail(dest) => write!(f, "error adding DMA trace on satellite #{}", dest),
+                Error::DmaEraseFail(dest) => write!(f, "error erasing DMA trace on satellite #{}", dest),
+                Error::DmaPlaybackFail(dest) => write!(f, "error playing back DMA trace on satellite #{}", dest),
+                Error::SubkernelAddFail(dest) => write!(f, "error adding subkernel on satellite #{}", dest),
+                Error::SubkernelRunFail(dest) => write!(f, "error on subkernel run request on satellite #{}", dest),
+            }
+        }
+    }
+
+    impl<T> From<DrtioError<T>> for Error {
+        fn from(_error: DrtioError<T>) -> Self {
+            Error::AuxError
+        }
+    }
+
     pub fn startup(
         aux_mutex: &Rc<Mutex<bool>>,
-        routing_table: &Rc<RefCell<drtio_routing::RoutingTable>>,
+        routing_table: &Rc<RefCell<RoutingTable>>,
         up_destinations: &Rc<RefCell<[bool; drtio_routing::DEST_COUNT]>>,
         timer: GlobalTimer,
     ) {
@@ -41,78 +81,121 @@ pub mod drtio {
         unsafe { (csr::DRTIO[linkno].rx_up_read)() == 1 }
     }
 
-    async fn process_async_packets(aux_mutex: &Mutex<bool>, linkno: u8, packet: Packet) -> Option<Packet> {
+    async fn process_async_packets(linkno: u8, routing_table: &RoutingTable, packet: Packet) -> Option<Packet> {
-        // returns None if an async packet has been consumed
         match packet {
             Packet::DmaPlaybackStatus {
                 id,
-                destination,
+                source,
+                destination: 0,
                 error,
                 channel,
                 timestamp,
             } => {
-                remote_dma::playback_done(id, destination, error, channel, timestamp).await;
+                remote_dma::playback_done(id, source, error, channel, timestamp).await;
                 None
             }
-            Packet::SubkernelFinished { id, with_exception } => {
+            Packet::SubkernelFinished {
-                subkernel::subkernel_finished(id, with_exception).await;
+                id,
+                destination: 0,
+                with_exception,
+                exception_src,
+            } => {
+                subkernel::subkernel_finished(id, with_exception, exception_src).await;
                 None
             }
             Packet::SubkernelMessage {
                 id,
-                destination: from,
+                source,
-                last,
+                destination: 0,
+                status,
                 length,
                 data,
             } => {
-                subkernel::message_handle_incoming(id, last, length as usize, &data).await;
+                subkernel::message_handle_incoming(id, status, length as usize, &data).await;
                 // acknowledge receiving part of the message
-                let _lock = aux_mutex.async_lock().await;
+                drtioaux_async::send(linkno, &Packet::SubkernelMessageAck { destination: source })
-                drtioaux_async::send(linkno, &Packet::SubkernelMessageAck { destination: from })
                     .await
                     .unwrap();
                 None
             }
+            // routable packets
+            Packet::DmaAddTraceRequest { destination, .. }
+            | Packet::DmaAddTraceReply { destination, .. }
+            | Packet::DmaRemoveTraceRequest { destination, .. }
+            | Packet::DmaRemoveTraceReply { destination, .. }
+            | Packet::DmaPlaybackRequest { destination, .. }
+            | Packet::DmaPlaybackReply { destination, .. }
+            | Packet::SubkernelLoadRunRequest { destination, .. }
+            | Packet::SubkernelLoadRunReply { destination, .. }
+            | Packet::SubkernelMessage { destination, .. }
+            | Packet::SubkernelMessageAck { destination, .. }
+            | Packet::SubkernelException { destination, .. }
+            | Packet::SubkernelExceptionRequest { destination, .. }
+            | Packet::DmaPlaybackStatus { destination, .. }
+            | Packet::SubkernelFinished { destination, .. } => {
+                if destination == 0 {
+                    Some(packet)
+                } else {
+                    let dest_link = routing_table.0[destination as usize][0] - 1;
+                    if dest_link == linkno {
+                        warn!(
+                            "[LINK#{}] Re-routed packet would return to the same link, dropping: {:?}",
+                            linkno, packet
+                        );
+                    } else {
+                        drtioaux_async::send(dest_link, &packet).await.unwrap();
+                    }
+                    None
+                }
+            }
             other => Some(other),
         }
     }
 
-    async fn recv_aux_timeout(linkno: u8, timeout: u64, timer: GlobalTimer) -> Result<Packet, &'static str> {
+    async fn recv_aux_timeout(linkno: u8, timeout: u64, timer: GlobalTimer) -> Result<Packet, Error> {
         if !link_rx_up(linkno).await {
-            return Err("link went down");
+            return Err(Error::LinkDown);
         }
         match drtioaux_async::recv_timeout(linkno, Some(timeout), timer).await {
             Ok(packet) => return Ok(packet),
-            Err(Error::TimedOut) => return Err("timed out"),
+            Err(DrtioError::TimedOut) => return Err(Error::Timeout),
-            Err(_) => return Err("aux packet error"),
+            Err(_) => return Err(Error::AuxError),
         }
     }
 
     pub async fn aux_transact(
         aux_mutex: &Mutex<bool>,
         linkno: u8,
+        routing_table: &RoutingTable,
         request: &Packet,
         timer: GlobalTimer,
-    ) -> Result<Packet, &'static str> {
+    ) -> Result<Packet, Error> {
         if !link_rx_up(linkno).await {
-            return Err("link went down");
+            return Err(Error::LinkDown);
         }
         let _lock = aux_mutex.async_lock().await;
         drtioaux_async::send(linkno, request).await.unwrap();
-        Ok(recv_aux_timeout(linkno, 200, timer).await?)
+        loop {
+            let packet = recv_aux_timeout(linkno, 200, timer).await?;
+            if let Some(packet) = process_async_packets(linkno, routing_table, packet).await {
+                return Ok(packet);
+            }
+        }
     }
 
     async fn drain_buffer(linkno: u8, draining_time: Milliseconds, timer: GlobalTimer) {
         let max_time = timer.get_time() + draining_time;
-        loop {
+        while timer.get_time() < max_time {
-            if timer.get_time() > max_time {
-                return;
-            } //could this be cut short?
             let _ = drtioaux_async::recv(linkno).await;
         }
     }
 
-    async fn ping_remote(aux_mutex: &Rc<Mutex<bool>>, linkno: u8, timer: GlobalTimer) -> u32 {
+    async fn ping_remote(
+        aux_mutex: &Rc<Mutex<bool>>,
+        linkno: u8,
+        routing_table: &RoutingTable,
+        timer: GlobalTimer,
+    ) -> u32 {
         let mut count = 0;
         loop {
             if !link_rx_up(linkno).await {
@@ -122,7 +205,7 @@ pub mod drtio {
             if count > 100 {
                 return 0;
             }
-            let reply = aux_transact(aux_mutex, linkno, &Packet::EchoRequest, timer).await;
+            let reply = aux_transact(aux_mutex, linkno, routing_table, &Packet::EchoRequest, timer).await;
             match reply {
                 Ok(Packet::EchoReply) => {
                     // make sure receive buffer is drained
@@ -135,7 +218,7 @@ pub mod drtio {
         }
     }
 
-    async fn sync_tsc(aux_mutex: &Rc<Mutex<bool>>, linkno: u8, timer: GlobalTimer) -> Result<(), &'static str> {
+    async fn sync_tsc(aux_mutex: &Rc<Mutex<bool>>, linkno: u8, timer: GlobalTimer) -> Result<(), Error> {
         let _lock = aux_mutex.async_lock().await;
 
         unsafe {
@@ -146,22 +229,23 @@ pub mod drtio {
         // by the satellite, in response to a TSC set on the RT link.
         let reply = recv_aux_timeout(linkno, 10000, timer).await?;
         if reply == Packet::TSCAck {
-            return Ok(());
+            Ok(())
         } else {
-            return Err("unexpected reply");
+            Err(Error::UnexpectedReply)
         }
     }
 
     async fn load_routing_table(
         aux_mutex: &Rc<Mutex<bool>>,
         linkno: u8,
-        routing_table: &drtio_routing::RoutingTable,
+        routing_table: &RoutingTable,
         timer: GlobalTimer,
-    ) -> Result<(), &'static str> {
+    ) -> Result<(), Error> {
         for i in 0..drtio_routing::DEST_COUNT {
             let reply = aux_transact(
                 aux_mutex,
                 linkno,
+                routing_table,
                 &Packet::RoutingSetPath {
                     destination: i as u8,
                     hops: routing_table.0[i],
@@ -170,7 +254,7 @@ pub mod drtio {
             )
             .await?;
             if reply != Packet::RoutingAck {
-                return Err("unexpected reply");
+                return Err(Error::UnexpectedReply);
             }
         }
         Ok(())
@@ -180,13 +264,21 @@ pub mod drtio {
         aux_mutex: &Rc<Mutex<bool>>,
         linkno: u8,
         rank: u8,
+        routing_table: &RoutingTable,
         timer: GlobalTimer,
-    ) -> Result<(), &'static str> {
+    ) -> Result<(), Error> {
-        let reply = aux_transact(aux_mutex, linkno, &Packet::RoutingSetRank { rank: rank }, timer).await?;
+        let reply = aux_transact(
-        if reply != Packet::RoutingAck {
+            aux_mutex,
-            return Err("unexpected reply");
+            linkno,
+            routing_table,
+            &Packet::RoutingSetRank { rank: rank },
+            timer,
+        )
+        .await?;
+        match reply {
+            Packet::RoutingAck => Ok(()),
+            _ => Err(Error::UnexpectedReply),
         }
-        Ok(())
     }
 
     async fn init_buffer_space(destination: u8, linkno: u8) {
@@ -205,11 +297,11 @@ pub mod drtio {
         }
     }
 
-    async fn process_unsolicited_aux(aux_mutex: &Rc<Mutex<bool>>, linkno: u8) {
+    async fn process_unsolicited_aux(aux_mutex: &Mutex<bool>, linkno: u8, routing_table: &RoutingTable) {
         let _lock = aux_mutex.async_lock().await;
         match drtioaux_async::recv(linkno).await {
             Ok(Some(packet)) => {
-                if let Some(packet) = process_async_packets(aux_mutex, linkno, packet).await {
+                if let Some(packet) = process_async_packets(linkno, routing_table, packet).await {
                     warn!("[LINK#{}] unsolicited aux packet: {:?}", linkno, packet);
                 }
             }
@@ -240,7 +332,7 @@ pub mod drtio {
     }
 
     async fn destination_set_up(
-        routing_table: &drtio_routing::RoutingTable,
+        routing_table: &RoutingTable,
         up_destinations: &Rc<RefCell<[bool; drtio_routing::DEST_COUNT]>>,
         destination: u8,
         up: bool,
@@ -263,7 +355,7 @@ pub mod drtio {
 
     async fn destination_survey(
         aux_mutex: &Rc<Mutex<bool>>,
-        routing_table: &drtio_routing::RoutingTable,
+        routing_table: &RoutingTable,
         up_links: &[bool],
         up_destinations: &Rc<RefCell<[bool; drtio_routing::DEST_COUNT]>>,
         timer: GlobalTimer,
@@ -281,67 +373,54 @@ pub mod drtio {
                 let linkno = hop - 1;
                 if destination_up(up_destinations, destination).await {
                     if up_links[linkno as usize] {
-                        loop {
+                        let reply = aux_transact(
-                            let reply = aux_transact(
+                            aux_mutex,
-                                aux_mutex,
+                            linkno,
-                                linkno,
+                            routing_table,
-                                &Packet::DestinationStatusRequest {
+                            &Packet::DestinationStatusRequest {
-                                    destination: destination,
+                                destination: destination,
-                                },
+                            },
-                                timer,
+                            timer,
-                            )
+                        )
-                            .await;
+                        .await;
-                            match reply {
+                        match reply {
-                                Ok(Packet::DestinationDownReply) => {
+                            Ok(Packet::DestinationDownReply) => {
-                                    destination_set_up(routing_table, up_destinations, destination, false).await;
+                                destination_set_up(routing_table, up_destinations, destination, false).await;
-                                    remote_dma::destination_changed(
+                                remote_dma::destination_changed(aux_mutex, routing_table, timer, destination, false)
-                                        aux_mutex,
+                                    .await;
-                                        routing_table,
+                                subkernel::destination_changed(aux_mutex, routing_table, timer, destination, false)
-                                        timer,
-                                        destination,
-                                        false,
-                                    )
                                     .await;
-                                    subkernel::destination_changed(aux_mutex, routing_table, timer, destination, false)
-                                        .await;
-                                }
-                                Ok(Packet::DestinationOkReply) => (),
-                                Ok(Packet::DestinationSequenceErrorReply { channel }) => {
-                                    error!(
-                                        "[DEST#{}] RTIO sequence error involving channel 0x{:04x}:{}",
-                                        destination,
-                                        channel,
-                                        resolve_channel_name(channel as u32)
-                                    );
-                                    unsafe { SEEN_ASYNC_ERRORS |= ASYNC_ERROR_SEQUENCE_ERROR };
-                                }
-                                Ok(Packet::DestinationCollisionReply { channel }) => {
-                                    error!(
-                                        "[DEST#{}] RTIO collision involving channel 0x{:04x}:{}",
-                                        destination,
-                                        channel,
-                                        resolve_channel_name(channel as u32)
-                                    );
-                                    unsafe { SEEN_ASYNC_ERRORS |= ASYNC_ERROR_COLLISION };
-                                }
-                                Ok(Packet::DestinationBusyReply { channel }) => {
-                                    error!(
-                                        "[DEST#{}] RTIO busy error involving channel 0x{:04x}:{}",
-                                        destination,
-                                        channel,
-                                        resolve_channel_name(channel as u32)
-                                    );
-                                    unsafe { SEEN_ASYNC_ERRORS |= ASYNC_ERROR_BUSY };
-                                }
-                                Ok(packet) => match process_async_packets(aux_mutex, linkno, packet).await {
-                                    Some(packet) => {
-                                        error!("[DEST#{}] received unexpected aux packet: {:?}", destination, packet)
-                                    }
-                                    None => continue,
-                                },
-                                Err(e) => error!("[DEST#{}] communication failed ({})", destination, e),
                             }
-                            break;
+                            Ok(Packet::DestinationOkReply) => (),
+                            Ok(Packet::DestinationSequenceErrorReply { channel }) => {
+                                error!(
+                                    "[DEST#{}] RTIO sequence error involving channel 0x{:04x}:{}",
+                                    destination,
+                                    channel,
+                                    resolve_channel_name(channel as u32)
+                                );
+                                unsafe { SEEN_ASYNC_ERRORS |= ASYNC_ERROR_SEQUENCE_ERROR };
+                            }
+                            Ok(Packet::DestinationCollisionReply { channel }) => {
+                                error!(
+                                    "[DEST#{}] RTIO collision involving channel 0x{:04x}:{}",
+                                    destination,
+                                    channel,
+                                    resolve_channel_name(channel as u32)
+                                );
+                                unsafe { SEEN_ASYNC_ERRORS |= ASYNC_ERROR_COLLISION };
+                            }
+                            Ok(Packet::DestinationBusyReply { channel }) => {
+                                error!(
+                                    "[DEST#{}] RTIO busy error involving channel 0x{:04x}:{}",
+                                    destination,
+                                    channel,
+                                    resolve_channel_name(channel as u32)
+                                );
+                                unsafe { SEEN_ASYNC_ERRORS |= ASYNC_ERROR_BUSY };
+                            }
+                            Ok(packet) => error!("[DEST#{}] received unexpected aux packet: {:?}", destination, packet),
+                            Err(e) => error!("[DEST#{}] communication failed ({})", destination, e),
                         }
                     } else {
                         destination_set_up(routing_table, up_destinations, destination, false).await;
@@ -353,6 +432,7 @@ pub mod drtio {
                         let reply = aux_transact(
                             aux_mutex,
                             linkno,
+                            routing_table,
                             &Packet::DestinationStatusRequest {
                                 destination: destination,
                             },
@@ -380,7 +460,7 @@ pub mod drtio {
 
     pub async fn link_task(
         aux_mutex: &Rc<Mutex<bool>>,
-        routing_table: &drtio_routing::RoutingTable,
+        routing_table: &RoutingTable,
         up_destinations: &Rc<RefCell<[bool; drtio_routing::DEST_COUNT]>>,
         timer: GlobalTimer,
     ) {
@@ -391,7 +471,7 @@ pub mod drtio {
                 if up_links[linkno as usize] {
                     /* link was previously up */
                     if link_rx_up(linkno).await {
-                        process_unsolicited_aux(aux_mutex, linkno).await;
+                        process_unsolicited_aux(aux_mutex, linkno, routing_table).await;
                         process_local_errors(linkno).await;
                     } else {
                         info!("[LINK#{}] link is down", linkno);
@@ -401,7 +481,7 @@ pub mod drtio {
                     /* link was previously down */
                     if link_rx_up(linkno).await {
                         info!("[LINK#{}] link RX became up, pinging", linkno);
-                        let ping_count = ping_remote(aux_mutex, linkno, timer).await;
+                        let ping_count = ping_remote(aux_mutex, linkno, routing_table, timer).await;
                         if ping_count > 0 {
                             info!("[LINK#{}] remote replied after {} packets", linkno, ping_count);
                             up_links[linkno as usize] = true;
@@ -411,7 +491,7 @@ pub mod drtio {
                             if let Err(e) = load_routing_table(aux_mutex, linkno, routing_table, timer).await {
                                 error!("[LINK#{}] failed to load routing table ({})", linkno, e);
                             }
-                            if let Err(e) = set_rank(aux_mutex, linkno, 1 as u8, timer).await {
+                            if let Err(e) = set_rank(aux_mutex, linkno, 1 as u8, routing_table, timer).await {
                                 error!("[LINK#{}] failed to set rank ({})", linkno, e);
                             }
                             info!("[LINK#{}] link initialization completed", linkno);
@@ -428,7 +508,7 @@ pub mod drtio {
     }
 
     #[allow(dead_code)]
-    pub fn reset(aux_mutex: Rc<Mutex<bool>>, mut timer: GlobalTimer) {
+    pub fn reset(aux_mutex: Rc<Mutex<bool>>, routing_table: &RoutingTable, mut timer: GlobalTimer) {
         for linkno in 0..csr::DRTIO.len() {
             unsafe {
                 (csr::DRTIO[linkno].reset_write)(1);
@@ -444,7 +524,13 @@ pub mod drtio {
         for linkno in 0..csr::DRTIO.len() {
             let linkno = linkno as u8;
             if task::block_on(link_rx_up(linkno)) {
-                let reply = task::block_on(aux_transact(&aux_mutex, linkno, &Packet::ResetRequest, timer));
+                let reply = task::block_on(aux_transact(
+                    &aux_mutex,
+                    linkno,
+                    routing_table,
+                    &Packet::ResetRequest,
+                    timer,
+                ));
                 match reply {
                     Ok(Packet::ResetAck) => (),
                     Ok(_) => error!("[LINK#{}] reset failed, received unexpected aux packet", linkno),
@@ -457,14 +543,15 @@ pub mod drtio {
     async fn partition_data<PacketF, HandlerF>(
         linkno: u8,
         aux_mutex: &Rc<Mutex<bool>>,
+        routing_table: &RoutingTable,
         timer: GlobalTimer,
         data: &[u8],
         packet_f: PacketF,
         reply_handler_f: HandlerF,
-    ) -> Result<(), &'static str>
+    ) -> Result<(), Error>
     where
-        PacketF: Fn(&[u8; MASTER_PAYLOAD_MAX_SIZE], bool, usize) -> Packet,
+        PacketF: Fn(&[u8; MASTER_PAYLOAD_MAX_SIZE], PayloadStatus, usize) -> Packet,
-        HandlerF: Fn(&Packet) -> Result<(), &'static str>,
+        HandlerF: Fn(&Packet) -> Result<(), Error>,
     {
         let mut i = 0;
         while i < data.len() {
@@ -474,11 +561,13 @@ pub mod drtio {
             } else {
                 data.len() - i
             } as usize;
+            let first = i == 0;
             let last = i + len == data.len();
             slice[..len].clone_from_slice(&data[i..i + len]);
             i += len;
-            let packet = packet_f(&slice, last, len);
+            let status = PayloadStatus::from_status(first, last);
-            let reply = aux_transact(aux_mutex, linkno, &packet, timer).await?;
+            let packet = packet_f(&slice, status, len);
+            let reply = aux_transact(aux_mutex, linkno, routing_table, &packet, timer).await?;
             reply_handler_f(&reply)?;
         }
         Ok(())
@@ -486,29 +575,39 @@ pub mod drtio {
 
     pub async fn ddma_upload_trace(
         aux_mutex: &Rc<Mutex<bool>>,
-        routing_table: &drtio_routing::RoutingTable,
+        routing_table: &RoutingTable,
         timer: GlobalTimer,
         id: u32,
         destination: u8,
         trace: &Vec<u8>,
-    ) -> Result<(), &'static str> {
+    ) -> Result<(), Error> {
         let linkno = routing_table.0[destination as usize][0] - 1;
         partition_data(
             linkno,
             aux_mutex,
+            routing_table,
             timer,
             trace,
-            |slice, last, len| Packet::DmaAddTraceRequest {
+            |slice, status, len| Packet::DmaAddTraceRequest {
                 id: id,
+                source: 0,
                 destination: destination,
-                last: last,
+                status: status,
                 length: len as u16,
                 trace: *slice,
             },
             |reply| match reply {
-                Packet::DmaAddTraceReply { succeeded: true } => Ok(()),
+                Packet::DmaAddTraceReply {
-                Packet::DmaAddTraceReply { succeeded: false } => Err("error adding trace on satellite"),
+                    destination: 0,
-                _ => Err("adding DMA trace failed, unexpected aux packet"),
+                    succeeded: true,
+                    ..
+                } => Ok(()),
+                Packet::DmaAddTraceReply {
+                    destination: 0,
+                    succeeded: false,
+                    ..
+                } => Err(Error::DmaAddTraceFail(destination)),
+                _ => Err(Error::UnexpectedReply),
             },
         )
         .await
@@ -516,82 +615,96 @@ pub mod drtio {
 
     pub async fn ddma_send_erase(
         aux_mutex: &Rc<Mutex<bool>>,
-        routing_table: &drtio_routing::RoutingTable,
+        routing_table: &RoutingTable,
         timer: GlobalTimer,
         id: u32,
         destination: u8,
-    ) -> Result<(), &'static str> {
+    ) -> Result<(), Error> {
         let linkno = routing_table.0[destination as usize][0] - 1;
         let reply = aux_transact(
             aux_mutex,
             linkno,
+            routing_table,
             &Packet::DmaRemoveTraceRequest {
                 id: id,
+                source: 0,
                 destination: destination,
             },
             timer,
         )
-        .await;
+        .await?;
         match reply {
-            Ok(Packet::DmaRemoveTraceReply { succeeded: true }) => Ok(()),
+            Packet::DmaRemoveTraceReply {
-            Ok(Packet::DmaRemoveTraceReply { succeeded: false }) => Err("satellite DMA erase error"),
+                destination: 0,
-            Ok(_) => Err("adding trace failed, unexpected aux packet"),
+                succeeded: true,
-            Err(_) => Err("erasing trace failed, aux error"),
+            } => Ok(()),
+            Packet::DmaRemoveTraceReply {
+                destination: 0,
+                succeeded: false,
+            } => Err(Error::DmaEraseFail(destination)),
+            _ => Err(Error::UnexpectedReply),
         }
     }
 
     pub async fn ddma_send_playback(
         aux_mutex: &Rc<Mutex<bool>>,
-        routing_table: &drtio_routing::RoutingTable,
+        routing_table: &RoutingTable,
         timer: GlobalTimer,
         id: u32,
         destination: u8,
         timestamp: u64,
-    ) -> Result<(), &'static str> {
+    ) -> Result<(), Error> {
         let linkno = routing_table.0[destination as usize][0] - 1;
         let reply = aux_transact(
             aux_mutex,
             linkno,
+            routing_table,
             &Packet::DmaPlaybackRequest {
                 id: id,
+                source: 0,
                 destination: destination,
                 timestamp: timestamp,
             },
             timer,
         )
-        .await;
+        .await?;
         match reply {
-            Ok(Packet::DmaPlaybackReply { succeeded: true }) => Ok(()),
+            Packet::DmaPlaybackReply {
-            Ok(Packet::DmaPlaybackReply { succeeded: false }) => Err("error on DMA playback request"),
+                destination: 0,
-            Ok(_) => Err("received unexpected aux packet during DMA playback"),
+                succeeded: true,
-            Err(_) => Err("aux error on DMA playback"),
+            } => Ok(()),
+            Packet::DmaPlaybackReply {
+                destination: 0,
+                succeeded: false,
+            } => Err(Error::DmaPlaybackFail(destination)),
+            _ => Err(Error::UnexpectedReply),
         }
     }
 
     async fn analyzer_get_data(
         aux_mutex: &Rc<Mutex<bool>>,
-        routing_table: &drtio_routing::RoutingTable,
+        routing_table: &RoutingTable,
         timer: GlobalTimer,
         destination: u8,
-    ) -> Result<RemoteBuffer, &'static str> {
+    ) -> Result<RemoteBuffer, Error> {
         let linkno = routing_table.0[destination as usize][0] - 1;
         let reply = aux_transact(
             aux_mutex,
             linkno,
+            routing_table,
             &Packet::AnalyzerHeaderRequest {
                 destination: destination,
             },
             timer,
         )
-        .await;
+        .await?;
         let (sent, total, overflow) = match reply {
-            Ok(Packet::AnalyzerHeader {
+            Packet::AnalyzerHeader {
                 sent_bytes,
                 total_byte_count,
                 overflow_occurred,
-            }) => (sent_bytes, total_byte_count, overflow_occurred),
+            } => (sent_bytes, total_byte_count, overflow_occurred),
-            Ok(_) => return Err("received unexpected aux packet during remote analyzer header request"),
+            _ => return Err(Error::UnexpectedReply),
-            Err(e) => return Err(e),
         };
 
         let mut remote_data: Vec<u8> = Vec::new();
@@ -601,19 +714,19 @@ pub mod drtio {
                 let reply = aux_transact(
                     aux_mutex,
                     linkno,
+                    routing_table,
                     &Packet::AnalyzerDataRequest {
                         destination: destination,
                     },
                     timer,
                 )
-                .await;
+                .await?;
                 match reply {
-                    Ok(Packet::AnalyzerData { last, length, data }) => {
+                    Packet::AnalyzerData { last, length, data } => {
                         last_packet = last;
                         remote_data.extend(&data[0..length as usize]);
                     }
-                    Ok(_) => return Err("received unexpected aux packet during remote analyzer data request"),
+                    _ => return Err(Error::UnexpectedReply),
-                    Err(e) => return Err(e),
                 }
             }
         }
@@ -628,10 +741,10 @@ pub mod drtio {
 
     pub async fn analyzer_query(
         aux_mutex: &Rc<Mutex<bool>>,
-        routing_table: &drtio_routing::RoutingTable,
+        routing_table: &RoutingTable,
         up_destinations: &Rc<RefCell<[bool; drtio_routing::DEST_COUNT]>>,
         timer: GlobalTimer,
-    ) -> Result<Vec<RemoteBuffer>, &'static str> {
+    ) -> Result<Vec<RemoteBuffer>, Error> {
         let mut remote_buffers: Vec<RemoteBuffer> = Vec::new();
         for i in 1..drtio_routing::DEST_COUNT {
             if destination_up(up_destinations, i as u8).await {
@@ -643,29 +756,30 @@ pub mod drtio {
 
     pub async fn subkernel_upload(
         aux_mutex: &Rc<Mutex<bool>>,
-        routing_table: &drtio_routing::RoutingTable,
+        routing_table: &RoutingTable,
         timer: GlobalTimer,
         id: u32,
         destination: u8,
         data: &Vec<u8>,
-    ) -> Result<(), &'static str> {
+    ) -> Result<(), Error> {
         let linkno = routing_table.0[destination as usize][0] - 1;
         partition_data(
             linkno,
             aux_mutex,
+            routing_table,
             timer,
             data,
-            |slice, last, len| Packet::SubkernelAddDataRequest {
+            |slice, status, len| Packet::SubkernelAddDataRequest {
                 id: id,
                 destination: destination,
-                last: last,
+                status: status,
                 length: len as u16,
                 data: *slice,
             },
             |reply| match reply {
                 Packet::SubkernelAddDataReply { succeeded: true } => Ok(()),
-                Packet::SubkernelAddDataReply { succeeded: false } => Err("error adding subkernel on satellite"),
+                Packet::SubkernelAddDataReply { succeeded: false } => Err(Error::SubkernelAddFail(destination)),
-                _ => Err("adding subkernel failed, unexpected aux packet"),
+                _ => Err(Error::UnexpectedReply),
             },
         )
         .await
@@ -673,85 +787,104 @@ pub mod drtio {
 
     pub async fn subkernel_load(
         aux_mutex: &Rc<Mutex<bool>>,
-        routing_table: &drtio_routing::RoutingTable,
+        routing_table: &RoutingTable,
         timer: GlobalTimer,
         id: u32,
         destination: u8,
         run: bool,
-    ) -> Result<(), &'static str> {
+        timestamp: u64,
+    ) -> Result<(), Error> {
         let linkno = routing_table.0[destination as usize][0] - 1;
         let reply = aux_transact(
             aux_mutex,
             linkno,
+            routing_table,
             &Packet::SubkernelLoadRunRequest {
                 id: id,
+                source: 0,
                 destination: destination,
                 run: run,
+                timestamp,
             },
             timer,
         )
         .await?;
         match reply {
-            Packet::SubkernelLoadRunReply { succeeded: true } => return Ok(()),
+            Packet::SubkernelLoadRunReply {
-            Packet::SubkernelLoadRunReply { succeeded: false } => return Err("error on subkernel run request"),
+                destination: 0,
-            _ => return Err("received unexpected aux packet during subkernel run"),
+                succeeded: true,
+            } => return Ok(()),
+            Packet::SubkernelLoadRunReply {
+                destination: 0,
+                succeeded: false,
+            } => return Err(Error::SubkernelRunFail(destination)),
+            _ => Err(Error::UnexpectedReply),
         }
     }
 
     pub async fn subkernel_retrieve_exception(
         aux_mutex: &Rc<Mutex<bool>>,
-        routing_table: &drtio_routing::RoutingTable,
+        routing_table: &RoutingTable,
         timer: GlobalTimer,
         destination: u8,
-    ) -> Result<Vec<u8>, &'static str> {
+    ) -> Result<Vec<u8>, Error> {
         let linkno = routing_table.0[destination as usize][0] - 1;
         let mut remote_data: Vec<u8> = Vec::new();
         loop {
             let reply = aux_transact(
                 aux_mutex,
                 linkno,
+                routing_table,
                 &Packet::SubkernelExceptionRequest {
+                    source: 0,
                     destination: destination,
                 },
                 timer,
             )
             .await?;
             match reply {
-                Packet::SubkernelException { last, length, data } => {
+                Packet::SubkernelException {
+                    destination: 0,
+                    last,
+                    length,
+                    data,
+                } => {
                     remote_data.extend(&data[0..length as usize]);
                     if last {
                         return Ok(remote_data);
                     }
                 }
-                _ => return Err("received unexpected aux packet during subkernel exception request"),
+                _ => return Err(Error::UnexpectedReply),
             }
         }
     }
 
     pub async fn subkernel_send_message(
         aux_mutex: &Rc<Mutex<bool>>,
-        routing_table: &drtio_routing::RoutingTable,
+        routing_table: &RoutingTable,
         timer: GlobalTimer,
         id: u32,
         destination: u8,
         message: &[u8],
-    ) -> Result<(), &'static str> {
+    ) -> Result<(), Error> {
         let linkno = routing_table.0[destination as usize][0] - 1;
         partition_data(
             linkno,
             aux_mutex,
+            routing_table,
             timer,
             message,
-            |slice, last, len| Packet::SubkernelMessage {
+            |slice, status, len| Packet::SubkernelMessage {
+                source: 0,
                 destination: destination,
                 id: id,
-                last: last,
+                status: status,
                 length: len as u16,
                 data: *slice,
             },
             |reply| match reply {
                 Packet::SubkernelMessageAck { .. } => Ok(()),
-                _ => Err("sending message to subkernel failed, unexpected aux packet"),
+                _ => Err(Error::UnexpectedReply),
             },
         )
         .await
@@ -764,22 +897,46 @@ pub mod drtio {
 
     pub fn startup(
         _aux_mutex: &Rc<Mutex<bool>>,
-        _routing_table: &Rc<RefCell<drtio_routing::RoutingTable>>,
+        _routing_table: &Rc<RefCell<RoutingTable>>,
         _up_destinations: &Rc<RefCell<[bool; drtio_routing::DEST_COUNT]>>,
         _timer: GlobalTimer,
     ) {
     }
 
     #[allow(dead_code)]
-    pub fn reset(_aux_mutex: Rc<Mutex<bool>>, mut _timer: GlobalTimer) {}
+    pub fn reset(_aux_mutex: Rc<Mutex<bool>>, _routing_table: &RoutingTable, mut _timer: GlobalTimer) {}
+}
+
+fn toggle_sed_spread(val: u8) {
+    unsafe {
+        csr::rtio_core::sed_spread_enable_write(val);
+    }
+}
+
+fn setup_sed_spread(cfg: &Config) {
+    if let Ok(spread_enable) = cfg.read_str("sed_spread_enable") {
+        match spread_enable.as_ref() {
+            "1" => toggle_sed_spread(1),
+            "0" => toggle_sed_spread(0),
+            _ => {
+                warn!("sed_spread_enable value not supported (only 1, 0 allowed), disabling by default");
+                toggle_sed_spread(0)
+            }
+        };
+    } else {
+        info!("SED spreading disabled by default");
+        toggle_sed_spread(0);
+    }
 }
 
 pub fn startup(
     aux_mutex: &Rc<Mutex<bool>>,
-    routing_table: &Rc<RefCell<drtio_routing::RoutingTable>>,
+    routing_table: &Rc<RefCell<RoutingTable>>,
     up_destinations: &Rc<RefCell<[bool; drtio_routing::DEST_COUNT]>>,
+    cfg: &Config,
     timer: GlobalTimer,
 ) {
+    setup_sed_spread(cfg);
     drtio::startup(aux_mutex, routing_table, up_destinations, timer);
     unsafe {
         csr::rtio_core::reset_phy_write(1);
@@ -787,9 +944,9 @@ pub fn startup(
 }
 
 #[allow(dead_code)]
-pub fn reset(aux_mutex: Rc<Mutex<bool>>, timer: GlobalTimer) {
+pub fn reset(aux_mutex: Rc<Mutex<bool>>, routing_table: &RoutingTable, timer: GlobalTimer) {
     unsafe {
         csr::rtio_core::reset_write(1);
     }
-    drtio::reset(aux_mutex, timer)
+    drtio::reset(aux_mutex, routing_table, timer)
 }

src/runtime/src/subkernel.rs

@@ -1,18 +1,19 @@
 use alloc::{collections::BTreeMap, rc::Rc, vec::Vec};
 
 use libasync::task;
-use libboard_artiq::{drtio_routing::RoutingTable, drtioaux_proto::MASTER_PAYLOAD_MAX_SIZE};
+use libboard_artiq::{drtio_routing::RoutingTable,
+                     drtioaux_proto::{PayloadStatus, MASTER_PAYLOAD_MAX_SIZE}};
 use libboard_zynq::{time::Milliseconds, timer::GlobalTimer};
 use libcortex_a9::mutex::Mutex;
-use log::error;
+use log::{error, warn};
 
-use crate::rtio_mgt::drtio;
+use crate::rtio_mgt::{drtio, drtio::Error as DrtioError};
 
 #[derive(Debug, PartialEq, Clone, Copy)]
 pub enum FinishStatus {
     Ok,
     CommLost,
-    Exception,
+    Exception(u8), // exception source
 }
 
 #[derive(Debug, PartialEq, Clone, Copy)]
@@ -28,12 +29,13 @@ pub enum Error {
     Timeout,
     IncorrectState,
     SubkernelNotFound,
+    SubkernelException,
     CommLost,
-    DrtioError(&'static str),
+    DrtioError(DrtioError),
 }
 
-impl From<&'static str> for Error {
+impl From<DrtioError> for Error {
-    fn from(value: &'static str) -> Error {
+    fn from(value: DrtioError) -> Error {
         Error::DrtioError(value)
     }
 }
@@ -98,12 +100,22 @@ pub async fn load(
     timer: GlobalTimer,
     id: u32,
     run: bool,
+    timestamp: u64,
 ) -> Result<(), Error> {
     if let Some(subkernel) = SUBKERNELS.async_lock().await.get_mut(&id) {
         if subkernel.state != SubkernelState::Uploaded {
             return Err(Error::IncorrectState);
         }
-        drtio::subkernel_load(aux_mutex, routing_table, timer, id, subkernel.destination, run).await?;
+        drtio::subkernel_load(
+            aux_mutex,
+            routing_table,
+            timer,
+            id,
+            subkernel.destination,
+            run,
+            timestamp,
+        )
+        .await?;
         if run {
             subkernel.state = SubkernelState::Running;
         }
@@ -119,15 +131,17 @@ pub async fn clear_subkernels() {
     CURRENT_MESSAGES.async_lock().await.clear();
 }
 
-pub async fn subkernel_finished(id: u32, with_exception: bool) {
+pub async fn subkernel_finished(id: u32, with_exception: bool, exception_src: u8) {
     // called upon receiving DRTIO SubkernelRunDone
     // may be None if session ends and is cleared
     if let Some(subkernel) = SUBKERNELS.async_lock().await.get_mut(&id) {
-        subkernel.state = SubkernelState::Finished {
+        if subkernel.state == SubkernelState::Running {
-            status: match with_exception {
+            subkernel.state = SubkernelState::Finished {
-                true => FinishStatus::Exception,
+                status: match with_exception {
-                false => FinishStatus::Ok,
+                    true => FinishStatus::Exception(exception_src),
-            },
+                    false => FinishStatus::Ok,
+                },
+            }
         }
     }
 }
@@ -165,25 +179,34 @@ pub async fn await_finish(
     routing_table: &RoutingTable,
     timer: GlobalTimer,
     id: u32,
-    timeout: u64,
+    timeout: i64,
 ) -> Result<SubkernelFinished, Error> {
     match SUBKERNELS.async_lock().await.get(&id).unwrap().state {
         SubkernelState::Running | SubkernelState::Finished { .. } => (),
         _ => return Err(Error::IncorrectState),
     }
-    let max_time = timer.get_time() + Milliseconds(timeout);
+    if timeout > 0 {
-    while timer.get_time() < max_time {
+        let max_time = timer.get_time() + Milliseconds(timeout as u64);
-        {
+        while timer.get_time() < max_time {
             match SUBKERNELS.async_lock().await.get(&id).unwrap().state {
                 SubkernelState::Finished { .. } => break,
                 _ => (),
             };
+            task::r#yield().await;
+        }
+        if timer.get_time() >= max_time {
+            error!("Remote subkernel finish await timed out");
+            return Err(Error::Timeout);
+        }
+    } else {
+        // no timeout, wait forever
+        loop {
+            match SUBKERNELS.async_lock().await.get(&id).unwrap().state {
+                SubkernelState::Finished { .. } => break,
+                _ => (),
+            };
+            task::r#yield().await;
         }
-        task::r#yield().await;
-    }
-    if timer.get_time() >= max_time {
-        error!("Remote subkernel finish await timed out");
-        return Err(Error::Timeout);
     }
     if let Some(subkernel) = SUBKERNELS.async_lock().await.get_mut(&id) {
         match subkernel.state {
@@ -192,11 +215,8 @@ pub async fn await_finish(
                 Ok(SubkernelFinished {
                     id: id,
                     status: status,
-                    exception: if status == FinishStatus::Exception {
+                    exception: if let FinishStatus::Exception(dest) = status {
-                        Some(
+                        Some(drtio::subkernel_retrieve_exception(aux_mutex, routing_table, timer, dest).await?)
-                            drtio::subkernel_retrieve_exception(aux_mutex, routing_table, timer, subkernel.destination)
-                                .await?,
-                        )
                     } else {
                         None
                     },
@@ -220,13 +240,28 @@ static MESSAGE_QUEUE: Mutex<Vec<Message>> = Mutex::new(Vec::new());
 // currently under construction message(s) (can be from multiple sources)
 static CURRENT_MESSAGES: Mutex<BTreeMap<u32, Message>> = Mutex::new(BTreeMap::new());
 
-pub async fn message_handle_incoming(id: u32, last: bool, length: usize, data: &[u8; MASTER_PAYLOAD_MAX_SIZE]) {
+pub async fn message_handle_incoming(
+    id: u32,
+    status: PayloadStatus,
+    length: usize,
+    data: &[u8; MASTER_PAYLOAD_MAX_SIZE],
+) {
     // called when receiving a message from satellite
-    if SUBKERNELS.async_lock().await.get(&id).is_none() {
+    {
-        // do not add messages for non-existing or deleted subkernels
+        let subkernel_lock = SUBKERNELS.async_lock().await;
-        return;
+        let subkernel = subkernel_lock.get(&id);
+        if subkernel.is_some() && subkernel.unwrap().state != SubkernelState::Running {
+            // do not add messages for non-running or deleted subkernels
+            warn!("received a message for a non-running subkernel #{}", id);
+            return;
+        }
     }
     let mut current_messages = CURRENT_MESSAGES.async_lock().await;
+
+    if status.is_first() {
+        current_messages.remove(&id);
+    }
+
     match current_messages.get_mut(&id) {
         Some(message) => message.data.extend(&data[..length]),
         None => {
@@ -240,7 +275,7 @@ pub async fn message_handle_incoming(id: u32, last: bool, length: usize, data: &
             );
         }
     };
-    if last {
+    if status.is_last() {
         // when done, remove from working queue
         MESSAGE_QUEUE
             .async_lock()
@@ -249,16 +284,19 @@ pub async fn message_handle_incoming(id: u32, last: bool, length: usize, data: &
     }
 }
 
-pub async fn message_await(id: u32, timeout: u64, timer: GlobalTimer) -> Result<Message, Error> {
+pub async fn message_await(id: u32, timeout: i64, timer: GlobalTimer) -> Result<Message, Error> {
-    match SUBKERNELS.async_lock().await.get(&id).unwrap().state {
+    let is_subkernel = SUBKERNELS.async_lock().await.get(&id).is_some();
-        SubkernelState::Finished {
+    if is_subkernel {
-            status: FinishStatus::CommLost,
+        match SUBKERNELS.async_lock().await.get(&id).unwrap().state {
-        } => return Err(Error::CommLost),
+            SubkernelState::Finished {
-        SubkernelState::Running | SubkernelState::Finished { .. } => (),
+                status: FinishStatus::CommLost,
-        _ => return Err(Error::IncorrectState),
+            } => return Err(Error::CommLost),
+            SubkernelState::Running | SubkernelState::Finished { .. } => (),
+            _ => return Err(Error::IncorrectState),
+        }
     }
-    let max_time = timer.get_time() + Milliseconds(timeout);
+    let max_time = timer.get_time() + Milliseconds(timeout as u64);
-    while timer.get_time() < max_time {
+    while timeout < 0 || (timeout > 0 && timer.get_time() < max_time) {
         {
             let mut message_queue = MESSAGE_QUEUE.async_lock().await;
             for i in 0..message_queue.len() {
@@ -269,6 +307,17 @@ pub async fn message_await(id: u32, timeout: u64, timer: GlobalTimer) -> Result<
                 }
             }
         }
+        if is_subkernel {
+            match SUBKERNELS.async_lock().await.get(&id).unwrap().state {
+                SubkernelState::Finished {
+                    status: FinishStatus::CommLost,
+                } => return Err(Error::CommLost),
+                SubkernelState::Finished {
+                    status: FinishStatus::Exception(_),
+                } => return Err(Error::SubkernelException),
+                _ => (),
+            }
+        }
         task::r#yield().await;
     }
     Err(Error::Timeout)
@@ -279,9 +328,8 @@ pub async fn message_send<'a>(
     routing_table: &RoutingTable,
     timer: GlobalTimer,
     id: u32,
+    destination: u8,
     message: Vec<u8>,
 ) -> Result<(), Error> {
-    let destination = SUBKERNELS.async_lock().await.get(&id).unwrap().destination;
-    // rpc data prepared by the kernel core already
     Ok(drtio::subkernel_send_message(aux_mutex, routing_table, timer, id, destination, &message).await?)
 }

src/satman/Cargo.toml.tpl

@@ -7,6 +7,7 @@ build = "build.rs"
 [features]
 target_zc706 = ["libboard_zynq/target_zc706", "libsupport_zynq/target_zc706", "libconfig/target_zc706", "libboard_artiq/target_zc706"]
 target_kasli_soc = ["libboard_zynq/target_kasli_soc", "libsupport_zynq/target_kasli_soc", "libconfig/target_kasli_soc", "libboard_artiq/target_kasli_soc"]
+calibrate_wrpll_skew = ["libboard_artiq/calibrate_wrpll_skew"]
 default = ["target_zc706", ]
 
 [build-dependencies]
@@ -14,7 +15,6 @@ build_zynq = { path = "../libbuild_zynq" }
 
 [dependencies]
 log = { version = "0.4", default-features = false }
-core_io = { version = "0.1", features = ["collections"] }
 cslice = "0.3"
 embedded-hal = "0.2"
 

src/satman/src/dma.rs

@@ -1,7 +1,13 @@
-use alloc::{collections::btree_map::BTreeMap, vec::Vec};
+use alloc::{collections::btree_map::BTreeMap, string::String, vec::Vec};
+use core::mem;
 
-use libboard_artiq::pl::csr;
+use ksupport::kernel::DmaRecorder;
+use libboard_artiq::{drtio_routing::RoutingTable,
+                     drtioaux_proto::{Packet, PayloadStatus, MASTER_PAYLOAD_MAX_SIZE},
+                     pl::csr};
 use libcortex_a9::cache::dcci_slice;
+use routing::{Router, Sliceable};
+use subkernel::Manager as KernelManager;
 
 const ALIGNMENT: usize = 64;
 
@@ -12,16 +18,20 @@ enum ManagerState {
 }
 
 pub struct RtioStatus {
+    pub source: u8,
     pub id: u32,
     pub error: u8,
     pub channel: u32,
     pub timestamp: u64,
 }
 
+#[derive(Debug)]
 pub enum Error {
     IdNotFound,
     PlaybackInProgress,
     EntryNotComplete,
+    MasterDmaFound,
+    UploadFail,
 }
 
 #[derive(Debug)]
@@ -29,13 +39,228 @@ struct Entry {
     trace: Vec<u8>,
     padding_len: usize,
     complete: bool,
+    duration: i64, // relevant for local DMA
+}
+
+impl Entry {
+    pub fn from_vec(data: Vec<u8>, duration: i64) -> Entry {
+        let mut entry = Entry {
+            trace: data,
+            padding_len: 0,
+            complete: true,
+            duration: duration,
+        };
+        entry.realign();
+        entry
+    }
+
+    pub fn id(&self) -> u32 {
+        self.trace[self.padding_len..].as_ptr() as u32
+    }
+
+    pub fn realign(&mut self) {
+        self.trace.push(0);
+        let data_len = self.trace.len();
+
+        self.trace.reserve(ALIGNMENT - 1);
+        let padding = ALIGNMENT - self.trace.as_ptr() as usize % ALIGNMENT;
+        let padding = if padding == ALIGNMENT { 0 } else { padding };
+        for _ in 0..padding {
+            // Vec guarantees that this will not reallocate
+            self.trace.push(0)
+        }
+        for i in 1..data_len + 1 {
+            self.trace[data_len + padding - i] = self.trace[data_len - i]
+        }
+        self.complete = true;
+        self.padding_len = padding;
+
+        dcci_slice(&self.trace);
+    }
+}
+
+#[derive(Debug)]
+enum RemoteTraceState {
+    Unsent,
+    Sending(usize),
+    Ready,
+    Running(usize),
+}
+
+#[derive(Debug)]
+struct RemoteTraces {
+    remote_traces: BTreeMap<u8, Sliceable>,
+    state: RemoteTraceState,
+}
+
+impl RemoteTraces {
+    pub fn new(traces: BTreeMap<u8, Sliceable>) -> RemoteTraces {
+        RemoteTraces {
+            remote_traces: traces,
+            state: RemoteTraceState::Unsent,
+        }
+    }
+
+    // on subkernel request
+    pub fn upload_traces(
+        &mut self,
+        id: u32,
+        router: &mut Router,
+        rank: u8,
+        self_destination: u8,
+        routing_table: &RoutingTable,
+    ) -> usize {
+        let len = self.remote_traces.len();
+        if len > 0 {
+            self.state = RemoteTraceState::Sending(self.remote_traces.len());
+            for (dest, trace) in self.remote_traces.iter_mut() {
+                // queue up the first packet for all destinations, rest will be sent after first ACK
+                let mut data_slice: [u8; MASTER_PAYLOAD_MAX_SIZE] = [0; MASTER_PAYLOAD_MAX_SIZE];
+                let meta = trace.get_slice_master(&mut data_slice);
+                router.route(
+                    Packet::DmaAddTraceRequest {
+                        source: self_destination,
+                        destination: *dest,
+                        id: id,
+                        status: meta.status,
+                        length: meta.len,
+                        trace: data_slice,
+                    },
+                    routing_table,
+                    rank,
+                    self_destination,
+                );
+            }
+        }
+        len
+    }
+
+    // on incoming Packet::DmaAddTraceReply
+    pub fn ack_upload(
+        &mut self,
+        kernel_manager: &mut KernelManager,
+        source: u8,
+        id: u32,
+        succeeded: bool,
+        router: &mut Router,
+        rank: u8,
+        self_destination: u8,
+        routing_table: &RoutingTable,
+    ) {
+        if let RemoteTraceState::Sending(count) = self.state {
+            if let Some(trace) = self.remote_traces.get_mut(&source) {
+                if trace.at_end() {
+                    if count - 1 == 0 {
+                        self.state = RemoteTraceState::Ready;
+                        if let Some((id, timestamp)) = kernel_manager.ddma_remote_uploaded(succeeded) {
+                            self.playback(id, timestamp, router, rank, self_destination, routing_table);
+                        }
+                    } else {
+                        self.state = RemoteTraceState::Sending(count - 1);
+                    }
+                } else {
+                    // send next slice
+                    let mut data_slice: [u8; MASTER_PAYLOAD_MAX_SIZE] = [0; MASTER_PAYLOAD_MAX_SIZE];
+                    let meta = trace.get_slice_master(&mut data_slice);
+                    router.route(
+                        Packet::DmaAddTraceRequest {
+                            source: self_destination,
+                            destination: meta.destination,
+                            id: id,
+                            status: meta.status,
+                            length: meta.len,
+                            trace: data_slice,
+                        },
+                        routing_table,
+                        rank,
+                        self_destination,
+                    );
+                }
+            }
+        }
+    }
+
+    // on subkernel request
+    pub fn playback(
+        &mut self,
+        id: u32,
+        timestamp: u64,
+        router: &mut Router,
+        rank: u8,
+        self_destination: u8,
+        routing_table: &RoutingTable,
+    ) {
+        // route all the playback requests
+        // remote traces (local trace runs on core1 unlike mainline firmware)
+        self.state = RemoteTraceState::Running(self.remote_traces.len());
+        for (dest, _) in self.remote_traces.iter() {
+            router.route(
+                Packet::DmaPlaybackRequest {
+                    source: self_destination,
+                    destination: *dest,
+                    id: id,
+                    timestamp: timestamp,
+                },
+                routing_table,
+                rank,
+                self_destination,
+            );
+            // response will be ignored (succeeded = false handled by the main thread)
+        }
+    }
+
+    // on incoming Packet::DmaPlaybackDone
+    pub fn remote_finished(&mut self, kernel_manager: &mut KernelManager, error: u8, channel: u32, timestamp: u64) {
+        if let RemoteTraceState::Running(count) = self.state {
+            if error != 0 || count - 1 == 0 {
+                // notify the kernel about a DDMA error or finish
+                kernel_manager.ddma_finished(error, channel, timestamp);
+                self.state = RemoteTraceState::Ready;
+                // further messages will be ignored (if there was an error)
+            } else {
+                // no error and not the last one awaited
+                self.state = RemoteTraceState::Running(count - 1);
+            }
+        }
+    }
+
+    pub fn erase(
+        &mut self,
+        id: u32,
+        router: &mut Router,
+        rank: u8,
+        self_destination: u8,
+        routing_table: &RoutingTable,
+    ) {
+        for (dest, _) in self.remote_traces.iter() {
+            router.route(
+                Packet::DmaRemoveTraceRequest {
+                    source: self_destination,
+                    destination: *dest,
+                    id: id,
+                },
+                routing_table,
+                rank,
+                self_destination,
+            );
+            // response will be ignored as this object will stop existing too
+        }
+    }
+
+    pub fn has_remote_traces(&self) -> bool {
+        self.remote_traces.len() > 0
+    }
 }
 
 #[derive(Debug)]
 pub struct Manager {
-    entries: BTreeMap<u32, Entry>,
+    entries: BTreeMap<(u8, u32), Entry>,
     state: ManagerState,
-    currentid: u32,
+    current_id: u32,
+    current_source: u8,
+
+    remote_entries: BTreeMap<u32, RemoteTraces>,
+    name_map: BTreeMap<String, u32>,
 }
 
 impl Manager {
@@ -45,79 +270,238 @@ impl Manager {
         unsafe { while csr::rtio_dma::enable_read() != 0 {} }
         Manager {
             entries: BTreeMap::new(),
-            currentid: 0,
+            current_id: 0,
+            current_source: 0,
             state: ManagerState::Idle,
+            remote_entries: BTreeMap::new(),
+            name_map: BTreeMap::new(),
         }
     }
 
-    pub fn add(&mut self, id: u32, last: bool, trace: &[u8], trace_len: usize) -> Result<(), Error> {
+    pub fn add(
-        let entry = match self.entries.get_mut(&id) {
+        &mut self,
+        source: u8,
+        id: u32,
+        status: PayloadStatus,
+        trace: &[u8],
+        trace_len: usize,
+    ) -> Result<(), Error> {
+        let entry = match self.entries.get_mut(&(source, id)) {
             Some(entry) => {
-                if entry.complete {
+                if entry.complete || status.is_first() {
                     // replace entry
-                    self.entries.remove(&id);
+                    self.entries.remove(&(source, id));
                     self.entries.insert(
-                        id,
+                        (source, id),
                         Entry {
                             trace: Vec::new(),
                             padding_len: 0,
                             complete: false,
+                            duration: 0,
                         },
                     );
-                    self.entries.get_mut(&id).unwrap()
+                    self.entries.get_mut(&(source, id)).unwrap()
                 } else {
                     entry
                 }
             }
             None => {
                 self.entries.insert(
-                    id,
+                    (source, id),
                     Entry {
                         trace: Vec::new(),
                         padding_len: 0,
                         complete: false,
+                        duration: 0,
                     },
                 );
-                self.entries.get_mut(&id).unwrap()
+                self.entries.get_mut(&(source, id)).unwrap()
             }
         };
         entry.trace.extend(&trace[0..trace_len]);
 
-        if last {
+        if status.is_last() {
-            entry.trace.push(0);
+            entry.realign();
-            let data_len = entry.trace.len();
-
-            // Realign.
-            entry.trace.reserve(ALIGNMENT - 1);
-            let padding = ALIGNMENT - entry.trace.as_ptr() as usize % ALIGNMENT;
-            let padding = if padding == ALIGNMENT { 0 } else { padding };
-            for _ in 0..padding {
-                // Vec guarantees that this will not reallocate
-                entry.trace.push(0)
-            }
-            for i in 1..data_len + 1 {
-                entry.trace[data_len + padding - i] = entry.trace[data_len - i]
-            }
-            entry.complete = true;
-            entry.padding_len = padding;
-            dcci_slice(&entry.trace);
         }
         Ok(())
     }
 
-    pub fn erase(&mut self, id: u32) -> Result<(), Error> {
+    // api for DRTIO
-        match self.entries.remove(&id) {
+    pub fn erase(&mut self, source: u8, id: u32) -> Result<(), Error> {
+        match self.entries.remove(&(source, id)) {
             Some(_) => Ok(()),
             None => Err(Error::IdNotFound),
         }
     }
 
-    pub fn playback(&mut self, id: u32, timestamp: u64) -> Result<(), Error> {
+    // API for subkernel
+    pub fn erase_name(
+        &mut self,
+        name: &str,
+        router: &mut Router,
+        rank: u8,
+        self_destination: u8,
+        routing_table: &RoutingTable,
+    ) {
+        if let Some(id) = self.name_map.get(name) {
+            if let Some(traces) = self.remote_entries.get_mut(&id) {
+                traces.erase(*id, router, rank, self_destination, routing_table);
+                self.remote_entries.remove(&id);
+            }
+            self.entries.remove(&(self_destination, *id));
+            self.name_map.remove(name);
+        }
+    }
+
+    pub fn remote_finished(
+        &mut self,
+        kernel_manager: &mut KernelManager,
+        id: u32,
+        error: u8,
+        channel: u32,
+        timestamp: u64,
+    ) {
+        if let Some(entry) = self.remote_entries.get_mut(&id) {
+            entry.remote_finished(kernel_manager, error, channel, timestamp);
+        }
+    }
+
+    pub fn ack_upload(
+        &mut self,
+        kernel_manager: &mut KernelManager,
+        source: u8,
+        id: u32,
+        succeeded: bool,
+        router: &mut Router,
+        rank: u8,
+        self_destination: u8,
+        routing_table: &RoutingTable,
+    ) {
+        if let Some(entry) = self.remote_entries.get_mut(&id) {
+            entry.ack_upload(
+                kernel_manager,
+                source,
+                id,
+                succeeded,
+                router,
+                rank,
+                self_destination,
+                routing_table,
+            );
+        }
+    }
+
+    // API for subkernel
+    pub fn upload_traces(
+        &mut self,
+        id: u32,
+        router: &mut Router,
+        rank: u8,
+        self_destination: u8,
+        routing_table: &RoutingTable,
+    ) -> Result<usize, Error> {
+        let remote_traces = self.remote_entries.get_mut(&id);
+        let mut len = 0;
+        if let Some(traces) = remote_traces {
+            len = traces.upload_traces(id, router, rank, self_destination, routing_table);
+        }
+        Ok(len)
+    }
+
+    // API for subkernel
+    pub fn playback_remote(
+        &mut self,
+        id: u32,
+        timestamp: u64,
+        router: &mut Router,
+        rank: u8,
+        self_destination: u8,
+        routing_table: &RoutingTable,
+    ) -> Result<(), Error> {
+        if let Some(traces) = self.remote_entries.get_mut(&id) {
+            traces.playback(id, timestamp, router, rank, self_destination, routing_table);
+            Ok(())
+        } else {
+            Err(Error::IdNotFound)
+        }
+    }
+
+    // API for subkernel
+    pub fn cleanup(&mut self, router: &mut Router, rank: u8, self_destination: u8, routing_table: &RoutingTable) {
+        // after subkernel ends, remove all self-generated traces
+        for (_, id) in self.name_map.iter_mut() {
+            if let Some(traces) = self.remote_entries.get_mut(&id) {
+                traces.erase(*id, router, rank, self_destination, routing_table);
+                self.remote_entries.remove(&id);
+            }
+            self.entries.remove(&(self_destination, *id));
+        }
+        self.name_map.clear();
+    }
+
+    // API for subkernel
+    pub fn retrieve(&self, self_destination: u8, name: &String) -> Option<(i32, i64, bool)> {
+        let id = self.name_map.get(name)?;
+        let duration = self.entries.get(&(self_destination, *id))?.duration;
+        let uses_ddma = self.has_remote_traces(*id);
+        Some((*id as i32, duration, uses_ddma))
+    }
+
+    pub fn has_remote_traces(&self, id: u32) -> bool {
+        match self.remote_entries.get(&id) {
+            Some(traces) => traces.has_remote_traces(),
+            _ => false,
+        }
+    }
+
+    pub fn put_record(&mut self, mut recorder: DmaRecorder, self_destination: u8) -> Result<u32, Error> {
+        let mut remote_traces: BTreeMap<u8, Sliceable> = BTreeMap::new();
+
+        let mut local_trace: Vec<u8> = Vec::new();
+        // analyze each entry and put in proper buckets, as the kernel core
+        // sends whole chunks, to limit comms/kernel CPU communication,
+        // and as only comms core has access to varios DMA buffers.
+        let mut ptr = 0;
+        recorder.buffer.push(0);
+        while recorder.buffer[ptr] != 0 {
+            // ptr + 3 = tgt >> 24 (destination)
+            let len = recorder.buffer[ptr] as usize;
+            let destination = recorder.buffer[ptr + 3];
+            if destination == 0 {
+                return Err(Error::MasterDmaFound);
+            } else if destination == self_destination {
+                local_trace.extend(&recorder.buffer[ptr..ptr + len]);
+            } else {
+                if let Some(remote_trace) = remote_traces.get_mut(&destination) {
+                    remote_trace.extend(&recorder.buffer[ptr..ptr + len]);
+                } else {
+                    remote_traces.insert(
+                        destination,
+                        Sliceable::new(destination, recorder.buffer[ptr..ptr + len].to_vec()),
+                    );
+                }
+            }
+            // and jump to the next event
+            ptr += len;
+        }
+        let local_entry = Entry::from_vec(local_trace, recorder.duration);
+
+        let id = local_entry.id();
+        self.entries.insert((self_destination, id), local_entry);
+        self.remote_entries.insert(id, RemoteTraces::new(remote_traces));
+        let mut name = String::new();
+        mem::swap(&mut recorder.name, &mut name);
+        self.name_map.insert(name, id);
+
+        Ok(id)
+    }
+
+    pub fn playback(&mut self, source: u8, id: u32, timestamp: u64) -> Result<(), Error> {
         if self.state != ManagerState::Idle {
             return Err(Error::PlaybackInProgress);
         }
 
-        let entry = match self.entries.get(&id) {
+        let entry = match self.entries.get(&(source, id)) {
             Some(entry) => entry,
             None => {
                 return Err(Error::IdNotFound);
@@ -130,7 +514,8 @@ impl Manager {
         assert!(ptr as u32 % 64 == 0);
 
         self.state = ManagerState::Playback;
-        self.currentid = id;
+        self.current_id = id;
+        self.current_source = source;
 
         unsafe {
             csr::rtio_dma::base_address_write(ptr as u32);
@@ -162,7 +547,8 @@ impl Manager {
                     csr::rtio_dma::error_write(1);
                 }
                 return Some(RtioStatus {
-                    id: self.currentid,
+                    source: self.current_source,
+                    id: self.current_id,
                     error: error,
                     channel: channel,
                     timestamp: timestamp,

src/satman/src/repeater.rs

@@ -6,6 +6,7 @@ use libboard_artiq::{drtio_routing, drtioaux};
 #[cfg(has_drtio_routing)]
 use libboard_zynq::time::Milliseconds;
 use libboard_zynq::timer::GlobalTimer;
+use routing::Router;
 
 #[cfg(has_drtio_routing)]
 fn rep_link_rx_up(repno: u8) -> bool {
@@ -53,7 +54,14 @@ impl Repeater {
         self.state == RepeaterState::Up
     }
 
-    pub fn service(&mut self, routing_table: &drtio_routing::RoutingTable, rank: u8, timer: &mut GlobalTimer) {
+    pub fn service(
+        &mut self,
+        routing_table: &drtio_routing::RoutingTable,
+        rank: u8,
+        destination: u8,
+        router: &mut Router,
+        timer: &mut GlobalTimer,
+    ) {
         self.process_local_errors();
 
         match self.state {
@@ -79,6 +87,10 @@ impl Repeater {
                 if rep_link_rx_up(self.repno) {
                     if let Ok(Some(drtioaux::Packet::EchoReply)) = drtioaux::recv(self.auxno) {
                         info!("[REP#{}] remote replied after {} packets", self.repno, ping_count);
+                        let max_time = timer.get_time() + Milliseconds(200);
+                        while timer.get_time() < max_time {
+                            let _ = drtioaux::recv(self.auxno);
+                        }
                         self.state = RepeaterState::Up;
                         if let Err(e) = self.sync_tsc(timer) {
                             error!("[REP#{}] failed to sync TSC ({:?})", self.repno, e);
@@ -111,7 +123,7 @@ impl Repeater {
                 }
             }
             RepeaterState::Up => {
-                self.process_unsolicited_aux();
+                self.process_unsolicited_aux(routing_table, rank, destination, router);
                 if !rep_link_rx_up(self.repno) {
                     info!("[REP#{}] link is down", self.repno);
                     self.state = RepeaterState::Down;
@@ -126,9 +138,15 @@ impl Repeater {
         }
     }
 
-    fn process_unsolicited_aux(&self) {
+    fn process_unsolicited_aux(
+        &self,
+        routing_table: &drtio_routing::RoutingTable,
+        rank: u8,
+        destination: u8,
+        router: &mut Router,
+    ) {
         match drtioaux::recv(self.auxno) {
-            Ok(Some(packet)) => warn!("[REP#{}] unsolicited aux packet: {:?}", self.repno, packet),
+            Ok(Some(packet)) => router.route(packet, routing_table, rank, destination),
             Ok(None) => (),
             Err(_) => warn!("[REP#{}] aux packet error", self.repno),
         }
@@ -173,7 +191,7 @@ impl Repeater {
         }
     }
 
-    fn recv_aux_timeout(&self, timeout: u32, timer: &mut GlobalTimer) -> Result<drtioaux::Packet, drtioaux::Error> {
+    fn recv_aux_timeout(&self, timeout: u32, timer: &mut GlobalTimer) -> Result<drtioaux::Packet, drtioaux::Error<!>> {
         let max_time = timer.get_time() + Milliseconds(timeout.into());
         loop {
             if !rep_link_rx_up(self.repno) {
@@ -190,17 +208,48 @@ impl Repeater {
         }
     }
 
-    pub fn aux_forward(&self, request: &drtioaux::Packet, timer: &mut GlobalTimer) -> Result<(), drtioaux::Error> {
+    pub fn aux_forward(
-        if self.state != RepeaterState::Up {
+        &self,
-            return Err(drtioaux::Error::LinkDown);
+        request: &drtioaux::Packet,
+        router: &mut Router,
+        routing_table: &drtio_routing::RoutingTable,
+        rank: u8,
+        self_destination: u8,
+        timer: &mut GlobalTimer,
+    ) -> Result<(), drtioaux::Error<!>> {
+        self.aux_send(request)?;
+        loop {
+            let reply = self.recv_aux_timeout(200, timer)?;
+            match reply {
+                // async/locally requested packets to be consumed or routed
+                // these may come while a packet would be forwarded
+                drtioaux::Packet::DmaPlaybackStatus { .. }
+                | drtioaux::Packet::SubkernelFinished { .. }
+                | drtioaux::Packet::SubkernelMessage { .. }
+                | drtioaux::Packet::SubkernelMessageAck { .. }
+                | drtioaux::Packet::SubkernelLoadRunReply { .. }
+                | drtioaux::Packet::SubkernelException { .. }
+                | drtioaux::Packet::DmaAddTraceReply { .. }
+                | drtioaux::Packet::DmaPlaybackReply { .. } => {
+                    router.route(reply, routing_table, rank, self_destination);
+                }
+                _ => {
+                    drtioaux::send(0, &reply).unwrap();
+                    break;
+                }
+            }
         }
-        drtioaux::send(self.auxno, request).unwrap();
-        let reply = self.recv_aux_timeout(200, timer)?;
-        drtioaux::send(0, &reply).unwrap();
         Ok(())
     }
 
-    pub fn sync_tsc(&self, timer: &mut GlobalTimer) -> Result<(), drtioaux::Error> {
+    pub fn aux_send(&self, request: &drtioaux::Packet) -> Result<(), drtioaux::Error<!>> {
+        if self.state != RepeaterState::Up {
+            return Err(drtioaux::Error::LinkDown);
+        }
+        drtioaux::send(self.auxno, request)
+    }
+
+    pub fn sync_tsc(&self, timer: &mut GlobalTimer) -> Result<(), drtioaux::Error<!>> {
         if self.state != RepeaterState::Up {
             return Ok(());
         }
@@ -226,7 +275,7 @@ impl Repeater {
         destination: u8,
         hops: &[u8; drtio_routing::MAX_HOPS],
         timer: &mut GlobalTimer,
-    ) -> Result<(), drtioaux::Error> {
+    ) -> Result<(), drtioaux::Error<!>> {
         if self.state != RepeaterState::Up {
             return Ok(());
         }
@@ -250,14 +299,14 @@ impl Repeater {
         &self,
         routing_table: &drtio_routing::RoutingTable,
         timer: &mut GlobalTimer,
-    ) -> Result<(), drtioaux::Error> {
+    ) -> Result<(), drtioaux::Error<!>> {
         for i in 0..drtio_routing::DEST_COUNT {
             self.set_path(i as u8, &routing_table.0[i], timer)?;
         }
         Ok(())
     }
 
-    pub fn set_rank(&self, rank: u8, timer: &mut GlobalTimer) -> Result<(), drtioaux::Error> {
+    pub fn set_rank(&self, rank: u8, timer: &mut GlobalTimer) -> Result<(), drtioaux::Error<!>> {
         if self.state != RepeaterState::Up {
             return Ok(());
         }
@@ -269,7 +318,7 @@ impl Repeater {
         Ok(())
     }
 
-    pub fn rtio_reset(&self, timer: &mut GlobalTimer) -> Result<(), drtioaux::Error> {
+    pub fn rtio_reset(&self, timer: &mut GlobalTimer) -> Result<(), drtioaux::Error<!>> {
         let repno = self.repno as usize;
         unsafe {
             (csr::DRTIOREP[repno].reset_write)(1);
@@ -302,13 +351,21 @@ impl Repeater {
         Repeater::default()
     }
 
-    pub fn service(&self, _routing_table: &drtio_routing::RoutingTable, _rank: u8, _timer: &mut GlobalTimer) {}
+    pub fn service(
+        &self,
+        _routing_table: &drtio_routing::RoutingTable,
+        _rank: u8,
+        _destination: u8,
+        _router: &mut Router,
+        _timer: &mut GlobalTimer,
+    ) {
+    }
 
-    pub fn sync_tsc(&self, _timer: &mut GlobalTimer) -> Result<(), drtioaux::Error> {
+    pub fn sync_tsc(&self, _timer: &mut GlobalTimer) -> Result<(), drtioaux::Error<!>> {
         Ok(())
     }
 
-    pub fn rtio_reset(&self, _timer: &mut GlobalTimer) -> Result<(), drtioaux::Error> {
+    pub fn rtio_reset(&self, _timer: &mut GlobalTimer) -> Result<(), drtioaux::Error<!>> {
         Ok(())
     }
 }

src/satman/src/routing.rs

@@ -0,0 +1,173 @@
+use alloc::{collections::vec_deque::VecDeque, vec::Vec};
+use core::cmp::min;
+
+#[cfg(has_drtio_routing)]
+use libboard_artiq::pl::csr;
+use libboard_artiq::{drtio_routing, drtioaux,
+                     drtioaux_proto::{PayloadStatus, MASTER_PAYLOAD_MAX_SIZE}};
+
+pub struct SliceMeta {
+    pub destination: u8,
+    pub len: u16,
+    pub status: PayloadStatus,
+}
+
+/* represents data that has to be sent to Master */
+#[derive(Debug)]
+pub struct Sliceable {
+    it: usize,
+    data: Vec<u8>,
+    destination: u8,
+}
+
+macro_rules! get_slice_fn {
+    ($name:tt, $size:expr) => {
+        pub fn $name(&mut self, data_slice: &mut [u8; $size]) -> SliceMeta {
+            let first = self.it == 0;
+            let len = min($size, self.data.len() - self.it);
+            let last = self.it + len == self.data.len();
+            let status = PayloadStatus::from_status(first, last);
+
+            data_slice[..len].clone_from_slice(&self.data[self.it..self.it + len]);
+            self.it += len;
+
+            SliceMeta {
+                destination: self.destination,
+                len: len as u16,
+                status: status,
+            }
+        }
+    };
+}
+
+impl Sliceable {
+    pub fn new(destination: u8, data: Vec<u8>) -> Sliceable {
+        Sliceable {
+            it: 0,
+            data: data,
+            destination: destination,
+        }
+    }
+
+    pub fn at_end(&self) -> bool {
+        self.it == self.data.len()
+    }
+
+    pub fn extend(&mut self, data: &[u8]) {
+        self.data.extend(data);
+    }
+
+    get_slice_fn!(get_slice_master, MASTER_PAYLOAD_MAX_SIZE);
+}
+
+// Packets from downstream (further satellites) are received and routed appropriately.
+// they're passed as soon as possible downstream (within the subtree), or sent upstream,
+// which is notified about pending packets.
+// for rank 1 (connected to master) satellites, these packets are passed as an answer to DestinationStatusRequest;
+// for higher ranks, after getting a notification, it will transact with downstream to get the pending packets.
+
+// forward! macro is not deprecated, as routable packets are only these that can originate
+// from both master and satellite, e.g. DDMA and Subkernel.
+
+pub struct Router {
+    upstream_queue: VecDeque<drtioaux::Packet>,
+    local_queue: VecDeque<drtioaux::Packet>,
+    #[cfg(has_drtio_routing)]
+    downstream_queue: VecDeque<(usize, drtioaux::Packet)>,
+}
+
+impl Router {
+    pub fn new() -> Router {
+        Router {
+            upstream_queue: VecDeque::new(),
+            local_queue: VecDeque::new(),
+            #[cfg(has_drtio_routing)]
+            downstream_queue: VecDeque::new(),
+        }
+    }
+
+    // Called by local sources (DDMA, kernel) and by repeaters on receiving async data;
+    // messages are always buffered for both upstream and downstream
+    pub fn route(
+        &mut self,
+        packet: drtioaux::Packet,
+        _routing_table: &drtio_routing::RoutingTable,
+        _rank: u8,
+        self_destination: u8,
+    ) {
+        let destination = packet.routable_destination();
+        #[cfg(has_drtio_routing)]
+        {
+            if let Some(destination) = destination {
+                let hop = _routing_table.0[destination as usize][_rank as usize] as usize;
+                if destination == self_destination {
+                    self.local_queue.push_back(packet);
+                } else if hop > 0 && hop < csr::DRTIOREP.len() {
+                    let repno = (hop - 1) as usize;
+                    self.downstream_queue.push_back((repno, packet));
+                } else {
+                    self.upstream_queue.push_back(packet);
+                }
+            } else {
+                error!("Received an unroutable packet: {:?}", packet);
+            }
+        }
+        #[cfg(not(has_drtio_routing))]
+        {
+            if destination == Some(self_destination) {
+                self.local_queue.push_back(packet);
+            } else {
+                self.upstream_queue.push_back(packet);
+            }
+        }
+    }
+
+    // Sends a packet to a required destination, routing if necessary
+    pub fn send(
+        &mut self,
+        packet: drtioaux::Packet,
+        _routing_table: &drtio_routing::RoutingTable,
+        _rank: u8,
+        _destination: u8,
+    ) -> Result<(), drtioaux::Error<!>> {
+        #[cfg(has_drtio_routing)]
+        {
+            let destination = packet.routable_destination();
+            if let Some(destination) = destination {
+                let hop = _routing_table.0[destination as usize][_rank as usize] as usize;
+                if destination == 0 {
+                    // response is needed immediately if master required it
+                    drtioaux::send(0, &packet)?;
+                } else if !(hop > 0 && hop < csr::DRTIOREP.len()) {
+                    // higher rank can wait
+                    self.upstream_queue.push_back(packet);
+                } else {
+                    let repno = (hop - 1) as usize;
+                    // transaction will occur at closest possible opportunity
+                    self.downstream_queue.push_back((repno, packet));
+                }
+                Ok(())
+            } else {
+                // packet not supported in routing, fallback - sent directly
+                drtioaux::send(0, &packet)
+            }
+        }
+        #[cfg(not(has_drtio_routing))]
+        {
+            drtioaux::send(0, &packet)
+        }
+    }
+
+    pub fn get_upstream_packet(&mut self) -> Option<drtioaux::Packet> {
+        self.upstream_queue.pop_front()
+    }
+
+    #[cfg(has_drtio_routing)]
+    pub fn get_downstream_packet(&mut self) -> Option<(usize, drtioaux::Packet)> {
+        self.downstream_queue.pop_front()
+    }
+
+    pub fn get_local_packet(&mut self) -> Option<drtioaux::Packet> {
+        self.local_queue.pop_front()
+    }
+}