rename `build` derivation to `board-package-set`

rtio_clocking: update mmcm setting to use HIGH bandwidth

.gitignore

@@ -3,3 +3,11 @@ examples/*.elf
 __pycache__
 
 build
+
+src/libboard_artiq/Cargo.toml
+src/libc/Cargo.toml
+src/libdyld/Cargo.toml
+src/libio/Cargo.toml
+src/libksupport/Cargo.toml
+src/runtime/Cargo.toml
+src/satman/Cargo.toml

README.md

@@ -4,14 +4,13 @@ ARTIQ on Zynq
 How to use
 ----------
 
-1. Install the ARTIQ version that corresponds to the artiq-zynq version are targeting.
-2. Select the latest successful build on [Hydra](https://nixbld.m-labs.hk/) for the targeted artiq-zynq version, or use AFWS to obtain firmware binaries.
-3. Search for the job named ``<board>-<variant>-sd`` (for example: ``zc706-nist_clock-sd`` or ``zc706-nist_qc2-sd``).
-4. Download the ``boot.bin`` "binary distribution" and place it at the root of a FAT-formatted SD card.
-5. Optionally, create a ``config.txt`` configuration file at the root of the SD card containing ``key=value`` pairs on each line. Use the ``ip``, ``ip6`` and ``mac`` keys to respectively set the IPv4, IPv6 and MAC address of the board. Configuring an IPv6 address is entirely optional. If these keys are not found, the firmware will use default values that may or may not be compatible with your network.
-6. Insert the SD card into the board and set up the board to boot from the SD card. For the ZC706, this is achieved by placing the large DIP switch SW11 in the 00110 position.
-7. Power up the board. After the firmware starts successfully, it should respond to ping at its IP addresses, and boot messages can be observed from its UART at 115200bps.
-8. Create and use an ARTIQ device database as usual, but set ``"target": "cortexa9"`` in the arguments of the core device.
+1. Install the ARTIQ version that corresponds to the artiq-zynq version you are targeting.
+2. To obtain firmware binaries, select the latest successful build on [Hydra](https://nixbld.m-labs.hk/) for the targeted artiq-zynq version, or use AFWS. If using Hydra, search for the job named ``<board>-<variant>-sd`` (for example: ``zc706-nist_clock-sd`` or ``zc706-nist_qc2-sd``).
+3. Place the ``boot.bin`` file, obtained from Hydra's "binary distribution" download link or from AFWS, at the root of a FAT-formatted SD card.
+4. Optionally, create a ``config.txt`` configuration file at the root of the SD card containing ``key=value`` pairs on each line. Use the ``ip``, ``ip6`` and ``mac`` keys to respectively set the IPv4, IPv6 and MAC address of the board. Configuring an IPv6 address is entirely optional. If these keys are not found, the firmware will use default values that may or may not be compatible with your network.
+5. Insert the SD card into the board and set up the board to boot from the SD card. For the ZC706, this is achieved by placing the large DIP switch SW11 in the 00110 position.
+6. Power up the board. After the firmware starts successfully, it should respond to ping at its IP addresses, and boot messages can be observed from its UART at 115200bps.
+7. Create and use an ARTIQ device database as usual, but set ``"target": "cortexa9"`` in the arguments of the core device.
 
 Configuration
 -------------
@@ -59,12 +58,11 @@ Notes:
 - When calling make, you need to specify both the variant and firmware type.
 - Firmware type must be either ``runtime`` for DRTIO-less or DRTIO master variants, or ``satman`` for DRTIO satellite.
 - If the board is connected to the local machine, use the ``local_run.sh`` script.
-- To update ``zynq-rs``, update the cargo files as per usual for Rust projects, but also keep ``flake.lock`` in sync.
 
 License
 -------
 
-Copyright (C) 2019-2022 M-Labs Limited.
+Copyright (C) 2019-2024 M-Labs Limited.
 
 ARTIQ is free software: you can redistribute it and/or modify
 it under the terms of the GNU Lesser General Public License as published by

flake.lock

@@ -11,11 +11,11 @@
         "src-pythonparser": "src-pythonparser"
       },
       "locked": {
-        "lastModified": 1680398257,
-        "narHash": "sha256-ANODm+Xjx/OYHkmQpybDuIPXiu22IkoQ0wdI0ReQsNk=",
-        "ref": "refs/heads/master",
-        "rev": "7ba06bfe61cad4ad41f478188917cac6571a6875",
-        "revCount": 8323,
+        "lastModified": 1714013217,
+        "narHash": "sha256-P0pVHTSgAkTYWFPjddcPhufe5oH8mjJAsxG8x8mo4NA=",
+        "ref": "master",
+        "rev": "7204feae1f504e4a1dcd54b95c59e8f36c62b701",
+        "revCount": 8776,
         "type": "git",
         "url": "https://github.com/m-labs/artiq.git"
       },
@@ -37,11 +37,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1664405593,
-        "narHash": "sha256-yP441NerlLGig7n+9xHsx8yCtZ+Ggd0VqfBSzc20E04=",
+        "lastModified": 1707216368,
+        "narHash": "sha256-ZXoqzG2QsVsybALLYXs473avXcyKSZNh2kIgcPo60XQ=",
         "owner": "m-labs",
         "repo": "artiq-comtools",
-        "rev": "15ddac62813ef623a076ccf982b3bc63d314e651",
+        "rev": "e5d0204490bccc07ef9141b0d7c405ab01cb8273",
         "type": "github"
       },
       "original": {
@@ -51,12 +51,15 @@
       }
     },
     "flake-utils": {
+      "inputs": {
+        "systems": "systems"
+      },
       "locked": {
-        "lastModified": 1659877975,
-        "narHash": "sha256-zllb8aq3YO3h8B/U0/J1WBgAL8EX5yWf5pMj3G0NAmc=",
+        "lastModified": 1694529238,
+        "narHash": "sha256-zsNZZGTGnMOf9YpHKJqMSsa0dXbfmxeoJ7xHlrt+xmY=",
         "owner": "numtide",
         "repo": "flake-utils",
-        "rev": "c0e246b9b83f637f4681389ecabcb2681b4f3af0",
+        "rev": "ff7b65b44d01cf9ba6a71320833626af21126384",
         "type": "github"
       },
       "original": {
@@ -68,11 +71,11 @@
     "mozilla-overlay": {
       "flake": false,
       "locked": {
-        "lastModified": 1672878308,
-        "narHash": "sha256-0+fl6PHokhtSV+w58z2QD2rTf8QhcOGsT9o4LwHHZHE=",
+        "lastModified": 1704373101,
+        "narHash": "sha256-+gi59LRWRQmwROrmE1E2b3mtocwueCQqZ60CwLG+gbg=",
         "owner": "mozilla",
         "repo": "nixpkgs-mozilla",
-        "rev": "d38863db88e100866b3e494a651ee4962b762fcc",
+        "rev": "9b11a87c0cc54e308fa83aac5b4ee1816d5418a2",
         "type": "github"
       },
       "original": {
@@ -84,11 +87,11 @@
     "mozilla-overlay_2": {
       "flake": false,
       "locked": {
-        "lastModified": 1677493379,
-        "narHash": "sha256-A1gO8zlWLv3+tZ3cGVB1WYvvoN9pbFyv0xIJHcTsckw=",
+        "lastModified": 1704373101,
+        "narHash": "sha256-+gi59LRWRQmwROrmE1E2b3mtocwueCQqZ60CwLG+gbg=",
         "owner": "mozilla",
         "repo": "nixpkgs-mozilla",
-        "rev": "78e723925daf5c9e8d0a1837ec27059e61649cb6",
+        "rev": "9b11a87c0cc54e308fa83aac5b4ee1816d5418a2",
         "type": "github"
       },
       "original": {
@@ -100,11 +103,11 @@
     "mozilla-overlay_3": {
       "flake": false,
       "locked": {
-        "lastModified": 1664789696,
-        "narHash": "sha256-UGWJHQShiwLCr4/DysMVFrYdYYHcOqAOVsWNUu+l6YU=",
+        "lastModified": 1695805681,
+        "narHash": "sha256-1ElPLD8eFfnuIk0G52HGGpRtQZ4QPCjChRlEOfkZ5ro=",
         "owner": "mozilla",
         "repo": "nixpkgs-mozilla",
-        "rev": "80627b282705101e7b38e19ca6e8df105031b072",
+        "rev": "6eabade97bc28d707a8b9d82ad13ef143836736e",
         "type": "github"
       },
       "original": {
@@ -115,16 +118,16 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1673345971,
-        "narHash": "sha256-4DfFcKLRfVUTyuGrGNNmw37IeIZSoku9tgTVmu/iD98=",
+        "lastModified": 1711668574,
+        "narHash": "sha256-u1dfs0ASQIEr1icTVrsKwg2xToIpn7ZXxW3RHfHxshg=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "54644f409ab471e87014bb305eac8c50190bcf48",
+        "rev": "219951b495fc2eac67b1456824cc1ec1fd2ee659",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "nixos-22.11",
+        "ref": "nixos-23.11",
         "repo": "nixpkgs",
         "type": "github"
       }
@@ -144,11 +147,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1673433867,
-        "narHash": "sha256-a7Oq35YoDzPtISbqAsaT+2/v15HZ7G1q0ukXmKWdb7Q=",
+        "lastModified": 1708937641,
+        "narHash": "sha256-Hkb9VYFzFgkYxfbh4kYcDSn7DbMUYehoQDeTALrxo2Q=",
         "owner": "m-labs",
         "repo": "sipyco",
-        "rev": "38f8f4185d7db6b68bd7f71546da9077b1e2561c",
+        "rev": "4a28b311ce0069454b4e8fe1e6049db11b9f1296",
         "type": "github"
       },
       "original": {
@@ -160,11 +163,11 @@
     "src-migen": {
       "flake": false,
       "locked": {
-        "lastModified": 1673433200,
-        "narHash": "sha256-ribBG06gsucz5oBS+O6aL8s2oJjx+qfl+vXmspts8gg=",
+        "lastModified": 1702942348,
+        "narHash": "sha256-gKIfHZxsv+jcgDFRW9mPqmwqbZXuRvXefkZcSFjOGHw=",
         "owner": "m-labs",
         "repo": "migen",
-        "rev": "f3e9145c9825514a1b4225378936569da4df8e12",
+        "rev": "50934ad10a87ade47219b796535978b9bdf24023",
         "type": "github"
       },
       "original": {
@@ -176,11 +179,11 @@
     "src-misoc": {
       "flake": false,
       "locked": {
-        "lastModified": 1671158014,
-        "narHash": "sha256-50w0K2E2ympYrG1Tte/HVbsp4FS2U+yohqZByXTOo4I=",
+        "lastModified": 1699352904,
+        "narHash": "sha256-SglyTmXOPv8jJOjwAjJrj/WhAkItQfUbvKfUqrynwRg=",
         "ref": "refs/heads/master",
-        "rev": "26f039f9f6931a20a04ccd0f0a5402f67f553916",
-        "revCount": 2436,
+        "rev": "a53859f2167c31ab5225b6c09f30cf05527b94f4",
+        "revCount": 2452,
         "submodules": true,
         "type": "git",
         "url": "https://github.com/m-labs/misoc.git"
@@ -207,6 +210,21 @@
         "type": "github"
       }
     },
+    "systems": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    },
     "zynq-rs": {
       "inputs": {
         "mozilla-overlay": "mozilla-overlay_3",
@@ -216,11 +234,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1669819016,
-        "narHash": "sha256-WvNMUekL4Elc55RdqX8XP43QPnBrK8Rbd0bsoI61E5U=",
-        "ref": "refs/heads/master",
-        "rev": "67dbb5932fa8ff5f143983476f741f945871d286",
-        "revCount": 624,
+        "lastModified": 1711358419,
+        "narHash": "sha256-LzetYaLsnov9pHVWSCTSowXUAXkxemAa61CIMY98xsc=",
+        "ref": "master",
+        "rev": "195a21fe78e4dde1fd705cb2899ab5d2763ae037",
+        "revCount": 643,
         "type": "git",
         "url": "https://git.m-labs.hk/m-labs/zynq-rs"
       },

flake.nix

@@ -12,6 +12,7 @@
     zynqpkgs = zynq-rs.packages.x86_64-linux;
     artiqpkgs = artiq.packages.x86_64-linux;
 
+    rust = zynq-rs.rust;
     rustPlatform = zynq-rs.rustPlatform;
 
     fastnumbers = pkgs.python3Packages.buildPythonPackage rec {
@@ -49,7 +50,7 @@
       nativeBuildInputs = with pkgs.python3Packages; [ pbr ];
       propagatedBuildInputs = with pkgs.python3Packages; [ future fastnumbers ];
 
-      checkInputs = with pkgs.python3Packages; [ pytest pytest-flake8 ];
+      checkInputs = with pkgs.python3Packages; [ pytest ];
       checkPhase = "pytest";
       doCheck = false;
 
@@ -73,15 +74,17 @@
 
       propagatedBuildInputs = with pkgs.python3Packages; [ setuptools click numpy toolz jinja2 ramda artiqpkgs.migen artiqpkgs.misoc ];
 
-      checkInputs = with pkgs.python3Packages; [ pytest pytest-timeout pytest-flake8 ];
-      checkPhase = "pytest";
+      checkInputs = with pkgs.python3Packages; [ pytest-runner pytestCheckHook pytest-timeout ];
 
       # migen/misoc version checks are broken with pyproject for some reason
       postPatch = ''
+        sed -i "1,4d" pyproject.toml
         substituteInPlace pyproject.toml \
           --replace '"migen@git+https://github.com/m-labs/migen",' ""
         substituteInPlace pyproject.toml \
           --replace '"misoc@git+https://github.com/m-labs/misoc.git",' ""
+        # pytest-flake8 is broken with recent flake8. Re-enable after fix.
+        substituteInPlace setup.cfg --replace '--flake8' ""
       '';
     };
     binutils = { platform, target, zlib }: pkgs.stdenv.mkDerivation rec {
@@ -110,34 +113,33 @@
       "nist_clock_satellite" "nist_qc2_satellite" "acpki_nist_clock_satellite" "acpki_nist_qc2_satellite" 
       "nist_clock_satellite_100mhz" "nist_qc2_satellite_100mhz" "acpki_nist_clock_satellite_100mhz" "acpki_nist_qc2_satellite_100mhz"
     ];
-    build = { target, variant, json ? null }: let
+    board-package-set = { target, variant, json ? null }: let
       szl = zynqpkgs."${target}-szl";
       fsbl = zynqpkgs."${target}-fsbl";
       fwtype = if builtins.elem variant sat_variants then "satman" else "runtime";
 
       firmware = rustPlatform.buildRustPackage rec {
-
         name = "firmware";
         src = ./src;
         cargoLock = { 
           lockFile = src/Cargo.lock;
           outputHashes = {
-            "libasync-0.0.0" = "sha256-WvNMUekL4Elc55RdqX8XP43QPnBrK8Rbd0bsoI61E5U=";
-          };
+              "tar-no-std-0.1.8" = "sha256-xm17108v4smXOqxdLvHl9CxTCJslmeogjm4Y87IXFuM=";
+            };
         };
 
         nativeBuildInputs = [
           pkgs.gnumake
-          (pkgs.python3.withPackages(ps: (with artiqpkgs; [ ps.jsonschema migen migen-axi misoc artiq ])))
-          artiqpkgs.artiq
+          (pkgs.python3.withPackages(ps: [ ps.jsonschema artiqpkgs.migen migen-axi artiqpkgs.misoc artiqpkgs.artiq ]))
           zynqpkgs.cargo-xbuild
           pkgs.llvmPackages_9.llvm
           pkgs.llvmPackages_9.clang-unwrapped
         ];
         buildPhase = ''
-          export XARGO_RUST_SRC="${rustPlatform.rust.rustc}/lib/rustlib/src/rust/library"
+          export XARGO_RUST_SRC="${rust}/lib/rustlib/src/rust/library"
           export CLANG_EXTRA_INCLUDE_DIR="${pkgs.llvmPackages_9.clang-unwrapped.lib}/lib/clang/9.0.1/include"
           export CARGO_HOME=$(mktemp -d cargo-home.XXX)
+          export ZYNQ_RS=${zynq-rs}
           make TARGET=${target} GWARGS="${if json == null then "-V ${variant}" else json}" ${fwtype}
         '';
 
@@ -151,12 +153,12 @@
 
         doCheck = false;
         dontFixup = true;
+        auditable = false;
       };
       gateware = pkgs.runCommand "${target}-${variant}-gateware"
         {
           nativeBuildInputs = [ 
-            (pkgs.python3.withPackages(ps: (with artiqpkgs; [ ps.jsonschema migen migen-axi misoc artiq ])))
-            artiqpkgs.artiq
+            (pkgs.python3.withPackages(ps: [ ps.jsonschema artiqpkgs.migen migen-axi artiqpkgs.misoc artiqpkgs.artiq ]))
             artiqpkgs.vivado
           ];
         }
@@ -240,8 +242,7 @@
       name = "gateware-sim";
       
       nativeBuildInputs = [ 
-        (pkgs.python3.withPackages(ps: (with artiqpkgs; [ migen migen-axi artiq ]))) 
-        artiqpkgs.artiq
+        (pkgs.python3.withPackages(ps: [ artiqpkgs.migen migen-axi artiqpkgs.artiq ]))
       ];
 
       phases = [ "buildPhase" ];
@@ -253,25 +254,26 @@
         '';
     };
 
-    fmt-check = pkgs.stdenv.mkDerivation {
+    fmt-check = pkgs.stdenvNoCC.mkDerivation {
       name = "fmt-check";
 
-      nativeBuildInputs = [
-        rustPlatform.rust.cargo
-      ];
+      src = ./src;
 
-      phases = [ "buildPhase" ];
+      nativeBuildInputs = [ rust pkgs.gnumake ];
+
+      phases = [ "unpackPhase" "buildPhase" ];
 
       buildPhase =
         ''
-        cd ${self}/src
+        export ZYNQ_RS=${zynq-rs}
+        make manifests
         cargo fmt -- --check
         touch $out
         '';
     };
 
     # for hitl-tests
-    zc706-nist_qc2 = (build { target = "zc706"; variant = "nist_qc2"; });
+    zc706-nist_qc2 = (board-package-set { target = "zc706"; variant = "nist_qc2"; });
     zc706-hitl-tests = pkgs.stdenv.mkDerivation {
       name = "zc706-hitl-tests";
 
@@ -338,33 +340,36 @@
       {
         inherit fastnumbers artiq-netboot ramda migen-axi binutils-arm;
       } //
-      (build { target = "zc706"; variant = "nist_clock"; }) //
-      (build { target = "zc706"; variant = "nist_clock_master"; }) //
-      (build { target = "zc706"; variant = "nist_clock_satellite"; }) //
-      (build { target = "zc706"; variant = "nist_clock_satellite_100mhz"; }) //
-      (build { target = "zc706"; variant = "nist_qc2"; }) //
-      (build { target = "zc706"; variant = "nist_qc2_master"; }) //
-      (build { target = "zc706"; variant = "nist_qc2_satellite"; }) //
-      (build { target = "zc706"; variant = "nist_qc2_satellite_100mhz"; }) //
-      (build { target = "zc706"; variant = "acpki_nist_clock"; }) //
-      (build { target = "zc706"; variant = "acpki_nist_clock_master"; }) //
-      (build { target = "zc706"; variant = "acpki_nist_clock_satellite"; }) //
-      (build { target = "zc706"; variant = "acpki_nist_clock_satellite_100mhz"; }) //
-      (build { target = "zc706"; variant = "acpki_nist_qc2"; }) //
-      (build { target = "zc706"; variant = "acpki_nist_qc2_master"; }) //
-      (build { target = "zc706"; variant = "acpki_nist_qc2_satellite"; }) //
-      (build { target = "zc706"; variant = "acpki_nist_qc2_satellite_100mhz"; }) //
-      (build { target = "kasli_soc"; variant = "demo"; json = ./demo.json; }) //
-      (build { target = "kasli_soc"; variant = "master"; json = ./kasli-soc-master.json; }) //
-      (build { target = "kasli_soc"; variant = "satellite"; json = ./kasli-soc-satellite.json; });
+      (board-package-set { target = "zc706"; variant = "nist_clock"; }) //
+      (board-package-set { target = "zc706"; variant = "nist_clock_master"; }) //
+      (board-package-set { target = "zc706"; variant = "nist_clock_master_100mhz"; }) //
+      (board-package-set { target = "zc706"; variant = "nist_clock_satellite"; }) //
+      (board-package-set { target = "zc706"; variant = "nist_clock_satellite_100mhz"; }) //
+      (board-package-set { target = "zc706"; variant = "nist_qc2"; }) //
+      (board-package-set { target = "zc706"; variant = "nist_qc2_master"; }) //
+      (board-package-set { target = "zc706"; variant = "nist_qc2_master_100mhz"; }) //
+      (board-package-set { target = "zc706"; variant = "nist_qc2_satellite"; }) //
+      (board-package-set { target = "zc706"; variant = "nist_qc2_satellite_100mhz"; }) //
+      (board-package-set { target = "zc706"; variant = "acpki_nist_clock"; }) //
+      (board-package-set { target = "zc706"; variant = "acpki_nist_clock_master"; }) //
+      (board-package-set { target = "zc706"; variant = "acpki_nist_clock_master_100mhz"; }) //
+      (board-package-set { target = "zc706"; variant = "acpki_nist_clock_satellite"; }) //
+      (board-package-set { target = "zc706"; variant = "acpki_nist_clock_satellite_100mhz"; }) //
+      (board-package-set { target = "zc706"; variant = "acpki_nist_qc2"; }) //
+      (board-package-set { target = "zc706"; variant = "acpki_nist_qc2_master"; }) //
+      (board-package-set { target = "zc706"; variant = "acpki_nist_qc2_master_100mhz"; }) //
+      (board-package-set { target = "zc706"; variant = "acpki_nist_qc2_satellite"; }) //
+      (board-package-set { target = "zc706"; variant = "acpki_nist_qc2_satellite_100mhz"; }) //
+      (board-package-set { target = "kasli_soc"; variant = "demo"; json = ./demo.json; }) //
+      (board-package-set { target = "kasli_soc"; variant = "master"; json = ./kasli-soc-master.json; }) //
+      (board-package-set { target = "kasli_soc"; variant = "satellite"; json = ./kasli-soc-satellite.json; });
 
     hydraJobs = packages.x86_64-linux // { inherit zc706-hitl-tests; inherit gateware-sim; inherit fmt-check; };
 
     devShell.x86_64-linux = pkgs.mkShell {
       name = "artiq-zynq-dev-shell";
       buildInputs = with pkgs; [
-        rustPlatform.rust.rustc
-        rustPlatform.rust.cargo
+        rust
         llvmPackages_9.llvm
         llvmPackages_9.clang-unwrapped
         gnumake
@@ -378,13 +383,14 @@
         artiqpkgs.vivado
         binutils-arm
       ];
-      XARGO_RUST_SRC = "${rustPlatform.rust.rustc}/lib/rustlib/src/rust/library";
+      XARGO_RUST_SRC = "${rust}/lib/rustlib/src/rust/library";
       CLANG_EXTRA_INCLUDE_DIR = "${pkgs.llvmPackages_9.clang-unwrapped.lib}/lib/clang/9.0.1/include";
+      ZYNQ_RS = "${zynq-rs}";
       OPENOCD_ZYNQ = "${zynq-rs}/openocd";
       SZL = "${zynqpkgs.szl}";
     };
 
-    makeArtiqZynqPackage = build;
+    makeArtiqZynqPackage = board-package-set;
 
   };
 }

src/Cargo.lock

@@ -2,6 +2,12 @@
 # It is not intended for manual editing.
 version = 3
 
+[[package]]
+name = "arrayvec"
+version = "0.7.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711"
+
 [[package]]
 name = "async-recursion"
 version = "0.3.2"
@@ -218,10 +224,37 @@ dependencies = [
  "libsupport_zynq",
 ]
 
+[[package]]
+name = "ksupport"
+version = "0.1.0"
+dependencies = [
+ "build_zynq",
+ "byteorder",
+ "core_io",
+ "cslice",
+ "dwarf",
+ "dyld",
+ "io",
+ "libasync",
+ "libboard_artiq",
+ "libboard_zynq",
+ "libc",
+ "libconfig",
+ "libcortex_a9",
+ "libm",
+ "libregister",
+ "libsupport_zynq",
+ "log",
+ "log_buffer",
+ "nb 0.1.3",
+ "unwind",
+ "vcell",
+ "void",
+]
+
 [[package]]
 name = "libasync"
 version = "0.0.0"
-source = "git+https://git.m-labs.hk/M-Labs/zynq-rs.git#67dbb5932fa8ff5f143983476f741f945871d286"
 dependencies = [
  "embedded-hal",
  "libcortex_a9",
@@ -244,6 +277,7 @@ dependencies = [
  "libconfig",
  "libcortex_a9",
  "libregister",
+ "libsupport_zynq",
  "log",
  "log_buffer",
  "nb 1.0.0",
@@ -253,7 +287,6 @@ dependencies = [
 [[package]]
 name = "libboard_zynq"
 version = "0.0.0"
-source = "git+https://git.m-labs.hk/M-Labs/zynq-rs.git#67dbb5932fa8ff5f143983476f741f945871d286"
 dependencies = [
  "bit_field",
  "embedded-hal",
@@ -278,7 +311,6 @@ dependencies = [
 [[package]]
 name = "libconfig"
 version = "0.1.0"
-source = "git+https://git.m-labs.hk/M-Labs/zynq-rs.git#67dbb5932fa8ff5f143983476f741f945871d286"
 dependencies = [
  "core_io",
  "fatfs",
@@ -289,7 +321,6 @@ dependencies = [
 [[package]]
 name = "libcortex_a9"
 version = "0.0.0"
-source = "git+https://git.m-labs.hk/M-Labs/zynq-rs.git#67dbb5932fa8ff5f143983476f741f945871d286"
 dependencies = [
  "bit_field",
  "libregister",
@@ -305,7 +336,6 @@ checksum = "348108ab3fba42ec82ff6e9564fc4ca0247bdccdc68dd8af9764bbc79c3c8ffb"
 [[package]]
 name = "libregister"
 version = "0.0.0"
-source = "git+https://git.m-labs.hk/M-Labs/zynq-rs.git#67dbb5932fa8ff5f143983476f741f945871d286"
 dependencies = [
  "bit_field",
  "vcell",
@@ -315,7 +345,6 @@ dependencies = [
 [[package]]
 name = "libsupport_zynq"
 version = "0.0.0"
-source = "git+https://git.m-labs.hk/M-Labs/zynq-rs.git#67dbb5932fa8ff5f143983476f741f945871d286"
 dependencies = [
  "cc",
  "compiler_builtins",
@@ -438,20 +467,20 @@ dependencies = [
  "embedded-hal",
  "futures",
  "io",
+ "ksupport",
  "libasync",
  "libboard_artiq",
  "libboard_zynq",
  "libc",
  "libconfig",
  "libcortex_a9",
- "libm",
  "libregister",
  "libsupport_zynq",
  "log",
  "log_buffer",
- "nb 0.1.3",
  "num-derive",
  "num-traits",
+ "tar-no-std",
  "unwind",
  "vcell",
  "void",
@@ -471,7 +500,11 @@ name = "satman"
 version = "0.0.0"
 dependencies = [
  "build_zynq",
+ "core_io",
+ "cslice",
  "embedded-hal",
+ "io",
+ "ksupport",
  "libasync",
  "libboard_artiq",
  "libboard_zynq",
@@ -512,6 +545,16 @@ dependencies = [
  "unicode-ident",
 ]
 
+[[package]]
+name = "tar-no-std"
+version = "0.1.8"
+source = "git+https://git.m-labs.hk/M-Labs/tar-no-std?rev=2ab6dc5#2ab6dc58e5249c59c4eb03eaf3a119bcdd678d32"
+dependencies = [
+ "arrayvec",
+ "bitflags",
+ "log",
+]
+
 [[package]]
 name = "unicode-ident"
 version = "1.0.5"

src/Cargo.toml

@@ -5,6 +5,7 @@ members = [
   "libdwarf",
   "libio",
   "libunwind",
+  "libksupport",
   "runtime",
   "satman"
 ]

src/Makefile

@@ -7,13 +7,20 @@ runtime: ../build/runtime.bin
 
 satman: ../build/satman.bin
 
-.PHONY: all runtime_target satman_target
+.PHONY: all manifests
+
+manifests = libboard_artiq/Cargo.toml libc/Cargo.toml libdyld/Cargo.toml libio/Cargo.toml libksupport/Cargo.toml runtime/Cargo.toml satman/Cargo.toml
+
+$(manifests): %.toml: %.toml.tpl
+	sed s+@@ZYNQ_RS@@+$(ZYNQ_RS)+g $< > $@
+
+manifests: $(manifests)
 
 ../build/pl.rs ../build/rustc-cfg ../build/mem.rs: gateware/* 
 	mkdir -p ../build
 	python gateware/$(TARGET).py -r ../build/pl.rs -c ../build/rustc-cfg -m ../build/mem.rs $(GWARGS)
 
-../build/firmware/armv7-none-eabihf/release/runtime: ../build/pl.rs ../build/rustc-cfg ../build/mem.rs $(shell find . -type f -print)
+../build/firmware/armv7-none-eabihf/release/runtime: ../build/pl.rs ../build/rustc-cfg ../build/mem.rs $(manifests) $(shell find . -type f -not -name Cargo.toml -print)
 	cd runtime && \
 	XBUILD_SYSROOT_PATH=`pwd`/../../build/sysroot \
 	cargo xbuild --release \
@@ -23,7 +30,7 @@ satman: ../build/satman.bin
 ../build/runtime.bin: ../build/firmware/armv7-none-eabihf/release/runtime
 	llvm-objcopy -O binary ../build/firmware/armv7-none-eabihf/release/runtime ../build/runtime.bin
 
-../build/firmware/armv7-none-eabihf/release/satman: ../build/pl.rs ../build/rustc-cfg ../build/mem.rs $(shell find . -type f -print)
+../build/firmware/armv7-none-eabihf/release/satman: ../build/pl.rs ../build/rustc-cfg ../build/mem.rs $(manifests) $(shell find . -type f -not -name Cargo.toml -print)
 	cd satman && \
 	XBUILD_SYSROOT_PATH=`pwd`/../../build/sysroot \
 	cargo xbuild --release \
@@ -31,4 +38,4 @@ satman: ../build/satman.bin
 		--no-default-features --features=target_$(TARGET)
 
 ../build/satman.bin: ../build/firmware/armv7-none-eabihf/release/satman
-	llvm-objcopy -O binary ../build/firmware/armv7-none-eabihf/release/satman ../build/satman.bin
+	llvm-objcopy -O binary ../build/firmware/armv7-none-eabihf/release/satman ../build/satman.bin

src/gateware/config.py

@@ -0,0 +1,16 @@
+from misoc.integration import cpu_interface
+
+def write_csr_file(soc, filename):
+    with open(filename, "w") as f:
+        f.write(cpu_interface.get_csr_rust(
+            soc.get_csr_regions(), soc.get_csr_groups(), soc.get_constants()))
+
+def write_mem_file(soc, filename):
+    with open(filename, "w") as f:
+        f.write(cpu_interface.get_mem_rust(
+            soc.get_memory_regions(), soc.get_memory_groups(), None))
+
+def write_rustc_cfg_file(soc, filename):
+    with open(filename, "w") as f:
+        f.write(cpu_interface.get_rust_cfg(
+            soc.get_csr_regions(), soc.get_constants()))

src/gateware/ddmtd.py

@@ -0,0 +1,119 @@
+from migen import *
+from migen.genlib.cdc import PulseSynchronizer, MultiReg
+from misoc.interconnect.csr import *
+
+
+class DDMTDSampler(Module):
+    def __init__(self, cd_ref, main_clk_se):
+        self.ref_beating = Signal()
+        self.main_beating = Signal()
+
+        # # #
+
+        ref_clk = Signal()
+        self.specials +=[
+            # ISERDESE2 can only be driven from fabric via IDELAYE2 (see UG471)
+            Instance("IDELAYE2",
+                    p_DELAY_SRC="DATAIN",
+                    p_HIGH_PERFORMANCE_MODE="TRUE",
+                    p_REFCLK_FREQUENCY=208.3,   # REFCLK frequency from IDELAYCTRL
+                    p_IDELAY_VALUE=0,
+
+                    i_DATAIN=cd_ref.clk,
+
+                    o_DATAOUT=ref_clk
+            ),
+            Instance("ISERDESE2",
+                    p_IOBDELAY="IFD",   # use DDLY as input
+                    p_DATA_RATE="SDR",
+                    p_DATA_WIDTH=2,     # min is 2
+                    p_NUM_CE=1,
+
+                    i_DDLY=ref_clk,
+                    i_CE1=1,
+                    i_CLK=ClockSignal("helper"),
+                    i_CLKDIV=ClockSignal("helper"),
+
+                    o_Q1=self.ref_beating
+            ),
+            Instance("ISERDESE2",
+                    p_DATA_RATE="SDR",
+                    p_DATA_WIDTH=2,     # min is 2
+                    p_NUM_CE=1,
+
+                    i_D=main_clk_se,
+                    i_CE1=1,
+                    i_CLK=ClockSignal("helper"),
+                    i_CLKDIV=ClockSignal("helper"),
+
+                    o_Q1=self.main_beating,
+            ),
+        ]
+
+
+class DDMTDDeglitcherMedianEdge(Module):
+    def __init__(self, counter, input_signal, stable_0_period=100, stable_1_period=100):
+        self.tag = Signal(len(counter))
+        self.detect = Signal()
+
+        stable_0_counter = Signal(reset=stable_0_period - 1, max=stable_0_period)
+        stable_1_counter = Signal(reset=stable_1_period - 1, max=stable_1_period)
+
+        # # #
+
+        # Based on CERN's median edge deglitcher FSM
+        # https://white-rabbit.web.cern.ch/documents/Precise_time_and_frequency_transfer_in_a_White_Rabbit_network.pdf (p.72)
+        fsm = ClockDomainsRenamer("helper")(FSM(reset_state="WAIT_STABLE_0"))
+        self.submodules += fsm
+
+        fsm.act("WAIT_STABLE_0",
+            If(stable_0_counter != 0,
+                NextValue(stable_0_counter, stable_0_counter - 1)
+            ).Else(
+                NextValue(stable_0_counter, stable_0_period - 1),
+                NextState("WAIT_EDGE")
+            ),
+            If(input_signal,
+                NextValue(stable_0_counter, stable_0_period - 1)
+            ),
+        )
+        fsm.act("WAIT_EDGE",
+            If(input_signal,
+                NextValue(self.tag, counter),
+                NextState("GOT_EDGE")
+            )
+        )
+        fsm.act("GOT_EDGE",
+            If(stable_1_counter != 0,
+                NextValue(stable_1_counter, stable_1_counter - 1)
+            ).Else(
+                NextValue(stable_1_counter, stable_1_period - 1),
+                self.detect.eq(1),
+                NextState("WAIT_STABLE_0")
+            ),
+            If(~input_signal,
+                NextValue(self.tag, self.tag + 1),
+                NextValue(stable_1_counter, stable_1_period - 1)
+            ),
+        )
+        
+
+class DDMTD(Module):
+    def __init__(self, counter, input_signal):
+
+        # in helper clock domain
+        self.h_tag = Signal(len(counter))
+        self.h_tag_update = Signal()
+
+        # # #
+
+        deglitcher = DDMTDDeglitcherMedianEdge(counter, input_signal)
+        self.submodules += deglitcher
+
+        self.sync.helper += [
+            self.h_tag_update.eq(0),
+            If(deglitcher.detect,
+                self.h_tag_update.eq(1),
+                self.h_tag.eq(deglitcher.tag)
+               )
+        ]

src/gateware/drtio_aux_controller.py

@@ -1,12 +1,12 @@
 """Auxiliary controller, common to satellite and master"""
 
-from artiq.gateware.drtio.aux_controller import Transmitter, Receiver
+from artiq.gateware.drtio.aux_controller import (max_packet, aux_buffer_count,
+    Transmitter, Receiver)
 from migen.fhdl.simplify import FullMemoryWE
 from misoc.interconnect.csr import *
 from migen_axi.interconnect.sram import SRAM
 from migen_axi.interconnect import axi
 
-max_packet = 1024
 
 class _DRTIOAuxControllerBase(Module):
     def __init__(self, link_layer):
@@ -27,12 +27,12 @@ class DRTIOAuxControllerAxi(_DRTIOAuxControllerBase):
         tx_sdram_if = SRAM(self.transmitter.mem, read_only=False)
         rx_sdram_if = SRAM(self.receiver.mem, read_only=True)
         aw_decoder = axi.AddressDecoder(self.bus.aw,
-            [(lambda a: a[log2_int(max_packet)] == 0, tx_sdram_if.bus.aw),
-             (lambda a: a[log2_int(max_packet)] == 1, rx_sdram_if.bus.aw)],
+            [(lambda a: a[log2_int(max_packet*aux_buffer_count)] == 0, tx_sdram_if.bus.aw),
+             (lambda a: a[log2_int(max_packet*aux_buffer_count)] == 1, rx_sdram_if.bus.aw)],
             register=True)
         ar_decoder = axi.AddressDecoder(self.bus.ar,
-            [(lambda a: a[log2_int(max_packet)] == 0, tx_sdram_if.bus.ar),
-             (lambda a: a[log2_int(max_packet)] == 1, rx_sdram_if.bus.ar)],
+            [(lambda a: a[log2_int(max_packet*aux_buffer_count)] == 0, tx_sdram_if.bus.ar),
+             (lambda a: a[log2_int(max_packet*aux_buffer_count)] == 1, rx_sdram_if.bus.ar)],
             register=True)
         # unlike wb, axi address decoder only connects ar/aw lanes,
         # the rest must also be connected!
@@ -82,4 +82,4 @@ class DRTIOAuxControllerBare(_DRTIOAuxControllerBase):
         return self.receiver.mem.get_port(write_capable=False)
 
     def get_mem_size(self):
-        return max_packet
+        return max_packet*aux_buffer_count

src/gateware/kasli_soc.py

@@ -10,13 +10,13 @@ from migen.genlib.cdc import MultiReg
 from migen_axi.integration.soc_core import SoCCore
 from migen_axi.platforms import kasli_soc
 from misoc.interconnect.csr import *
-from misoc.integration import cpu_interface
+from misoc.cores import virtual_leds
 
 from artiq.coredevice import jsondesc
 from artiq.gateware import rtio, eem_7series
 from artiq.gateware.rtio.xilinx_clocking import fix_serdes_timing_path
 from artiq.gateware.rtio.phy import ttl_simple
-from artiq.gateware.drtio.transceiver import gtx_7series
+from artiq.gateware.drtio.transceiver import gtx_7series, eem_serdes
 from artiq.gateware.drtio.siphaser import SiPhaser7Series
 from artiq.gateware.drtio.rx_synchronizer import XilinxRXSynchronizer
 from artiq.gateware.drtio import *
@@ -26,7 +26,8 @@ import analyzer
 import acpki
 import drtio_aux_controller
 import zynq_clocking
-
+import wrpll
+from config import write_csr_file, write_mem_file, write_rustc_cfg_file
 
 eem_iostandard_dict = {
      0: "LVDS_25",
@@ -61,27 +62,54 @@ class SMAClkinForward(Module):
         ]
 
 
-class GTP125BootstrapClock(Module):
-    def __init__(self, platform):
+class GTPBootstrapClock(Module):
+    def __init__(self, platform, freq=125e6):
         self.clock_domains.cd_bootstrap = ClockDomain(reset_less=True)
         self.cd_bootstrap.clk.attr.add("keep")
 
         bootstrap_125 = platform.request("clk125_gtp")
         bootstrap_se = Signal()
+        clk_out = Signal()
         platform.add_period_constraint(bootstrap_125.p, 8.0)
         self.specials += [
             Instance("IBUFDS_GTE2",
-                p_CLKSWING_CFG="0b11",
                 i_CEB=0,
-                i_I=bootstrap_125.p, i_IB=bootstrap_125.n, o_O=bootstrap_se),
-            Instance("BUFG", i_I=bootstrap_se, o_O=self.cd_bootstrap.clk)
+                i_I=bootstrap_125.p, i_IB=bootstrap_125.n, 
+                o_O=bootstrap_se,
+                p_CLKCM_CFG="TRUE",
+                p_CLKRCV_TRST="TRUE",
+                p_CLKSWING_CFG=3),
+            Instance("BUFG", i_I=bootstrap_se, o_O=clk_out)
         ]
+        if freq == 125e6:
+            self.comb += self.cd_bootstrap.clk.eq(clk_out)
+        elif freq == 100e6:
+            pll_fb = Signal()
+            pll_out = Signal()
+            self.specials += [
+                Instance("PLLE2_BASE",
+                    p_CLKIN1_PERIOD=8.0,
+                    i_CLKIN1=clk_out,
+                    i_CLKFBIN=pll_fb,
+                    o_CLKFBOUT=pll_fb,
+
+                    # VCO @ 1GHz
+                    p_CLKFBOUT_MULT=8, p_DIVCLK_DIVIDE=1,
+
+                    # 100MHz for bootstrap
+                    p_CLKOUT1_DIVIDE=10, p_CLKOUT1_PHASE=0.0, o_CLKOUT1=pll_out,
+                ),
+                Instance("BUFG", i_I=pll_out, o_O=self.cd_bootstrap.clk)
+            ]
+        else:
+            raise ValueError("Bootstrap frequency must be 100 or 125MHz")
 
 
 class GenericStandalone(SoCCore):
     def __init__(self, description, acpki=False):
         self.acpki = acpki
-        self.rustc_cfg = dict()
+        clk_freq = description["rtio_frequency"]
+        with_wrpll = description["enable_wrpll"]
 
         platform = kasli_soc.Platform()
         platform.toolchain.bitstream_commands.extend([
@@ -92,29 +120,47 @@ class GenericStandalone(SoCCore):
             ident = "acpki_" + ident
         SoCCore.__init__(self, platform=platform, csr_data_width=32, ident=ident, ps_cd_sys=False)
 
-        self.submodules += SMAClkinForward(self.platform)
-
-        self.rustc_cfg["has_si5324"] = None
-        self.rustc_cfg["si5324_soft_reset"] = None
-
+        self.config["HW_REV"] = description["hw_rev"]
         clk_synth = platform.request("cdr_clk_clean_fabric")
         clk_synth_se = Signal()
+        clk_synth_se_buf = Signal()
         platform.add_period_constraint(clk_synth.p, 8.0)
 
-        self.specials += Instance("IBUFGDS",
+        self.specials += [ 
+            Instance("IBUFGDS",
                 p_DIFF_TERM="TRUE", p_IBUF_LOW_PWR="FALSE",
-                i_I=clk_synth.p, i_IB=clk_synth.n, o_O=clk_synth_se)
+                i_I=clk_synth.p, i_IB=clk_synth.n, o_O=clk_synth_se
+            ),
+            Instance("BUFG", i_I=clk_synth_se, o_O=clk_synth_se_buf),
+        ]
         fix_serdes_timing_path(platform)
-        self.submodules.bootstrap = GTP125BootstrapClock(self.platform)
+        self.submodules.bootstrap = GTPBootstrapClock(self.platform, clk_freq)
+        self.config["CLOCK_FREQUENCY"] = int(clk_freq)
 
-
-        self.submodules.sys_crg = zynq_clocking.SYSCRG(self.platform, self.ps7, clk_synth_se)
+        self.submodules.sys_crg = zynq_clocking.SYSCRG(self.platform, self.ps7, clk_synth_se_buf)
         platform.add_false_path_constraints(
             self.bootstrap.cd_bootstrap.clk, self.sys_crg.cd_sys.clk)
         self.csr_devices.append("sys_crg")
         self.crg = self.ps7 # HACK for eem_7series to find the clock
         self.crg.cd_sys = self.sys_crg.cd_sys
 
+        if with_wrpll:
+            self.submodules.wrpll_refclk = wrpll.SMAFrequencyMultiplier(platform.request("sma_clkin"))
+            self.submodules.wrpll = wrpll.WRPLL(
+                platform=self.platform,
+                cd_ref=self.wrpll_refclk.cd_ref,
+                main_clk_se=clk_synth_se)
+            self.csr_devices.append("wrpll_refclk")
+            self.csr_devices.append("wrpll")
+            self.comb += self.ps7.core.core0.nfiq.eq(self.wrpll.ev.irq)
+            self.config["HAS_SI549"] = None
+            self.config["WRPLL_REF_CLK"] = "SMA_CLKIN"
+        else:
+            self.submodules += SMAClkinForward(self.platform)
+            self.config["HAS_SI5324"] = None
+            self.config["SI5324_SOFT_RESET"] = None
+
+
         self.rtio_channels = []
         has_grabber = any(peripheral["type"] == "grabber" for peripheral in description["peripherals"])
         if has_grabber:
@@ -130,18 +176,20 @@ class GenericStandalone(SoCCore):
         self.rtio_channels.append(rtio.LogChannel())
 
         self.submodules.rtio_tsc = rtio.TSC(glbl_fine_ts_width=3)
-        self.submodules.rtio_core = rtio.Core(self.rtio_tsc, self.rtio_channels)
+        self.submodules.rtio_core = rtio.Core(
+            self.rtio_tsc, self.rtio_channels, lane_count=description["sed_lanes"]
+        )
         self.csr_devices.append("rtio_core")
 
         if self.acpki:
-            self.rustc_cfg["ki_impl"] = "acp"
+            self.config["KI_IMPL"] = "acp"
             self.submodules.rtio = acpki.KernelInitiator(self.rtio_tsc,
                                                          bus=self.ps7.s_axi_acp,
                                                          user=self.ps7.s_axi_acp_user,
                                                          evento=self.ps7.event.o)
             self.csr_devices.append("rtio")
         else:
-            self.rustc_cfg["ki_impl"] = "csr"
+            self.config["KI_IMPL"] = "csr"
             self.submodules.rtio = rtio.KernelInitiator(self.rtio_tsc, now64=True)
             self.csr_devices.append("rtio")
 
@@ -161,7 +209,7 @@ class GenericStandalone(SoCCore):
         self.csr_devices.append("rtio_analyzer")
 
         if has_grabber:
-            self.rustc_cfg["has_grabber"] = None
+            self.config["HAS_GRABBER"] = None
             self.add_csr_group("grabber", self.grabber_csr_group)
             for grabber in self.grabber_csr_group:
                 self.platform.add_false_path_constraints(
@@ -171,9 +219,10 @@ class GenericStandalone(SoCCore):
 class GenericMaster(SoCCore):
     def __init__(self, description, acpki=False):
         clk_freq = description["rtio_frequency"]
+        with_wrpll = description["enable_wrpll"]
 
+        has_drtio_over_eem = any(peripheral["type"] == "shuttler" for peripheral in description["peripherals"])
         self.acpki = acpki
-        self.rustc_cfg = dict()
 
         platform = kasli_soc.Platform()
         platform.toolchain.bitstream_commands.extend([
@@ -184,26 +233,31 @@ class GenericMaster(SoCCore):
             ident = "acpki_" + ident
         SoCCore.__init__(self, platform=platform, csr_data_width=32, ident=ident, ps_cd_sys=False)
 
-        self.submodules += SMAClkinForward(self.platform)
+        self.config["HW_REV"] = description["hw_rev"]
 
         data_pads = [platform.request("sfp", i) for i in range(4)]
 
-        self.submodules.drtio_transceiver = gtx_7series.GTX(
+        self.submodules.gt_drtio = gtx_7series.GTX(
             clock_pads=platform.request("clk_gtp"),
             pads=data_pads,
             clk_freq=clk_freq)
-        self.csr_devices.append("drtio_transceiver")
+        self.csr_devices.append("gt_drtio")
+        self.config["CLOCK_FREQUENCY"] = int(clk_freq)
 
         txout_buf = Signal()
-        gtx0 = self.drtio_transceiver.gtxs[0]
+        gtx0 = self.gt_drtio.gtxs[0]
         self.specials += Instance("BUFG", i_I=gtx0.txoutclk, o_O=txout_buf)
 
-        self.submodules.bootstrap = GTP125BootstrapClock(self.platform)
+        ext_async_rst = Signal()
+
+        self.submodules.bootstrap = GTPBootstrapClock(self.platform, clk_freq)
         self.submodules.sys_crg = zynq_clocking.SYSCRG(
             self.platform,
             self.ps7,
             txout_buf,
-            clk_sw=gtx0.tx_init.done)
+            clk_sw=self.gt_drtio.stable_clkin.storage,
+            clk_sw_status=gtx0.tx_init.done,
+            ext_async_rst=ext_async_rst)
         self.csr_devices.append("sys_crg")
         self.crg = self.ps7 # HACK for eem_7series to find the clock
         self.crg.cd_sys = self.sys_crg.cd_sys
@@ -211,11 +265,33 @@ class GenericMaster(SoCCore):
             self.bootstrap.cd_bootstrap.clk, self.sys_crg.cd_sys.clk)
         fix_serdes_timing_path(platform)
 
-        self.rustc_cfg["has_si5324"] = None
-        self.rustc_cfg["si5324_soft_reset"] = None
+        self.comb += ext_async_rst.eq(self.sys_crg.clk_sw_fsm.o_clk_sw & ~gtx0.tx_init.done)
+        self.specials += MultiReg(self.sys_crg.clk_sw_fsm.o_clk_sw & self.sys_crg.mmcm_locked, self.gt_drtio.clk_path_ready, odomain="bootstrap")
+
+        if with_wrpll:
+            clk_synth = platform.request("cdr_clk_clean_fabric")
+            clk_synth_se = Signal()
+            platform.add_period_constraint(clk_synth.p, 8.0)
+            self.specials += Instance("IBUFGDS", p_DIFF_TERM="TRUE", p_IBUF_LOW_PWR="FALSE", i_I=clk_synth.p, i_IB=clk_synth.n, o_O=clk_synth_se)
+            self.submodules.wrpll_refclk = wrpll.SMAFrequencyMultiplier(platform.request("sma_clkin"))
+            self.submodules.wrpll = wrpll.WRPLL(
+                platform=self.platform,
+                cd_ref=self.wrpll_refclk.cd_ref,
+                main_clk_se=clk_synth_se)
+            self.csr_devices.append("wrpll_refclk")
+            self.csr_devices.append("wrpll")
+            self.comb += self.ps7.core.core0.nfiq.eq(self.wrpll.ev.irq)
+            self.config["HAS_SI549"] = None
+            self.config["WRPLL_REF_CLK"] = "SMA_CLKIN"
+        else:
+            self.submodules += SMAClkinForward(self.platform)
+            self.config["HAS_SI5324"] = None
+            self.config["SI5324_SOFT_RESET"] = None
 
         self.rtio_channels = []
         has_grabber = any(peripheral["type"] == "grabber" for peripheral in description["peripherals"])
+        if has_drtio_over_eem:
+            self.eem_drtio_channels = []
         if has_grabber:
             self.grabber_csr_group = []
         eem_7series.add_peripherals(self, description["peripherals"], iostandard=eem_iostandard)
@@ -230,21 +306,21 @@ class GenericMaster(SoCCore):
 
         self.submodules.rtio_tsc = rtio.TSC(glbl_fine_ts_width=3)
 
-        drtio_csr_group = []
-        drtioaux_csr_group = []
-        drtioaux_memory_group = []
+        self.drtio_csr_group = []
+        self.drtioaux_csr_group = []
+        self.drtioaux_memory_group = []
         self.drtio_cri = []
-        for i in range(len(self.drtio_transceiver.channels)):
+        for i in range(len(self.gt_drtio.channels)):
             core_name = "drtio" + str(i)
             coreaux_name = "drtioaux" + str(i)
             memory_name = "drtioaux" + str(i) + "_mem"
-            drtio_csr_group.append(core_name)
-            drtioaux_csr_group.append(coreaux_name)
-            drtioaux_memory_group.append(memory_name)
+            self.drtio_csr_group.append(core_name)
+            self.drtioaux_csr_group.append(coreaux_name)
+            self.drtioaux_memory_group.append(memory_name)
 
             cdr = ClockDomainsRenamer({"rtio_rx": "rtio_rx" + str(i)})
 
-            core = cdr(DRTIOMaster(self.rtio_tsc, self.drtio_transceiver.channels[i]))
+            core = cdr(DRTIOMaster(self.rtio_tsc, self.gt_drtio.channels[i]))
             setattr(self.submodules, core_name, core)
             self.drtio_cri.append(core.cri)
             self.csr_devices.append(core_name)
@@ -257,24 +333,27 @@ class GenericMaster(SoCCore):
             memory_address = self.axi2csr.register_port(coreaux.get_tx_port(), size)
             self.axi2csr.register_port(coreaux.get_rx_port(), size)
             self.add_memory_region(memory_name, self.mem_map["csr"] + memory_address, size * 2)
-        self.rustc_cfg["has_drtio"] = None
-        self.rustc_cfg["has_drtio_routing"] = None
-        self.add_csr_group("drtio", drtio_csr_group)
-        self.add_csr_group("drtioaux", drtioaux_csr_group)
-        self.add_memory_group("drtioaux_mem", drtioaux_memory_group)
+        self.config["HAS_DRTIO"] = None
+        self.config["HAS_DRTIO_ROUTING"] = None
 
-        self.submodules.rtio_core = rtio.Core(self.rtio_tsc, self.rtio_channels)
+        if has_drtio_over_eem:
+            self.add_eem_drtio(self.eem_drtio_channels)
+        self.add_drtio_cpuif_groups()
+
+        self.submodules.rtio_core = rtio.Core(
+            self.rtio_tsc, self.rtio_channels, lane_count=description["sed_lanes"]
+        )
         self.csr_devices.append("rtio_core")
 
         if self.acpki:
-            self.rustc_cfg["ki_impl"] = "acp"
+            self.config["KI_IMPL"] = "acp"
             self.submodules.rtio = acpki.KernelInitiator(self.rtio_tsc,
                                                          bus=self.ps7.s_axi_acp,
                                                          user=self.ps7.s_axi_acp_user,
                                                          evento=self.ps7.event.o)
             self.csr_devices.append("rtio")
         else:
-            self.rustc_cfg["ki_impl"] = "csr"
+            self.config["KI_IMPL"] = "csr"
             self.submodules.rtio = rtio.KernelInitiator(self.rtio_tsc, now64=True)
             self.csr_devices.append("rtio")
 
@@ -298,16 +377,60 @@ class GenericMaster(SoCCore):
         self.csr_devices.append("rtio_analyzer")
 
         if has_grabber:
-            self.rustc_cfg["has_grabber"] = None
+            self.config["HAS_GRABBER"] = None
             self.add_csr_group("grabber", self.grabber_csr_group)
+        
+
+        self.submodules.virtual_leds = virtual_leds.VirtualLeds()
+        self.csr_devices.append("virtual_leds")
+
+        self.comb += [self.virtual_leds.get(i).eq(channel.rx_ready)
+                for i, channel in enumerate(self.gt_drtio.channels)]
+
+    def add_eem_drtio(self, eem_drtio_channels):
+        # Must be called before invoking add_rtio() to construct the CRI
+        # interconnect properly
+        self.submodules.eem_transceiver = eem_serdes.EEMSerdes(self.platform, eem_drtio_channels)
+        self.csr_devices.append("eem_transceiver")
+        self.config["HAS_DRTIO_EEM"] = None
+        self.config["EEM_DRTIO_COUNT"] = len(eem_drtio_channels)
+
+        cdr = ClockDomainsRenamer({"rtio_rx": "sys"})
+        for i in range(len(self.eem_transceiver.channels)):
+            channel = i + len(self.gt_drtio.channels)
+            core_name = "drtio" + str(channel)
+            coreaux_name = "drtioaux" + str(channel)
+            memory_name = "drtioaux" + str(channel) + "_mem"
+            self.drtio_csr_group.append(core_name)
+            self.drtioaux_csr_group.append(coreaux_name)
+            self.drtioaux_memory_group.append(memory_name)
+
+            core = cdr(DRTIOMaster(self.rtio_tsc, self.eem_transceiver.channels[i]))
+            setattr(self.submodules, core_name, core)
+            self.drtio_cri.append(core.cri)
+            self.csr_devices.append(core_name)
+
+            coreaux = cdr(drtio_aux_controller.DRTIOAuxControllerBare(core.link_layer))
+            setattr(self.submodules, coreaux_name, coreaux)
+            self.csr_devices.append(coreaux_name)
+
+            size = coreaux.get_mem_size()
+            memory_address = self.axi2csr.register_port(coreaux.get_tx_port(), size)
+            self.axi2csr.register_port(coreaux.get_rx_port(), size)
+            self.add_memory_region(memory_name, self.mem_map["csr"] + memory_address, size * 2)
+
+    def add_drtio_cpuif_groups(self):
+        self.add_csr_group("drtio", self.drtio_csr_group)
+        self.add_csr_group("drtioaux", self.drtioaux_csr_group)
+        self.add_memory_group("drtioaux_mem", self.drtioaux_memory_group)
 
 
 class GenericSatellite(SoCCore):
     def __init__(self, description, acpki=False):
         clk_freq = description["rtio_frequency"]
+        with_wrpll = description["enable_wrpll"]
 
         self.acpki = acpki
-        self.rustc_cfg = dict()
 
         platform = kasli_soc.Platform()
         platform.toolchain.bitstream_commands.extend([
@@ -318,24 +441,30 @@ class GenericSatellite(SoCCore):
             ident = "acpki_" + ident
         SoCCore.__init__(self, platform=platform, csr_data_width=32, ident=ident, ps_cd_sys=False)
 
+        self.config["HW_REV"] = description["hw_rev"]
+
         data_pads = [platform.request("sfp", i) for i in range(4)]
         
-        self.submodules.drtio_transceiver = gtx_7series.GTX(
+        self.submodules.gt_drtio = gtx_7series.GTX(
             clock_pads=platform.request("clk_gtp"),  
             pads=data_pads,
             clk_freq=clk_freq)
-        self.csr_devices.append("drtio_transceiver")
+        self.csr_devices.append("gt_drtio")
 
         txout_buf = Signal()
-        gtx0 = self.drtio_transceiver.gtxs[0]
+        gtx0 = self.gt_drtio.gtxs[0]
         self.specials += Instance("BUFG", i_I=gtx0.txoutclk, o_O=txout_buf)
 
-        self.submodules.bootstrap = GTP125BootstrapClock(self.platform)
+        ext_async_rst = Signal()
+
+        self.submodules.bootstrap = GTPBootstrapClock(self.platform, clk_freq)
         self.submodules.sys_crg = zynq_clocking.SYSCRG(
             self.platform, 
             self.ps7,
             txout_buf,
-            clk_sw=gtx0.tx_init.done)
+            clk_sw=self.gt_drtio.stable_clkin.storage,
+            clk_sw_status=gtx0.tx_init.done,
+            ext_async_rst=ext_async_rst)
         platform.add_false_path_constraints(
             self.bootstrap.cd_bootstrap.clk, self.sys_crg.cd_sys.clk)
         self.csr_devices.append("sys_crg")
@@ -344,6 +473,9 @@ class GenericSatellite(SoCCore):
 
         fix_serdes_timing_path(platform)
 
+        self.comb += ext_async_rst.eq(self.sys_crg.clk_sw_fsm.o_clk_sw & ~gtx0.tx_init.done)
+        self.specials += MultiReg(self.sys_crg.clk_sw_fsm.o_clk_sw & self.sys_crg.mmcm_locked, self.gt_drtio.clk_path_ready, odomain="bootstrap")
+
         self.rtio_channels = []
         has_grabber = any(peripheral["type"] == "grabber" for peripheral in description["peripherals"])
         if has_grabber:
@@ -364,7 +496,7 @@ class GenericSatellite(SoCCore):
         drtioaux_memory_group = []
         drtiorep_csr_group = []
         self.drtio_cri = []
-        for i in range(len(self.drtio_transceiver.channels)):
+        for i in range(len(self.gt_drtio.channels)):
             coreaux_name = "drtioaux" + str(i)
             memory_name = "drtioaux" + str(i) + "_mem"
             drtioaux_csr_group.append(coreaux_name)
@@ -375,7 +507,7 @@ class GenericSatellite(SoCCore):
             if i == 0:
                 self.submodules.rx_synchronizer = cdr(XilinxRXSynchronizer())
                 core = cdr(DRTIOSatellite(
-                    self.rtio_tsc, self.drtio_transceiver.channels[i],
+                    self.rtio_tsc, self.gt_drtio.channels[i],
                     self.rx_synchronizer))
                 self.submodules.drtiosat = core
                 self.csr_devices.append("drtiosat")
@@ -384,7 +516,7 @@ class GenericSatellite(SoCCore):
                 drtiorep_csr_group.append(corerep_name)
 
                 core = cdr(DRTIORepeater(
-                    self.rtio_tsc, self.drtio_transceiver.channels[i]))
+                    self.rtio_tsc, self.gt_drtio.channels[i]))
                 setattr(self.submodules, corerep_name, core)
                 self.drtio_cri.append(core.cri)
                 self.csr_devices.append(corerep_name)
@@ -402,32 +534,34 @@ class GenericSatellite(SoCCore):
             # and registered in PS interface
             # manually, because software refers to rx/tx by halves of entire memory block, not names
             self.add_memory_region(memory_name, self.mem_map["csr"] + memory_address, mem_size * 2)
-        self.rustc_cfg["has_drtio"] = None
-        self.rustc_cfg["has_drtio_routing"] = None
+        self.config["HAS_DRTIO"] = None
+        self.config["HAS_DRTIO_ROUTING"] = None
         self.add_csr_group("drtioaux", drtioaux_csr_group)
         self.add_memory_group("drtioaux_mem", drtioaux_memory_group)
         self.add_csr_group("drtiorep", drtiorep_csr_group)
 
         if self.acpki:
-            self.rustc_cfg["ki_impl"] = "acp"
+            self.config["KI_IMPL"] = "acp"
             self.submodules.rtio = acpki.KernelInitiator(self.rtio_tsc,
                                                          bus=self.ps7.s_axi_acp,
                                                          user=self.ps7.s_axi_acp_user,
                                                          evento=self.ps7.event.o)
             self.csr_devices.append("rtio")
         else:
-            self.rustc_cfg["ki_impl"] = "csr"
+            self.config["KI_IMPL"] = "csr"
             self.submodules.rtio = rtio.KernelInitiator(self.rtio_tsc, now64=True)
             self.csr_devices.append("rtio")
 
         self.submodules.rtio_dma = dma.DMA(self.ps7.s_axi_hp0)
         self.csr_devices.append("rtio_dma")
 
-        self.submodules.local_io = SyncRTIO(self.rtio_tsc, self.rtio_channels)
+        self.submodules.local_io = SyncRTIO(
+            self.rtio_tsc, self.rtio_channels, lane_count=description["sed_lanes"]
+        )
         self.comb += self.drtiosat.async_errors.eq(self.local_io.async_errors)
 
         self.submodules.cri_con = rtio.CRIInterconnectShared(
-            [self.drtiosat.cri, self.rtio_dma.cri],
+            [self.drtiosat.cri, self.rtio_dma.cri, self.rtio.cri],
             [self.local_io.cri] + self.drtio_cri,
             enable_routing=True)
         self.csr_devices.append("cri_con")
@@ -438,49 +572,53 @@ class GenericSatellite(SoCCore):
         self.submodules.rtio_moninj = rtio.MonInj(self.rtio_channels)
         self.csr_devices.append("rtio_moninj")
 
+        self.submodules.rtio_analyzer = analyzer.Analyzer(self.rtio_tsc, self.local_io.cri,
+                                                          self.ps7.s_axi_hp1)
+        self.csr_devices.append("rtio_analyzer")
+
         rtio_clk_period = 1e9/clk_freq
-        self.rustc_cfg["rtio_frequency"] = str(clk_freq/1e6)
+        self.config["RTIO_FREQUENCY"] = str(clk_freq/1e6)
+        self.config["CLOCK_FREQUENCY"] = int(clk_freq)
 
-        self.submodules.siphaser = SiPhaser7Series(
-            si5324_clkin=platform.request("cdr_clk"),
-            rx_synchronizer=self.rx_synchronizer,
-            ultrascale=False,
-            rtio_clk_freq=self.drtio_transceiver.rtio_clk_freq)
-        self.csr_devices.append("siphaser")
-        self.rustc_cfg["has_si5324"] = None
-        self.rustc_cfg["has_siphaser"] = None
-        self.rustc_cfg["si5324_soft_reset"] = None
+        if with_wrpll:
+            clk_synth = platform.request("cdr_clk_clean_fabric")
+            clk_synth_se = Signal()
+            platform.add_period_constraint(clk_synth.p, 8.0)
+            self.specials += Instance("IBUFGDS", p_DIFF_TERM="TRUE", p_IBUF_LOW_PWR="FALSE", i_I=clk_synth.p, i_IB=clk_synth.n, o_O=clk_synth_se)
+            self.submodules.wrpll = wrpll.WRPLL(
+                platform=self.platform,
+                cd_ref=self.gt_drtio.cd_rtio_rx0,
+                main_clk_se=clk_synth_se)
+            self.submodules.wrpll_skewtester = wrpll.SkewTester(self.rx_synchronizer)
+            self.csr_devices.append("wrpll_skewtester")
+            self.csr_devices.append("wrpll")
+            self.comb += self.ps7.core.core0.nfiq.eq(self.wrpll.ev.irq)
+            self.config["HAS_SI549"] = None
+            self.config["WRPLL_REF_CLK"] = "GT_CDR"
+        else:
+            self.submodules.siphaser = SiPhaser7Series(
+                si5324_clkin=platform.request("cdr_clk"),
+                rx_synchronizer=self.rx_synchronizer,
+                ultrascale=False,
+                rtio_clk_freq=self.gt_drtio.rtio_clk_freq)
+            self.csr_devices.append("siphaser")
+            self.config["HAS_SI5324"] = None
+            self.config["SI5324_SOFT_RESET"] = None
 
-        gtx0 = self.drtio_transceiver.gtxs[0]
+        gtx0 = self.gt_drtio.gtxs[0]
         platform.add_false_path_constraints(
             gtx0.txoutclk, gtx0.rxoutclk)
 
         if has_grabber:
-            self.rustc_cfg["has_grabber"] = None
+            self.config["HAS_GRABBER"] = None
             self.add_csr_group("grabber", self.grabber_csr_group)
             # no RTIO CRG here
-     
-
-def write_mem_file(soc, filename):
-    with open(filename, "w") as f:
-        f.write(cpu_interface.get_mem_rust(
-            soc.get_memory_regions(), soc.get_memory_groups(), None))
-
-
-def write_csr_file(soc, filename):
-    with open(filename, "w") as f:
-        f.write(cpu_interface.get_csr_rust(
-            soc.get_csr_regions(), soc.get_csr_groups(), soc.get_constants()))
-
-
-def write_rustc_cfg_file(soc, filename):
-    with open(filename, "w") as f:
-        for k, v in sorted(soc.rustc_cfg.items(), key=itemgetter(0)):
-            if v is None:
-                f.write("{}\n".format(k))
-            else:
-                f.write("{}=\"{}\"\n".format(k, v))
+        
+        self.submodules.virtual_leds = virtual_leds.VirtualLeds()
+        self.csr_devices.append("virtual_leds")
 
+        self.comb += [self.virtual_leds.get(i).eq(channel.rx_ready)
+                for i, channel in enumerate(self.gt_drtio.channels)]
 
 def main():
     parser = argparse.ArgumentParser(
@@ -503,14 +641,14 @@ def main():
     if description["target"] != "kasli_soc":
         raise ValueError("Description is for a different target")
 
-    if description["base"] == "standalone":
+    if description["drtio_role"] == "standalone":
         cls = GenericStandalone
-    elif description["base"] == "master":
+    elif description["drtio_role"] == "master":
         cls = GenericMaster
-    elif description["base"] == "satellite":
+    elif description["drtio_role"] == "satellite":
         cls = GenericSatellite
     else:
-        raise ValueError("Invalid base")
+        raise ValueError("Invalid DRTIO role")
 
     soc = cls(description, acpki=args.acpki)
     soc.finalize()

src/gateware/si549.py

@@ -0,0 +1,277 @@
+from migen import *
+from migen.genlib.fsm import *
+
+from misoc.interconnect.csr import *
+
+
+class I2CClockGen(Module):
+    def __init__(self, width):
+        self.load = Signal(width)
+        self.clk2x = Signal()
+
+        cnt = Signal.like(self.load)
+        self.comb += [
+            self.clk2x.eq(cnt == 0),
+        ]
+        self.sync += [
+            If(self.clk2x,
+                cnt.eq(self.load),
+            ).Else(
+                cnt.eq(cnt - 1),
+            )
+        ]
+
+
+class I2CMasterMachine(Module):
+    def __init__(self, clock_width):
+        self.scl = Signal(reset=1)
+        self.sda_o = Signal(reset=1)
+        self.sda_i = Signal()
+
+        self.submodules.cg = CEInserter()(I2CClockGen(clock_width))
+        self.start = Signal()
+        self.stop = Signal()
+        self.write = Signal()
+        self.ack = Signal()
+        self.data = Signal(8)
+        self.ready = Signal()
+
+        # # #
+
+        bits = Signal(4)
+        data = Signal(8)
+
+        fsm = CEInserter()(FSM("IDLE"))
+        self.submodules += fsm
+
+        fsm.act("IDLE",
+            self.ready.eq(1),
+            If(self.start,
+                NextState("START0"),
+            ).Elif(self.stop,
+                NextState("STOP0"),
+            ).Elif(self.write,
+                NextValue(bits, 8),
+                NextValue(data, self.data),
+                NextState("WRITE0")
+            )
+        )
+
+        fsm.act("START0",
+            NextValue(self.scl, 1),
+            NextState("START1")
+        )
+        fsm.act("START1",
+            NextValue(self.sda_o, 0),
+            NextState("IDLE")
+        )
+
+        fsm.act("STOP0",
+            NextValue(self.scl, 0),
+            NextState("STOP1")
+        )
+        fsm.act("STOP1",
+            NextValue(self.sda_o, 0),
+            NextState("STOP2")
+        )
+        fsm.act("STOP2",
+            NextValue(self.scl, 1), 
+            NextState("STOP3")
+        )
+        fsm.act("STOP3",
+            NextValue(self.sda_o, 1),
+            NextState("IDLE")
+        )
+
+        fsm.act("WRITE0",
+            NextValue(self.scl, 0),
+            NextState("WRITE1")
+        )
+        fsm.act("WRITE1",
+            If(bits == 0,
+                NextValue(self.sda_o, 1),
+                NextState("READACK0"),
+            ).Else(
+                NextValue(self.sda_o, data[7]),
+                NextState("WRITE2"),
+            )
+        )
+        fsm.act("WRITE2",
+            NextValue(self.scl, 1),
+            NextValue(data[1:], data[:-1]),
+            NextValue(bits, bits - 1),
+            NextState("WRITE0"),
+        )
+        fsm.act("READACK0",
+            NextValue(self.scl, 1),
+            NextState("READACK1"),
+        )
+        fsm.act("READACK1",
+            NextValue(self.ack, ~self.sda_i),
+            NextState("IDLE")
+        )
+
+        run = Signal()
+        idle = Signal()
+        self.comb += [
+            run.eq((self.start | self.stop | self.write) & self.ready),
+            idle.eq(~run & fsm.ongoing("IDLE")),
+            self.cg.ce.eq(~idle),
+            fsm.ce.eq(run | self.cg.clk2x),
+        ]
+
+
+class ADPLLProgrammer(Module):
+    def __init__(self):
+        self.i2c_divider = Signal(16)
+        self.i2c_address = Signal(7)
+
+        self.adpll = Signal(24)
+        self.stb = Signal()
+        self.busy = Signal()
+        self.nack = Signal()
+
+        self.scl = Signal()
+        self.sda_i = Signal()
+        self.sda_o = Signal()
+
+        # # #
+
+        master = I2CMasterMachine(16)
+        self.submodules += master
+
+        self.comb += [
+            master.cg.load.eq(self.i2c_divider),
+            self.scl.eq(master.scl),
+            master.sda_i.eq(self.sda_i),
+            self.sda_o.eq(master.sda_o)
+        ]
+
+        fsm = FSM()
+        self.submodules += fsm
+
+        fsm.act("IDLE",
+            If(self.stb,
+                NextValue(self.nack, 0),
+                NextState("START")
+            )
+        )
+        fsm.act("START",
+            master.start.eq(1),
+            If(master.ready, NextState("DEVADDRESS"))
+        )
+        fsm.act("DEVADDRESS",
+            master.data.eq(self.i2c_address << 1),
+            master.write.eq(1),
+            If(master.ready, NextState("REGADRESS"))
+        )
+        fsm.act("REGADRESS",
+            master.data.eq(231),
+            master.write.eq(1),
+            If(master.ready,
+                If(master.ack,
+                    NextState("DATA0")
+                ).Else(
+                    NextValue(self.nack, 1),
+                    NextState("STOP")
+                )
+            )
+        )
+        fsm.act("DATA0",
+            master.data.eq(self.adpll[0:8]),
+            master.write.eq(1),
+            If(master.ready,
+                If(master.ack,
+                    NextState("DATA1")
+                ).Else(
+                    NextValue(self.nack, 1),
+                    NextState("STOP")
+                )
+            )
+        )
+        fsm.act("DATA1",
+            master.data.eq(self.adpll[8:16]),
+            master.write.eq(1),
+            If(master.ready,
+                If(master.ack,
+                    NextState("DATA2")
+                ).Else(
+                    NextValue(self.nack, 1),
+                    NextState("STOP")
+                )
+            )
+        )
+        fsm.act("DATA2",
+            master.data.eq(self.adpll[16:24]),
+            master.write.eq(1),
+            If(master.ready,
+                If(~master.ack, NextValue(self.nack, 1)),
+                NextState("STOP")
+            )
+        )
+        fsm.act("STOP",
+            master.stop.eq(1),
+            If(master.ready,
+                If(~master.ack, NextValue(self.nack, 1)),
+                NextState("IDLE")
+            )
+        )
+
+        self.comb += self.busy.eq(~fsm.ongoing("IDLE"))
+
+
+class Si549(Module, AutoCSR):
+    def __init__(self, pads):
+        self.i2c_divider = CSRStorage(16, reset=75)
+        self.i2c_address = CSRStorage(7)
+
+        self.adpll = CSRStorage(24)
+        self.adpll_stb = CSR()
+        self.adpll_busy = CSRStatus()
+        self.nack = CSRStatus()
+
+        self.bitbang_enable = CSRStorage()
+
+        self.sda_oe = CSRStorage()
+        self.sda_out = CSRStorage()
+        self.sda_in = CSRStatus()
+        self.scl_oe = CSRStorage()
+        self.scl_out = CSRStorage()
+
+        # # #
+
+        self.submodules.programmer = ADPLLProgrammer()
+
+        self.sync += self.programmer.stb.eq(self.adpll_stb.re)
+
+        self.comb += [
+            self.programmer.i2c_divider.eq(self.i2c_divider.storage),
+            self.programmer.i2c_address.eq(self.i2c_address.storage),
+            self.programmer.adpll.eq(self.adpll.storage),
+            self.adpll_busy.status.eq(self.programmer.busy),
+            self.nack.status.eq(self.programmer.nack)
+        ]
+
+        # I2C with bitbang/gateware mode select
+        sda_t = TSTriple(1)
+        scl_t = TSTriple(1)
+        self.specials += [
+            sda_t.get_tristate(pads.sda),
+            scl_t.get_tristate(pads.scl)
+        ]
+
+        self.comb += [
+            If(self.bitbang_enable.storage,
+                sda_t.oe.eq(self.sda_oe.storage),
+                sda_t.o.eq(self.sda_out.storage),
+                self.sda_in.status.eq(sda_t.i),
+                scl_t.oe.eq(self.scl_oe.storage),
+                scl_t.o.eq(self.scl_out.storage)
+            ).Else(
+                sda_t.oe.eq(~self.programmer.sda_o),
+                sda_t.o.eq(0),
+                self.programmer.sda_i.eq(sda_t.i),
+                scl_t.oe.eq(~self.programmer.scl),
+                scl_t.o.eq(0),
+            )
+        ]

src/gateware/wrpll.py

@@ -0,0 +1,237 @@
+from migen import *
+from migen.genlib.cdc import MultiReg, AsyncResetSynchronizer, PulseSynchronizer
+from misoc.interconnect.csr import *
+from misoc.interconnect.csr_eventmanager import *
+
+from ddmtd import DDMTDSampler, DDMTD
+from si549 import Si549
+
+class FrequencyCounter(Module, AutoCSR):
+    def __init__(self, domains, counter_width=24):
+        self.update = CSR()
+        self.busy = CSRStatus()
+
+        counter_reset = Signal()
+        counter_stb = Signal()
+        timer = Signal(counter_width)
+
+        # # #
+
+        fsm = FSM()
+        self.submodules += fsm
+
+        fsm.act("IDLE",
+            counter_reset.eq(1),
+            If(self.update.re,
+                NextValue(timer, 2**counter_width - 1),
+                NextState("COUNTING")
+            )
+        )
+        fsm.act("COUNTING",
+            self.busy.status.eq(1),
+            If(timer != 0,
+                NextValue(timer, timer - 1)
+            ).Else( 
+                counter_stb.eq(1),
+                NextState("IDLE")
+            )
+        )
+
+        for domain in domains:
+            name = "counter_" + domain
+            counter_csr = CSRStatus(counter_width, name=name)
+            setattr(self, name, counter_csr)
+
+            divider = Signal(2)
+            divided = Signal()
+            divided_sys = Signal()
+            divided_sys_r = Signal()
+            divided_tick = Signal()
+            counter = Signal(counter_width)
+
+            # # #
+
+            sync_domain = getattr(self.sync, domain)
+            sync_domain +=[
+                divider.eq(divider + 1),
+                divided.eq(divider[-1])
+            ]
+            self.specials += MultiReg(divided, divided_sys)
+            self.sync += divided_sys_r.eq(divided_sys)
+            self.comb += divided_tick.eq(divided_sys & ~divided_sys_r)
+
+            self.sync += [
+                If(counter_stb, counter_csr.status.eq(counter)),
+                If(divided_tick, counter.eq(counter + 1)),
+                If(counter_reset, counter.eq(0))
+            ]
+
+class SkewTester(Module, AutoCSR):
+    def __init__(self, rx_synchronizer):
+        self.error = CSR()
+
+        # # #
+        
+        # The RX synchronizer is tested for setup/hold violations by feeding it a
+        # toggling pattern and checking that the same toggling pattern comes out.
+        toggle_in = Signal()
+        self.sync.rtio_rx0 += toggle_in.eq(~toggle_in)
+        toggle_out = rx_synchronizer.resync(toggle_in)
+
+        toggle_out_expected = Signal()
+        self.sync += toggle_out_expected.eq(~toggle_out)
+
+        error = Signal()
+        self.sync += [
+            If(toggle_out != toggle_out_expected, error.eq(1)),
+            If(self.error.re, error.eq(0))
+        ]
+        self.specials += MultiReg(error, self.error.w)
+
+
+class WRPLL(Module, AutoCSR):
+    def __init__(self, platform, cd_ref, main_clk_se, COUNTER_BIT=32):
+        self.helper_reset = CSRStorage(reset=1)
+        self.ref_tag = CSRStatus(COUNTER_BIT)
+        self.main_tag = CSRStatus(COUNTER_BIT)
+
+        ddmtd_counter = Signal(COUNTER_BIT)
+
+        ref_tag_sys = Signal(COUNTER_BIT)
+        main_tag_sys = Signal(COUNTER_BIT)
+        ref_tag_stb_sys = Signal()
+        main_tag_stb_sys = Signal()
+
+        # # #
+
+        self.submodules.main_dcxo = Si549(platform.request("ddmtd_main_dcxo_i2c"))
+        self.submodules.helper_dcxo = Si549(platform.request("ddmtd_helper_dcxo_i2c"))
+
+        helper_dcxo_pads = platform.request("ddmtd_helper_clk")
+        self.clock_domains.cd_helper = ClockDomain()
+        self.specials += [
+            Instance("IBUFGDS",
+                     i_I=helper_dcxo_pads.p, i_IB=helper_dcxo_pads.n,
+                     o_O=self.cd_helper.clk),
+            AsyncResetSynchronizer(self.cd_helper, self.helper_reset.storage)
+        ]
+
+        self.submodules.frequency_counter = FrequencyCounter(["sys", cd_ref.name])
+
+        self.submodules.ddmtd_sampler = DDMTDSampler(cd_ref, main_clk_se)
+
+        self.sync.helper += ddmtd_counter.eq(ddmtd_counter + 1)
+        self.submodules.ddmtd_ref = DDMTD(ddmtd_counter, self.ddmtd_sampler.ref_beating)
+        self.submodules.ddmtd_main = DDMTD(ddmtd_counter, self.ddmtd_sampler.main_beating)
+
+        # DDMTD tags collection
+
+        self.specials += [
+            MultiReg(self.ddmtd_ref.h_tag, ref_tag_sys),
+            MultiReg(self.ddmtd_main.h_tag, main_tag_sys)
+        ]
+
+        ref_tag_stb_ps = PulseSynchronizer("helper", "sys")
+        main_tag_stb_ps = PulseSynchronizer("helper", "sys")
+        self.submodules += [
+            ref_tag_stb_ps,
+            main_tag_stb_ps
+        ]
+        self.sync.helper += [
+            ref_tag_stb_ps.i.eq(self.ddmtd_ref.h_tag_update),
+            main_tag_stb_ps.i.eq(self.ddmtd_main.h_tag_update)
+        ]
+        self.sync += [
+            ref_tag_stb_sys.eq(ref_tag_stb_ps.o),
+            main_tag_stb_sys.eq(main_tag_stb_ps.o)
+        ]
+
+        self.sync += [
+            If(ref_tag_stb_sys,
+                self.ref_tag.status.eq(ref_tag_sys),
+            ),
+            If(main_tag_stb_sys,
+                self.main_tag.status.eq(main_tag_sys)
+            )
+        ]
+
+        # EventMangers for firmware interrupt
+
+        self.submodules.ref_tag_ev = EventManager()
+        self.ref_tag_ev.stb = EventSourcePulse()
+        self.ref_tag_ev.finalize()
+
+        self.submodules.main_tag_ev = EventManager()
+        self.main_tag_ev.stb = EventSourcePulse()
+        self.main_tag_ev.finalize()
+
+        self.sync += [
+            self.ref_tag_ev.stb.trigger.eq(ref_tag_stb_sys),
+            self.main_tag_ev.stb.trigger.eq(main_tag_stb_sys)
+        ]
+
+        self.submodules.ev = SharedIRQ(self.ref_tag_ev, self.main_tag_ev)
+
+
+class SMAFrequencyMultiplier(Module, AutoCSR):
+    def __init__(self, sma_clkin):
+        sma_clkin_se = Signal()
+        mmcm_locked = Signal()
+        mmcm_fb_clk = Signal()
+        ref_clk = Signal()
+        self.clock_domains.cd_ref = ClockDomain()
+        self.refclk_reset = CSRStorage(reset=1)
+
+        self.mmcm_bypass = CSRStorage()
+        self.mmcm_locked = CSRStatus()
+        self.mmcm_reset = CSRStorage(reset=1)
+
+        self.mmcm_daddr = CSRStorage(7)
+        self.mmcm_din = CSRStorage(16)
+        self.mmcm_dwen = CSRStorage()
+        self.mmcm_den = CSRStorage()
+        self.mmcm_dclk = CSRStorage()
+        self.mmcm_dout = CSRStatus(16)
+        self.mmcm_dready = CSRStatus()
+
+        # # #
+
+        self.specials += [
+            Instance("IBUFDS",
+                i_I=sma_clkin.p, i_IB=sma_clkin.n,
+                o_O=sma_clkin_se),
+            # MMCME2 is capable to accept 10MHz input while PLLE2 only support down to 19MHz input (DS191)
+            # The MMCME2 can be reconfiged during runtime using the Dynamic Reconfiguration Ports
+            Instance("MMCME2_ADV",
+                p_BANDWIDTH="HIGH",  # lower output jitter (see https://support.xilinx.com/s/question/0D52E00006iHqRqSAK)
+                o_LOCKED=self.mmcm_locked.status,
+                i_RST=self.mmcm_reset.storage,
+                
+                p_CLKIN1_PERIOD=8, # ns
+                i_CLKIN1=sma_clkin_se,
+                i_CLKINSEL=1,  # 1=CLKIN1 0=CLKIN2
+
+                # VCO @ 1.25GHz 
+                p_CLKFBOUT_MULT_F=10, p_DIVCLK_DIVIDE=1,
+                i_CLKFBIN=mmcm_fb_clk, o_CLKFBOUT=mmcm_fb_clk, 
+
+                # 125MHz for WRPLL
+                p_CLKOUT0_DIVIDE_F=10, p_CLKOUT0_PHASE=0.0, o_CLKOUT0=ref_clk,
+
+                # Dynamic Reconfiguration Ports
+                i_DADDR = self.mmcm_daddr.storage,
+                i_DI = self.mmcm_din.storage,
+                i_DWE = self.mmcm_dwen.storage,
+                i_DEN = self.mmcm_den.storage,
+                i_DCLK = self.mmcm_dclk.storage,
+                o_DO = self.mmcm_dout.status,
+                o_DRDY = self.mmcm_dready.status
+            ),
+            Instance("BUFGMUX",
+                i_I0=ref_clk,
+                i_I1=sma_clkin_se,
+                i_S=self.mmcm_bypass.storage,
+                o_O=self.cd_ref.clk
+            ),
+            AsyncResetSynchronizer(self.cd_ref, self.refclk_reset.storage),
+        ]

src/gateware/zc706.py

@@ -10,7 +10,6 @@ from migen.genlib.cdc import MultiReg
 from migen_axi.integration.soc_core import SoCCore
 from migen_axi.platforms import zc706
 from misoc.interconnect.csr import *
-from misoc.integration import cpu_interface
 from misoc.cores import gpio
 
 from artiq.gateware import rtio, nist_clock, nist_qc2
@@ -26,7 +25,7 @@ import analyzer
 import acpki
 import drtio_aux_controller
 import zynq_clocking
-
+from config import write_csr_file, write_mem_file, write_rustc_cfg_file
 
 class SMAClkinForward(Module):
     def __init__(self, platform):
@@ -127,7 +126,6 @@ def prepare_zc706_platform(platform):
 class ZC706(SoCCore):
     def __init__(self, acpki=False):
         self.acpki = acpki
-        self.rustc_cfg = dict()
 
         platform = zc706.Platform()
         prepare_zc706_platform(platform)
@@ -149,14 +147,14 @@ class ZC706(SoCCore):
                 i_CEB=0,
                 i_I=si5324_out.p, i_IB=si5324_out.n,
                 o_O=cdr_clk,
-                p_CLKCM_CFG="0b1",
-                p_CLKRCV_TRST="0b1",
-                p_CLKSWING_CFG="0b11"),
+                p_CLKCM_CFG="TRUE",
+                p_CLKRCV_TRST="TRUE",
+                p_CLKSWING_CFG=3),
             Instance("BUFG", i_I=cdr_clk, o_O=cdr_clk_buf)
         ]
-        self.rustc_cfg["has_si5324"] = None
-        self.rustc_cfg["si5324_as_synthesizer"] = None
-        self.rustc_cfg["si5324_soft_reset"] = None
+        self.config["HAS_SI5324"] = None
+        self.config["SI5324_AS_SYNTHESIZER"] = None
+        self.config["SI5324_SOFT_RESET"] = None
         
         self.submodules.bootstrap = CLK200BootstrapClock(platform)
         self.submodules.sys_crg = zynq_clocking.SYSCRG(self.platform, self.ps7, cdr_clk_buf)
@@ -170,14 +168,14 @@ class ZC706(SoCCore):
         self.csr_devices.append("rtio_core")
 
         if self.acpki:
-            self.rustc_cfg["ki_impl"] = "acp"
+            self.config["KI_IMPL"] = "acp"
             self.submodules.rtio = acpki.KernelInitiator(self.rtio_tsc,
                                                          bus=self.ps7.s_axi_acp,
                                                          user=self.ps7.s_axi_acp_user,
                                                          evento=self.ps7.event.o)
             self.csr_devices.append("rtio")
         else:
-            self.rustc_cfg["ki_impl"] = "csr"
+            self.config["KI_IMPL"] = "csr"
             self.submodules.rtio = rtio.KernelInitiator(self.rtio_tsc, now64=True)
             self.csr_devices.append("rtio")
 
@@ -200,7 +198,6 @@ class ZC706(SoCCore):
 class _MasterBase(SoCCore):
     def __init__(self, acpki=False, drtio100mhz=False):
         self.acpki = acpki
-        self.rustc_cfg = dict()
 
         clk_freq = 100e6 if drtio100mhz else 125e6
 
@@ -222,32 +219,38 @@ class _MasterBase(SoCCore):
         self.submodules += SMAClkinForward(self.platform)
 
         # 1000BASE_BX10 Ethernet compatible, 125MHz RTIO clock
-        self.submodules.drtio_transceiver = gtx_7series.GTX(
+        self.submodules.gt_drtio = gtx_7series.GTX(
             clock_pads=platform.request("si5324_clkout"),
             pads=data_pads,
             clk_freq=clk_freq)
-        self.csr_devices.append("drtio_transceiver")
+        self.csr_devices.append("gt_drtio")
 
         self.submodules.rtio_tsc = rtio.TSC(glbl_fine_ts_width=3)
+        ext_async_rst = Signal()
         txout_buf = Signal()
-        gtx0 = self.drtio_transceiver.gtxs[0]
+        gtx0 = self.gt_drtio.gtxs[0]
         self.specials += Instance("BUFG", i_I=gtx0.txoutclk, o_O=txout_buf)
         self.submodules.bootstrap = CLK200BootstrapClock(platform, clk_freq)
         self.submodules.sys_crg = zynq_clocking.SYSCRG(
             self.platform, 
             self.ps7, 
             txout_buf,
-            clk_sw=gtx0.tx_init.done,
+            clk_sw=self.gt_drtio.stable_clkin.storage,
+            clk_sw_status=gtx0.tx_init.done,
+            ext_async_rst=ext_async_rst,
             freq=clk_freq)
         platform.add_false_path_constraints(
             self.bootstrap.cd_bootstrap.clk, self.sys_crg.cd_sys.clk)
         self.csr_devices.append("sys_crg")
 
+        self.comb += ext_async_rst.eq(self.sys_crg.clk_sw_fsm.o_clk_sw & ~gtx0.tx_init.done)
+        self.specials += MultiReg(self.sys_crg.clk_sw_fsm.o_clk_sw & self.sys_crg.mmcm_locked, self.gt_drtio.clk_path_ready, odomain="bootstrap")
+
         drtio_csr_group = []
         drtioaux_csr_group = []
         drtioaux_memory_group = []
         self.drtio_cri = []
-        for i in range(len(self.drtio_transceiver.channels)):
+        for i in range(len(self.gt_drtio.channels)):
             core_name = "drtio" + str(i)
             coreaux_name = "drtioaux" + str(i)
             memory_name = "drtioaux" + str(i) + "_mem"
@@ -258,7 +261,7 @@ class _MasterBase(SoCCore):
             cdr = ClockDomainsRenamer({"rtio_rx": "rtio_rx" + str(i)})
 
             core = cdr(DRTIOMaster(
-                self.rtio_tsc, self.drtio_transceiver.channels[i]))
+                self.rtio_tsc, self.gt_drtio.channels[i]))
             setattr(self.submodules, core_name, core)
             self.drtio_cri.append(core.cri)
             self.csr_devices.append(core_name)
@@ -271,18 +274,18 @@ class _MasterBase(SoCCore):
             memory_address = self.axi2csr.register_port(coreaux.get_tx_port(), mem_size)
             self.axi2csr.register_port(coreaux.get_rx_port(), mem_size)
             self.add_memory_region(memory_name, self.mem_map["csr"] + memory_address, mem_size * 2)
-        self.rustc_cfg["has_drtio"] = None
-        self.rustc_cfg["has_drtio_routing"] = None
+        self.config["HAS_DRTIO"] = None
+        self.config["HAS_DRTIO_ROUTING"] = None
         self.add_csr_group("drtio", drtio_csr_group)
         self.add_csr_group("drtioaux", drtioaux_csr_group)
         self.add_memory_group("drtioaux_mem", drtioaux_memory_group)
 
-        self.rustc_cfg["rtio_frequency"] = str(self.drtio_transceiver.rtio_clk_freq/1e6)
+        self.config["RTIO_FREQUENCY"] = str(self.gt_drtio.rtio_clk_freq/1e6)
 
         self.submodules.si5324_rst_n = gpio.GPIOOut(platform.request("si5324_33").rst_n)
         self.csr_devices.append("si5324_rst_n")
-        self.rustc_cfg["has_si5324"] = None
-        self.rustc_cfg["si5324_as_synthesizer"] = None
+        self.config["HAS_SI5324"] = None
+        self.config["SI5324_AS_SYNTHESIZER"] = None
 
         # Constrain TX & RX timing for the first transceiver channel
         # (First channel acts as master for phase alignment for all channels' TX)
@@ -290,7 +293,7 @@ class _MasterBase(SoCCore):
             gtx0.txoutclk, gtx0.rxoutclk)
         # Constrain RX timing for the each transceiver channel
         # (Each channel performs single-lane phase alignment for RX)
-        for gtx in self.drtio_transceiver.gtxs[1:]:
+        for gtx in self.gt_drtio.gtxs[1:]:
             platform.add_false_path_constraints(
                 gtx0.txoutclk, gtx.rxoutclk)
 
@@ -302,14 +305,14 @@ class _MasterBase(SoCCore):
         self.csr_devices.append("rtio_core")
 
         if self.acpki:
-            self.rustc_cfg["ki_impl"] = "acp"
+            self.config["KI_IMPL"] = "acp"
             self.submodules.rtio = acpki.KernelInitiator(self.rtio_tsc,
                                                          bus=self.ps7.s_axi_acp,
                                                          user=self.ps7.s_axi_acp_user,
                                                          evento=self.ps7.event.o)
             self.csr_devices.append("rtio")
         else:
-            self.rustc_cfg["ki_impl"] = "csr"
+            self.config["KI_IMPL"] = "csr"
             self.submodules.rtio = rtio.KernelInitiator(self.rtio_tsc, now64=True)
             self.csr_devices.append("rtio")
 
@@ -336,7 +339,6 @@ class _MasterBase(SoCCore):
 class _SatelliteBase(SoCCore):
     def __init__(self, acpki=False, drtio100mhz=False):
         self.acpki = acpki
-        self.rustc_cfg = dict()
 
         clk_freq = 100e6 if drtio100mhz else 125e6
 
@@ -359,15 +361,16 @@ class _SatelliteBase(SoCCore):
         self.submodules.rtio_tsc = rtio.TSC(glbl_fine_ts_width=3)
 
         # 1000BASE_BX10 Ethernet compatible, 125MHz RTIO clock
-        self.submodules.drtio_transceiver = gtx_7series.GTX(
+        self.submodules.gt_drtio = gtx_7series.GTX(
             clock_pads=platform.request("si5324_clkout"),
             pads=data_pads,
             clk_freq=clk_freq)
-        self.csr_devices.append("drtio_transceiver")
+        self.csr_devices.append("gt_drtio")
 
+        ext_async_rst = Signal()
         txout_buf = Signal()
         txout_buf.attr.add("keep")
-        gtx0 = self.drtio_transceiver.gtxs[0]
+        gtx0 = self.gt_drtio.gtxs[0]
         self.specials += Instance(
             "BUFG", 
             i_I=gtx0.txoutclk, 
@@ -377,17 +380,22 @@ class _SatelliteBase(SoCCore):
             self.platform, 
             self.ps7, 
             txout_buf,
-            clk_sw=gtx0.tx_init.done,
+            clk_sw=self.gt_drtio.stable_clkin.storage,
+            clk_sw_status=gtx0.tx_init.done,
+            ext_async_rst=ext_async_rst,
             freq=clk_freq)
         platform.add_false_path_constraints(
             self.bootstrap.cd_bootstrap.clk, self.sys_crg.cd_sys.clk)
         self.csr_devices.append("sys_crg")
 
+        self.comb += ext_async_rst.eq(self.sys_crg.clk_sw_fsm.o_clk_sw & ~gtx0.tx_init.done)
+        self.specials += MultiReg(self.sys_crg.clk_sw_fsm.o_clk_sw & self.sys_crg.mmcm_locked, self.gt_drtio.clk_path_ready, odomain="bootstrap")
+
         drtioaux_csr_group = []
         drtioaux_memory_group = []
         drtiorep_csr_group = []
         self.drtio_cri = []
-        for i in range(len(self.drtio_transceiver.channels)):
+        for i in range(len(self.gt_drtio.channels)):
             coreaux_name = "drtioaux" + str(i)
             memory_name = "drtioaux" + str(i) + "_mem"
             drtioaux_csr_group.append(coreaux_name)
@@ -399,7 +407,7 @@ class _SatelliteBase(SoCCore):
             if i == 0:
                 self.submodules.rx_synchronizer = cdr(XilinxRXSynchronizer())
                 core = cdr(DRTIOSatellite(
-                    self.rtio_tsc, self.drtio_transceiver.channels[0], self.rx_synchronizer))
+                    self.rtio_tsc, self.gt_drtio.channels[0], self.rx_synchronizer))
                 self.submodules.drtiosat = core
                 self.csr_devices.append("drtiosat")
             # Repeaters
@@ -407,7 +415,7 @@ class _SatelliteBase(SoCCore):
                 corerep_name = "drtiorep" + str(i-1)
                 drtiorep_csr_group.append(corerep_name)
                 core = cdr(DRTIORepeater(
-                    self.rtio_tsc, self.drtio_transceiver.channels[i]))
+                    self.rtio_tsc, self.gt_drtio.channels[i]))
                 setattr(self.submodules, corerep_name, core)
                 self.drtio_cri.append(core.cri)
                 self.csr_devices.append(corerep_name)
@@ -425,36 +433,35 @@ class _SatelliteBase(SoCCore):
             # and registered in PS interface
             # manually, because software refers to rx/tx by halves of entire memory block, not names
             self.add_memory_region(memory_name, self.mem_map["csr"] + memory_address, mem_size * 2)
-        self.rustc_cfg["has_drtio"] = None
-        self.rustc_cfg["has_drtio_routing"] = None
+        self.config["HAS_DRTIO"] = None
+        self.config["HAS_DRTIO_ROUTING"] = None
         self.add_csr_group("drtioaux", drtioaux_csr_group)
         self.add_csr_group("drtiorep", drtiorep_csr_group)
         self.add_memory_group("drtioaux_mem", drtioaux_memory_group)
 
-        self.rustc_cfg["rtio_frequency"] = str(self.drtio_transceiver.rtio_clk_freq/1e6)
+        self.config["RTIO_FREQUENCY"] = str(self.gt_drtio.rtio_clk_freq/1e6)
 
         # Si5324 Phaser
         self.submodules.siphaser = SiPhaser7Series(
             si5324_clkin=platform.request("si5324_clkin"),
             rx_synchronizer=self.rx_synchronizer,
             ultrascale=False,
-            rtio_clk_freq=self.drtio_transceiver.rtio_clk_freq)
+            rtio_clk_freq=self.gt_drtio.rtio_clk_freq)
         platform.add_false_path_constraints(
             self.sys_crg.cd_sys.clk, self.siphaser.mmcm_freerun_output)
         self.csr_devices.append("siphaser")
         self.submodules.si5324_rst_n = gpio.GPIOOut(platform.request("si5324_33").rst_n)
         self.csr_devices.append("si5324_rst_n")
-        self.rustc_cfg["has_si5324"] = None
-        self.rustc_cfg["has_siphaser"] = None
+        self.config["HAS_SI5324"] = None
 
-        rtio_clk_period = 1e9/self.drtio_transceiver.rtio_clk_freq
+        rtio_clk_period = 1e9/self.gt_drtio.rtio_clk_freq
         # Constrain TX & RX timing for the first transceiver channel
         # (First channel acts as master for phase alignment for all channels' TX)
         platform.add_false_path_constraints(
             gtx0.txoutclk, gtx0.rxoutclk)
         # Constrain RX timing for the each transceiver channel
         # (Each channel performs single-lane phase alignment for RX)
-        for gtx in self.drtio_transceiver.gtxs[1:]:
+        for gtx in self.gt_drtio.gtxs[1:]:
             platform.add_false_path_constraints(
                 self.sys_crg.cd_sys.clk, gtx.rxoutclk)
 
@@ -465,14 +472,14 @@ class _SatelliteBase(SoCCore):
         self.csr_devices.append("rtio_moninj")
 
         if self.acpki:
-            self.rustc_cfg["ki_impl"] = "acp"
+            self.config["KI_IMPL"] = "acp"
             self.submodules.rtio = acpki.KernelInitiator(self.rtio_tsc,
                                                          bus=self.ps7.s_axi_acp,
                                                          user=self.ps7.s_axi_acp_user,
                                                          evento=self.ps7.event.o)
             self.csr_devices.append("rtio")
         else:
-            self.rustc_cfg["ki_impl"] = "csr"
+            self.config["KI_IMPL"] = "csr"
             self.submodules.rtio = rtio.KernelInitiator(self.rtio_tsc, now64=True)
             self.csr_devices.append("rtio")
 
@@ -481,11 +488,15 @@ class _SatelliteBase(SoCCore):
 
         self.submodules.local_io = SyncRTIO(self.rtio_tsc, rtio_channels)
         self.submodules.cri_con = rtio.CRIInterconnectShared(
-            [self.drtiosat.cri, self.rtio_dma.cri],
+            [self.drtiosat.cri, self.rtio_dma.cri, self.rtio.cri],
             [self.local_io.cri] + self.drtio_cri,
             enable_routing=True)
         self.csr_devices.append("cri_con")
 
+        self.submodules.rtio_analyzer = analyzer.Analyzer(self.rtio_tsc, self.local_io.cri,
+                                                          self.ps7.s_axi_hp1)
+        self.csr_devices.append("rtio_analyzer")
+
         self.submodules.routing_table = rtio.RoutingTableAccess(self.cri_con)
         self.csr_devices.append("routing_table")
 
@@ -672,27 +683,6 @@ class NIST_QC2_Satellite(_SatelliteBase, _NIST_QC2_RTIO):
 VARIANTS = {cls.__name__.lower(): cls for cls in [NIST_CLOCK, NIST_CLOCK_Master, NIST_CLOCK_Satellite,
                                                   NIST_QC2, NIST_QC2_Master, NIST_QC2_Satellite]}
 
-
-def write_csr_file(soc, filename):
-    with open(filename, "w") as f:
-        f.write(cpu_interface.get_csr_rust(
-            soc.get_csr_regions(), soc.get_csr_groups(), soc.get_constants()))
-
-def write_mem_file(soc, filename):
-    with open(filename, "w") as f:
-        f.write(cpu_interface.get_mem_rust(
-            soc.get_memory_regions(), soc.get_memory_groups(), None))
-
-
-def write_rustc_cfg_file(soc, filename):
-    with open(filename, "w") as f:
-        for k, v in sorted(soc.rustc_cfg.items(), key=itemgetter(0)):
-            if v is None:
-                f.write("{}\n".format(k))
-            else:
-                f.write("{}=\"{}\"\n".format(k, v))
-
-
 def main():
     parser = argparse.ArgumentParser(
         description="ARTIQ port to the ZC706 Zynq development kit")

src/gateware/zynq_clocking.py

@@ -65,10 +65,12 @@ class ClockSwitchFSM(Module):
 
 
 class SYSCRG(Module, AutoCSR):
-    def __init__(self, platform, ps7, main_clk, clk_sw=None, freq=125e6):
+    def __init__(self, platform, ps7, main_clk, clk_sw=None, clk_sw_status=None, freq=125e6, ext_async_rst=None, ):
         # assumes bootstrap clock is same freq as main and sys output
         self.clock_domains.cd_sys = ClockDomain()
         self.clock_domains.cd_sys4x = ClockDomain(reset_less=True)
+        self.clock_domains.cd_sys5x = ClockDomain(reset_less=True)
+        self.clock_domains.cd_clk200 = ClockDomain()
 
         self.current_clock = CSRStatus()
 
@@ -78,11 +80,6 @@ class SYSCRG(Module, AutoCSR):
 
         period = 1e9/freq
 
-        pll_locked = Signal()
-        pll_sys = Signal()
-        pll_sys4x = Signal()
-        fb_clk = Signal()
-
         self.submodules.clk_sw_fsm = ClockSwitchFSM()
 
         if clk_sw is None:
@@ -91,32 +88,67 @@ class SYSCRG(Module, AutoCSR):
         else:
             self.comb += self.clk_sw_fsm.i_clk_sw.eq(clk_sw)
 
+        self.mmcm_locked = Signal()
+        mmcm_sys = Signal()
+        mmcm_sys4x = Signal()
+        mmcm_sys5x = Signal()
+        mmcm_clk208 = Signal()
+        mmcm_fb_clk = Signal()
         self.specials += [
-            Instance("PLLE2_ADV",
-                     p_STARTUP_WAIT="FALSE", o_LOCKED=pll_locked,
-                     p_BANDWIDTH="HIGH",
-                     p_REF_JITTER1=0.001,
-                     p_CLKIN1_PERIOD=period, i_CLKIN1=main_clk,
-                     p_CLKIN2_PERIOD=period, i_CLKIN2=bootstrap_clk,
-                     i_CLKINSEL=self.clk_sw_fsm.o_clk_sw,
+            Instance("MMCME2_ADV",
+                p_STARTUP_WAIT="FALSE", o_LOCKED=self.mmcm_locked,
+                p_BANDWIDTH="HIGH",
+                p_REF_JITTER1=0.001,
+                p_CLKIN1_PERIOD=period, i_CLKIN1=main_clk,
+                p_CLKIN2_PERIOD=period, i_CLKIN2=bootstrap_clk,
+                i_CLKINSEL=self.clk_sw_fsm.o_clk_sw,
 
-                     # VCO @ 1.5GHz when using 125MHz input
-                     # 1.2GHz for 100MHz (zc706)
-                     p_CLKFBOUT_MULT=12, p_DIVCLK_DIVIDE=1,
-                     i_CLKFBIN=fb_clk,
-                     i_RST=self.clk_sw_fsm.o_reset,
+                # VCO @ 1.25GHz
+                p_CLKFBOUT_MULT_F=10, p_DIVCLK_DIVIDE=1,
+                i_CLKFBIN=mmcm_fb_clk,
+                i_RST=self.clk_sw_fsm.o_reset,
 
-                     o_CLKFBOUT=fb_clk,
+                o_CLKFBOUT=mmcm_fb_clk,
 
-                     p_CLKOUT0_DIVIDE=3, p_CLKOUT0_PHASE=0.0,
-                     o_CLKOUT0=pll_sys4x,
+                p_CLKOUT0_DIVIDE_F=2.5, p_CLKOUT0_PHASE=0.0, o_CLKOUT0=mmcm_sys4x,
 
-                     p_CLKOUT1_DIVIDE=12, p_CLKOUT1_PHASE=0.0,
-                     o_CLKOUT1=pll_sys),
-            Instance("BUFG", i_I=pll_sys, o_O=self.cd_sys.clk),
-            Instance("BUFG", i_I=pll_sys4x, o_O=self.cd_sys4x.clk),
+                # 125MHz
+                p_CLKOUT1_DIVIDE=10, p_CLKOUT1_PHASE=0.0, o_CLKOUT1=mmcm_sys,
 
-            AsyncResetSynchronizer(self.cd_sys, ~pll_locked),
+                # 625MHz
+                p_CLKOUT2_DIVIDE=2, p_CLKOUT2_PHASE=0.0, o_CLKOUT2=mmcm_sys5x,
+
+                # 208MHz
+                p_CLKOUT3_DIVIDE=6, p_CLKOUT3_PHASE=0.0, o_CLKOUT3=mmcm_clk208,
+            ),
+            Instance("BUFG", i_I=mmcm_sys5x, o_O=self.cd_sys5x.clk),
+            Instance("BUFG", i_I=mmcm_sys, o_O=self.cd_sys.clk),
+            Instance("BUFG", i_I=mmcm_sys4x, o_O=self.cd_sys4x.clk),
+            Instance("BUFG", i_I=mmcm_clk208, o_O=self.cd_clk200.clk),
         ]
 
-        self.comb += self.current_clock.status.eq(self.clk_sw_fsm.o_clk_sw)
+        if ext_async_rst is not None:
+            self.specials += [
+                AsyncResetSynchronizer(self.cd_sys, ~self.mmcm_locked | ext_async_rst),
+                AsyncResetSynchronizer(self.cd_clk200, ~self.mmcm_locked | ext_async_rst),
+            ]
+        else:
+            self.specials += [
+                AsyncResetSynchronizer(self.cd_sys, ~self.mmcm_locked),
+                AsyncResetSynchronizer(self.cd_clk200, ~self.mmcm_locked),
+            ]
+
+        reset_counter = Signal(4, reset=15)
+        ic_reset = Signal(reset=1)
+        self.sync.clk200 += \
+            If(reset_counter != 0,
+                reset_counter.eq(reset_counter - 1)
+            ).Else(
+                ic_reset.eq(0)
+            )
+        self.specials += Instance("IDELAYCTRL", i_REFCLK=ClockSignal("clk200"), i_RST=ic_reset)
+
+        if clk_sw_status is None:
+            self.comb += self.current_clock.status.eq(self.clk_sw_fsm.o_clk_sw)
+        else:
+            self.comb += self.current_clock.status.eq(clk_sw_status)

src/libboard_artiq/Cargo.toml.tpl

@@ -10,6 +10,7 @@ name = "libboard_artiq"
 [features]
 target_zc706 = ["libboard_zynq/target_zc706", "libconfig/target_zc706"]
 target_kasli_soc = ["libboard_zynq/target_kasli_soc", "libconfig/target_kasli_soc"]
+calibrate_wrpll_skew = []
 
 [build-dependencies]
 build_zynq = { path = "../libbuild_zynq" }
@@ -24,8 +25,9 @@ nb = "1.0"
 void = { version = "1", default-features = false }
 
 io = { path = "../libio", features = ["byteorder"] }
-libboard_zynq = { git = "https://git.m-labs.hk/M-Labs/zynq-rs.git"}
-libregister = { git = "https://git.m-labs.hk/M-Labs/zynq-rs.git" }
-libconfig = { git = "https://git.m-labs.hk/M-Labs/zynq-rs.git", features = ["fat_lfn"] }
-libcortex_a9 = { git = "https://git.m-labs.hk/M-Labs/zynq-rs.git" }
-libasync = { git = "https://git.m-labs.hk/M-Labs/zynq-rs.git" }
+libboard_zynq = { path = "@@ZYNQ_RS@@/libboard_zynq" }
+libsupport_zynq = { path = "@@ZYNQ_RS@@/libsupport_zynq", default-features = false, features = ["alloc_core"] }
+libregister = { path = "@@ZYNQ_RS@@/libregister" }
+libconfig = { path = "@@ZYNQ_RS@@/libconfig", features = ["fat_lfn"] }
+libcortex_a9 = { path = "@@ZYNQ_RS@@/libcortex_a9" }
+libasync = { path = "@@ZYNQ_RS@@/libasync" }

src/libboard_artiq/src/drtio_eem.rs

@@ -0,0 +1,228 @@
+use embedded_hal::prelude::_embedded_hal_blocking_delay_DelayUs;
+use libboard_zynq::timer::GlobalTimer;
+use libconfig::Config;
+use libsupport_zynq::alloc::format;
+use log::{debug, error, info};
+
+use crate::pl;
+
+struct SerdesConfig {
+    pub delay: [u8; 4],
+}
+
+impl SerdesConfig {
+    pub fn as_bytes(&self) -> &[u8] {
+        unsafe {
+            core::slice::from_raw_parts(
+                (self as *const SerdesConfig) as *const u8,
+                core::mem::size_of::<SerdesConfig>(),
+            )
+        }
+    }
+}
+
+fn select_lane(lane_no: u8) {
+    unsafe {
+        pl::csr::eem_transceiver::lane_sel_write(lane_no);
+    }
+}
+
+fn apply_delay(tap: u8, timer: &mut GlobalTimer) {
+    unsafe {
+        pl::csr::eem_transceiver::dly_cnt_in_write(tap);
+        pl::csr::eem_transceiver::dly_ld_write(1);
+        timer.delay_us(1);
+        assert!(tap as u8 == pl::csr::eem_transceiver::dly_cnt_out_read());
+    }
+}
+
+fn apply_config(config: &SerdesConfig, timer: &mut GlobalTimer) {
+    for lane_no in 0..4 {
+        select_lane(lane_no as u8);
+        apply_delay(config.delay[lane_no], timer);
+    }
+}
+
+unsafe fn assign_delay(timer: &mut GlobalTimer) -> SerdesConfig {
+    // Select an appropriate delay for lane 0
+    select_lane(0);
+
+    //
+
+    let mut best_dly = None;
+
+    loop {
+        let mut prev = None;
+        for curr_dly in 0..32 {
+            //let read_align = read_align_fn(curr_dly, timer);
+            let curr_low_rate = read_align(curr_dly, timer);
+
+            if let Some(prev_low_rate) = prev {
+                // This is potentially a crossover position
+                if prev_low_rate <= curr_low_rate && curr_low_rate >= 0.5 {
+                    let prev_dev = 0.5 - prev_low_rate;
+                    let curr_dev = curr_low_rate - 0.5;
+                    let selected_idx = if prev_dev < curr_dev { curr_dly - 1 } else { curr_dly };
+
+                    // The setup setup/hold calibration timing (even with
+                    // tolerance) might be invalid in other lanes due to skew.
+                    // 5 taps is very conservative, generally it is 1 or 2
+                    if selected_idx < 5 {
+                        prev = None;
+                        continue;
+                    } else {
+                        best_dly = Some(selected_idx);
+                        break;
+                    }
+                }
+            }
+
+            // Only rising slope from <= 0.5 can result in a rising low rate
+            // crossover at 50%.
+            if curr_low_rate <= 0.5 {
+                prev = Some(curr_low_rate);
+            }
+        }
+
+        if best_dly.is_none() {
+            error!("setup/hold timing calibration failed, retry in 1s...");
+            timer.delay_us(1_000_000);
+        } else {
+            break;
+        }
+    }
+
+    let best_dly = best_dly.unwrap();
+
+    apply_delay(best_dly, timer);
+    let mut delay_list = [best_dly; 4];
+
+    // Assign delay for other lanes
+    for lane_no in 1..=3 {
+        select_lane(lane_no as u8);
+
+        let mut min_deviation = 0.5;
+        let mut min_idx = 0;
+        for dly_delta in -3..=3 {
+            let index = (best_dly as isize + dly_delta) as u8;
+            let low_rate = read_align(index, timer);
+            // abs() from f32 is not available in core library
+            let deviation = if low_rate < 0.5 { 0.5 - low_rate } else { low_rate - 0.5 };
+
+            if deviation < min_deviation {
+                min_deviation = deviation;
+                min_idx = index;
+            }
+        }
+
+        apply_delay(min_idx, timer);
+        delay_list[lane_no] = min_idx;
+    }
+
+    debug!("setup/hold timing calibration: {:?}", delay_list);
+
+    SerdesConfig { delay: delay_list }
+}
+
+fn read_align(dly: u8, timer: &mut GlobalTimer) -> f32 {
+    unsafe {
+        apply_delay(dly, timer);
+        pl::csr::eem_transceiver::counter_reset_write(1);
+
+        pl::csr::eem_transceiver::counter_enable_write(1);
+        timer.delay_us(2000);
+        pl::csr::eem_transceiver::counter_enable_write(0);
+
+        let (high, low) = (
+            pl::csr::eem_transceiver::counter_high_count_read(),
+            pl::csr::eem_transceiver::counter_low_count_read(),
+        );
+        if pl::csr::eem_transceiver::counter_overflow_read() == 1 {
+            panic!("Unexpected phase detector counter overflow");
+        }
+
+        low as f32 / (low + high) as f32
+    }
+}
+
+unsafe fn align_comma(timer: &mut GlobalTimer) {
+    loop {
+        for slip in 1..=10 {
+            // The soft transceiver has 2 8b10b decoders, which receives lane
+            // 0/1 and lane 2/3 respectively. The decoder are time-multiplexed
+            // to decode exactly 1 lane each sysclk cycle.
+            //
+            // The decoder decodes lane 0/2 data on odd sysclk cycles, buffer
+            // on even cycles, and vice versa for lane 1/3. Data/Clock latency
+            // could change timing. The extend bit flips the decoding timing,
+            // so lane 0/2 data are decoded on even cycles, and lane 1/3 data
+            // are decoded on odd cycles.
+            //
+            // This is needed because transmitting/receiving a 8b10b character
+            // takes 2 sysclk cycles. Adjusting bitslip only via ISERDES
+            // limits the range to 1 cycle. The wordslip bit extends the range
+            // to 2 sysclk cycles.
+            pl::csr::eem_transceiver::wordslip_write((slip > 5) as u8);
+
+            // Apply a double bitslip since the ISERDES is 2x oversampled.
+            // Bitslip is used for comma alignment purposes once setup/hold
+            // timing is met.
+            pl::csr::eem_transceiver::bitslip_write(1);
+            pl::csr::eem_transceiver::bitslip_write(1);
+            timer.delay_us(1);
+
+            pl::csr::eem_transceiver::comma_align_reset_write(1);
+            timer.delay_us(100);
+
+            if pl::csr::eem_transceiver::comma_read() == 1 {
+                debug!("comma alignment completed after {} bitslips", slip);
+                return;
+            }
+        }
+
+        error!("comma alignment failed, retrying in 1s...");
+        timer.delay_us(1_000_000);
+    }
+}
+
+pub fn init(timer: &mut GlobalTimer, cfg: &Config) {
+    for trx_no in 0..pl::csr::CONFIG_EEM_DRTIO_COUNT {
+        unsafe {
+            pl::csr::eem_transceiver::transceiver_sel_write(trx_no as u8);
+        }
+
+        let key = format!("eem_drtio_delay{}", trx_no);
+
+        let cfg_read = cfg.read(&key);
+        match cfg_read {
+            Ok(record) => {
+                info!("loading calibrated timing values from sd card");
+                unsafe {
+                    apply_config(&*(record.as_ptr() as *const SerdesConfig), timer);
+                }
+            }
+            Err(_) => {
+                info!("calibrating...");
+                let config = unsafe { assign_delay(timer) };
+
+                match cfg.write(&key, config.as_bytes().to_vec()) {
+                    Ok(()) => {
+                        info!("storing calibration timing values into sd card");
+                    }
+                    Err(e) => {
+                        error!(
+                            "calibration successful but calibration timing values cannot be stored into sd card. \
+                             Error:{}",
+                            e
+                        );
+                    }
+                };
+            }
+        }
+
+        unsafe {
+            align_comma(timer);
+            pl::csr::eem_transceiver::rx_ready_write(1);
+        }
+    }
+}

src/libboard_artiq/src/drtioaux.rs

@@ -74,12 +74,15 @@ where F: FnOnce(&[u8]) -> Result<T, Error> {
     let linkidx = linkno as usize;
     unsafe {
         if (DRTIOAUX[linkidx].aux_rx_present_read)() == 1 {
-            let ptr = (DRTIOAUX_MEM[linkidx].base + DRTIOAUX_MEM[linkidx].size / 2) as *mut u32;
-            let len = (DRTIOAUX[linkidx].aux_rx_length_read)() as usize;
+            let read_ptr = (DRTIOAUX[linkidx].aux_read_pointer_read)() as usize;
+            let ptr = (DRTIOAUX_MEM[linkidx].base + DRTIOAUX_MEM[linkidx].size / 2 + read_ptr * 0x400) as *mut u32;
             // work buffer to accomodate axi burst reads
-            let mut buf: [u8; 1024] = [0; 1024];
-            copy_work_buffer(ptr, buf.as_mut_ptr() as *mut u32, len as isize);
-            let result = f(&buf[0..len]);
+            // buffer at maximum proto packet size, not maximum gateware supported size
+            // to minimize copying time
+            const LEN: usize = 512;
+            let mut buf: [u8; LEN] = [0; LEN];
+            copy_work_buffer(ptr, buf.as_mut_ptr() as *mut u32, LEN as isize);
+            let result = f(&buf);
             (DRTIOAUX[linkidx].aux_rx_present_write)(1);
             Ok(Some(result?))
         } else {
@@ -100,15 +103,15 @@ pub fn recv(linkno: u8) -> Result<Option<Packet>, Error> {
 
         let mut reader = Cursor::new(buffer);
 
-        let checksum_at = buffer.len() - 4;
+        let packet = Packet::read_from(&mut reader)?;
+        let padding = (12 - (reader.position() % 8)) % 8;
+        let checksum_at = reader.position() + padding;
         let checksum = crc::crc32::checksum_ieee(&reader.get_ref()[0..checksum_at]);
         reader.set_position(checksum_at);
         if reader.read_u32()? != checksum {
             return Err(Error::CorruptedPacket);
         }
-        reader.set_position(0);
-
-        Ok(Packet::read_from(&mut reader)?)
+        Ok(packet)
     })
 }
 
@@ -130,10 +133,9 @@ where F: FnOnce(&mut [u8]) -> Result<usize, Error> {
     unsafe {
         while (DRTIOAUX[linkno].aux_tx_read)() != 0 {}
         let ptr = DRTIOAUX_MEM[linkno].base as *mut u32;
-        let len = DRTIOAUX_MEM[linkno].size / 2;
         // work buffer, works with unaligned mem access
         let mut buf: [u8; 1024] = [0; 1024];
-        let len = f(&mut buf[0..len])?;
+        let len = f(&mut buf)?;
         copy_work_buffer(buf.as_mut_ptr() as *mut u32, ptr, len as isize);
         (DRTIOAUX[linkno].aux_tx_length_write)(len as u16);
         (DRTIOAUX[linkno].aux_tx_write)(1);

src/libboard_artiq/src/drtioaux_async.rs

@@ -38,12 +38,15 @@ where F: FnOnce(&[u8]) -> Result<T, Error> {
     let linkidx = linkno as usize;
     unsafe {
         if (DRTIOAUX[linkidx].aux_rx_present_read)() == 1 {
-            let ptr = (DRTIOAUX_MEM[linkidx].base + DRTIOAUX_MEM[linkidx].size / 2) as *mut u32;
-            let len = (DRTIOAUX[linkidx].aux_rx_length_read)() as usize;
+            let read_ptr = (DRTIOAUX[linkidx].aux_read_pointer_read)() as usize;
+            let ptr = (DRTIOAUX_MEM[linkidx].base + DRTIOAUX_MEM[linkidx].size / 2 + read_ptr * 0x400) as *mut u32;
             // work buffer to accomodate axi burst reads
-            let mut buf: [u8; 1024] = [0; 1024];
-            copy_work_buffer(ptr, buf.as_mut_ptr() as *mut u32, len as isize);
-            let result = f(&buf[0..len]);
+            // buffer at maximum proto packet size, not maximum gateware supported size
+            // to minimize required copying time
+            const LEN: usize = 512;
+            let mut buf: [u8; LEN] = [0; LEN];
+            copy_work_buffer(ptr, buf.as_mut_ptr() as *mut u32, LEN as isize);
+            let result = f(&buf);
             (DRTIOAUX[linkidx].aux_rx_present_write)(1);
             Ok(Some(result?))
         } else {
@@ -64,15 +67,15 @@ pub async fn recv(linkno: u8) -> Result<Option<Packet>, Error> {
 
         let mut reader = Cursor::new(buffer);
 
-        let checksum_at = buffer.len() - 4;
+        let packet = Packet::read_from(&mut reader)?;
+        let padding = (12 - (reader.position() % 8)) % 8;
+        let checksum_at = reader.position() + padding;
         let checksum = crc::crc32::checksum_ieee(&reader.get_ref()[0..checksum_at]);
         reader.set_position(checksum_at);
         if reader.read_u32()? != checksum {
             return Err(Error::CorruptedPacket);
         }
-        reader.set_position(0);
-
-        Ok(Packet::read_from(&mut reader)?)
+        Ok(packet)
     })
     .await
 }
@@ -103,10 +106,9 @@ where F: FnOnce(&mut [u8]) -> Result<usize, Error> {
     unsafe {
         let _ = block_async!(tx_ready(linkno)).await;
         let ptr = DRTIOAUX_MEM[linkno].base as *mut u32;
-        let len = DRTIOAUX_MEM[linkno].size / 2;
         // work buffer, works with unaligned mem access
         let mut buf: [u8; 1024] = [0; 1024];
-        let len = f(&mut buf[0..len])?;
+        let len = f(&mut buf)?;
         copy_work_buffer(buf.as_mut_ptr() as *mut u32, ptr, len as isize);
         (DRTIOAUX[linkno].aux_tx_length_write)(len as u16);
         (DRTIOAUX[linkno].aux_tx_write)(1);

src/libboard_artiq/src/drtioaux_proto.rs

@@ -1,8 +1,11 @@
 use core_io::{Error as IoError, Read, Write};
 use io::proto::{ProtoRead, ProtoWrite};
 
-/* 512 (max size) - 4 (CRC) - 1 (packet ID) - 1 (destination) - 4 (trace ID) - 1 (last) - 2 (length) */
-pub const DMA_TRACE_MAX_SIZE: usize = 499;
+// maximum size of arbitrary payloads
+// used by satellite -> master analyzer, subkernel exceptions
+pub const SAT_PAYLOAD_MAX_SIZE: usize  = /*max size*/512 - /*CRC*/4 - /*packet ID*/1 - /*last*/1 - /*length*/2;
+// used by DDMA, subkernel program data (need to provide extra ID and destination)
+pub const MASTER_PAYLOAD_MAX_SIZE: usize = SAT_PAYLOAD_MAX_SIZE - /*source*/1 - /*destination*/1 - /*ID*/4;
 
 #[derive(Debug)]
 pub enum Error {
@@ -16,6 +19,46 @@ impl From<IoError> for Error {
     }
 }
 
+#[derive(PartialEq, Clone, Copy, Debug)]
+#[repr(u8)]
+pub enum PayloadStatus {
+    Middle = 0,
+    First = 1,
+    Last = 2,
+    FirstAndLast = 3,
+}
+
+impl From<u8> for PayloadStatus {
+    fn from(value: u8) -> PayloadStatus {
+        match value {
+            0 => PayloadStatus::Middle,
+            1 => PayloadStatus::First,
+            2 => PayloadStatus::Last,
+            3 => PayloadStatus::FirstAndLast,
+            _ => unreachable!(),
+        }
+    }
+}
+
+impl PayloadStatus {
+    pub fn is_first(self) -> bool {
+        self == PayloadStatus::First || self == PayloadStatus::FirstAndLast
+    }
+
+    pub fn is_last(self) -> bool {
+        self == PayloadStatus::Last || self == PayloadStatus::FirstAndLast
+    }
+
+    pub fn from_status(first: bool, last: bool) -> PayloadStatus {
+        match (first, last) {
+            (true, true) => PayloadStatus::FirstAndLast,
+            (true, false) => PayloadStatus::First,
+            (false, true) => PayloadStatus::Last,
+            (false, false) => PayloadStatus::Middle,
+        }
+    }
+}
+
 #[derive(PartialEq, Debug)]
 pub enum Packet {
     EchoRequest,
@@ -136,38 +179,110 @@ pub enum Packet {
         succeeded: bool,
     },
 
-    DmaAddTraceRequest {
+    AnalyzerHeaderRequest {
         destination: u8,
-        id: u32,
+    },
+    AnalyzerHeader {
+        sent_bytes: u32,
+        total_byte_count: u64,
+        overflow_occurred: bool,
+    },
+    AnalyzerDataRequest {
+        destination: u8,
+    },
+    AnalyzerData {
         last: bool,
         length: u16,
-        trace: [u8; DMA_TRACE_MAX_SIZE],
+        data: [u8; SAT_PAYLOAD_MAX_SIZE],
+    },
+
+    DmaAddTraceRequest {
+        source: u8,
+        destination: u8,
+        id: u32,
+        status: PayloadStatus,
+        length: u16,
+        trace: [u8; MASTER_PAYLOAD_MAX_SIZE],
     },
     DmaAddTraceReply {
+        source: u8,
+        destination: u8,
+        id: u32,
         succeeded: bool,
     },
     DmaRemoveTraceRequest {
+        source: u8,
         destination: u8,
         id: u32,
     },
     DmaRemoveTraceReply {
+        destination: u8,
         succeeded: bool,
     },
     DmaPlaybackRequest {
+        source: u8,
         destination: u8,
         id: u32,
         timestamp: u64,
     },
     DmaPlaybackReply {
+        destination: u8,
         succeeded: bool,
     },
     DmaPlaybackStatus {
+        source: u8,
         destination: u8,
         id: u32,
         error: u8,
         channel: u32,
         timestamp: u64,
     },
+
+    SubkernelAddDataRequest {
+        destination: u8,
+        id: u32,
+        status: PayloadStatus,
+        length: u16,
+        data: [u8; MASTER_PAYLOAD_MAX_SIZE],
+    },
+    SubkernelAddDataReply {
+        succeeded: bool,
+    },
+    SubkernelLoadRunRequest {
+        source: u8,
+        destination: u8,
+        id: u32,
+        run: bool,
+    },
+    SubkernelLoadRunReply {
+        destination: u8,
+        succeeded: bool,
+    },
+    SubkernelFinished {
+        destination: u8,
+        id: u32,
+        with_exception: bool,
+        exception_src: u8,
+    },
+    SubkernelExceptionRequest {
+        destination: u8,
+    },
+    SubkernelException {
+        last: bool,
+        length: u16,
+        data: [u8; SAT_PAYLOAD_MAX_SIZE],
+    },
+    SubkernelMessage {
+        source: u8,
+        destination: u8,
+        id: u32,
+        status: PayloadStatus,
+        length: u16,
+        data: [u8; MASTER_PAYLOAD_MAX_SIZE],
+    },
+    SubkernelMessageAck {
+        destination: u8,
+    },
 }
 
 impl Packet {
@@ -298,40 +413,73 @@ impl Packet {
                 succeeded: reader.read_bool()?,
             },
 
-            0xb0 => {
-                let destination = reader.read_u8()?;
-                let id = reader.read_u32()?;
+            0xa0 => Packet::AnalyzerHeaderRequest {
+                destination: reader.read_u8()?,
+            },
+            0xa1 => Packet::AnalyzerHeader {
+                sent_bytes: reader.read_u32()?,
+                total_byte_count: reader.read_u64()?,
+                overflow_occurred: reader.read_bool()?,
+            },
+            0xa2 => Packet::AnalyzerDataRequest {
+                destination: reader.read_u8()?,
+            },
+            0xa3 => {
                 let last = reader.read_bool()?;
                 let length = reader.read_u16()?;
-                let mut trace: [u8; DMA_TRACE_MAX_SIZE] = [0; DMA_TRACE_MAX_SIZE];
+                let mut data: [u8; SAT_PAYLOAD_MAX_SIZE] = [0; SAT_PAYLOAD_MAX_SIZE];
+                reader.read_exact(&mut data[0..length as usize])?;
+                Packet::AnalyzerData {
+                    last: last,
+                    length: length,
+                    data: data,
+                }
+            }
+
+            0xb0 => {
+                let source = reader.read_u8()?;
+                let destination = reader.read_u8()?;
+                let id = reader.read_u32()?;
+                let status = reader.read_u8()?;
+                let length = reader.read_u16()?;
+                let mut trace: [u8; MASTER_PAYLOAD_MAX_SIZE] = [0; MASTER_PAYLOAD_MAX_SIZE];
                 reader.read_exact(&mut trace[0..length as usize])?;
                 Packet::DmaAddTraceRequest {
+                    source: source,
                     destination: destination,
                     id: id,
-                    last: last,
+                    status: PayloadStatus::from(status),
                     length: length as u16,
                     trace: trace,
                 }
             }
             0xb1 => Packet::DmaAddTraceReply {
+                source: reader.read_u8()?,
+                destination: reader.read_u8()?,
+                id: reader.read_u32()?,
                 succeeded: reader.read_bool()?,
             },
             0xb2 => Packet::DmaRemoveTraceRequest {
+                source: reader.read_u8()?,
                 destination: reader.read_u8()?,
                 id: reader.read_u32()?,
             },
             0xb3 => Packet::DmaRemoveTraceReply {
+                destination: reader.read_u8()?,
                 succeeded: reader.read_bool()?,
             },
             0xb4 => Packet::DmaPlaybackRequest {
+                source: reader.read_u8()?,
                 destination: reader.read_u8()?,
                 id: reader.read_u32()?,
                 timestamp: reader.read_u64()?,
             },
             0xb5 => Packet::DmaPlaybackReply {
+                destination: reader.read_u8()?,
                 succeeded: reader.read_bool()?,
             },
             0xb6 => Packet::DmaPlaybackStatus {
+                source: reader.read_u8()?,
                 destination: reader.read_u8()?,
                 id: reader.read_u32()?,
                 error: reader.read_u8()?,
@@ -339,6 +487,75 @@ impl Packet {
                 timestamp: reader.read_u64()?,
             },
 
+            0xc0 => {
+                let destination = reader.read_u8()?;
+                let id = reader.read_u32()?;
+                let status = PayloadStatus::from(reader.read_u8()?);
+                let length = reader.read_u16()?;
+                let mut data: [u8; MASTER_PAYLOAD_MAX_SIZE] = [0; MASTER_PAYLOAD_MAX_SIZE];
+                reader.read_exact(&mut data[0..length as usize])?;
+                Packet::SubkernelAddDataRequest {
+                    destination: destination,
+                    id: id,
+                    status: status,
+                    length: length as u16,
+                    data: data,
+                }
+            }
+            0xc1 => Packet::SubkernelAddDataReply {
+                succeeded: reader.read_bool()?,
+            },
+            0xc4 => Packet::SubkernelLoadRunRequest {
+                source: reader.read_u8()?,
+                destination: reader.read_u8()?,
+                id: reader.read_u32()?,
+                run: reader.read_bool()?,
+            },
+            0xc5 => Packet::SubkernelLoadRunReply {
+                destination: reader.read_u8()?,
+                succeeded: reader.read_bool()?,
+            },
+            0xc8 => Packet::SubkernelFinished {
+                destination: reader.read_u8()?,
+                id: reader.read_u32()?,
+                with_exception: reader.read_bool()?,
+                exception_src: reader.read_u8()?,
+            },
+            0xc9 => Packet::SubkernelExceptionRequest {
+                destination: reader.read_u8()?,
+            },
+            0xca => {
+                let last = reader.read_bool()?;
+                let length = reader.read_u16()?;
+                let mut data: [u8; SAT_PAYLOAD_MAX_SIZE] = [0; SAT_PAYLOAD_MAX_SIZE];
+                reader.read_exact(&mut data[0..length as usize])?;
+                Packet::SubkernelException {
+                    last: last,
+                    length: length,
+                    data: data,
+                }
+            }
+            0xcb => {
+                let source = reader.read_u8()?;
+                let destination = reader.read_u8()?;
+                let id = reader.read_u32()?;
+                let status = reader.read_u8()?;
+                let length = reader.read_u16()?;
+                let mut data: [u8; MASTER_PAYLOAD_MAX_SIZE] = [0; MASTER_PAYLOAD_MAX_SIZE];
+                reader.read_exact(&mut data[0..length as usize])?;
+                Packet::SubkernelMessage {
+                    source: source,
+                    destination: destination,
+                    id: id,
+                    status: PayloadStatus::from(status),
+                    length: length as u16,
+                    data: data,
+                }
+            }
+            0xcc => Packet::SubkernelMessageAck {
+                destination: reader.read_u8()?,
+            },
+
             ty => return Err(Error::UnknownPacket(ty)),
         })
     }
@@ -526,50 +743,95 @@ impl Packet {
                 writer.write_bool(succeeded)?;
             }
 
+            Packet::AnalyzerHeaderRequest { destination } => {
+                writer.write_u8(0xa0)?;
+                writer.write_u8(destination)?;
+            }
+            Packet::AnalyzerHeader {
+                sent_bytes,
+                total_byte_count,
+                overflow_occurred,
+            } => {
+                writer.write_u8(0xa1)?;
+                writer.write_u32(sent_bytes)?;
+                writer.write_u64(total_byte_count)?;
+                writer.write_bool(overflow_occurred)?;
+            }
+            Packet::AnalyzerDataRequest { destination } => {
+                writer.write_u8(0xa2)?;
+                writer.write_u8(destination)?;
+            }
+            Packet::AnalyzerData { last, length, data } => {
+                writer.write_u8(0xa3)?;
+                writer.write_bool(last)?;
+                writer.write_u16(length)?;
+                writer.write_all(&data[0..length as usize])?;
+            }
+
             Packet::DmaAddTraceRequest {
+                source,
                 destination,
                 id,
-                last,
+                status,
                 trace,
                 length,
             } => {
                 writer.write_u8(0xb0)?;
+                writer.write_u8(source)?;
                 writer.write_u8(destination)?;
                 writer.write_u32(id)?;
-                writer.write_bool(last)?;
+                writer.write_u8(status as u8)?;
                 // trace may be broken down to fit within drtio aux memory limit
                 // will be reconstructed by satellite
                 writer.write_u16(length)?;
                 writer.write_all(&trace[0..length as usize])?;
             }
-            Packet::DmaAddTraceReply { succeeded } => {
+            Packet::DmaAddTraceReply {
+                source,
+                destination,
+                id,
+                succeeded,
+            } => {
                 writer.write_u8(0xb1)?;
+                writer.write_u8(source)?;
+                writer.write_u8(destination)?;
+                writer.write_u32(id)?;
                 writer.write_bool(succeeded)?;
             }
-            Packet::DmaRemoveTraceRequest { destination, id } => {
+            Packet::DmaRemoveTraceRequest {
+                source,
+                destination,
+                id,
+            } => {
                 writer.write_u8(0xb2)?;
+                writer.write_u8(source)?;
                 writer.write_u8(destination)?;
                 writer.write_u32(id)?;
             }
-            Packet::DmaRemoveTraceReply { succeeded } => {
+            Packet::DmaRemoveTraceReply { destination, succeeded } => {
                 writer.write_u8(0xb3)?;
+                writer.write_u8(destination)?;
                 writer.write_bool(succeeded)?;
             }
             Packet::DmaPlaybackRequest {
+                source,
                 destination,
                 id,
                 timestamp,
             } => {
                 writer.write_u8(0xb4)?;
+                writer.write_u8(source)?;
                 writer.write_u8(destination)?;
                 writer.write_u32(id)?;
                 writer.write_u64(timestamp)?;
             }
-            Packet::DmaPlaybackReply { succeeded } => {
+            Packet::DmaPlaybackReply { destination, succeeded } => {
                 writer.write_u8(0xb5)?;
+                writer.write_u8(destination)?;
                 writer.write_bool(succeeded)?;
             }
             Packet::DmaPlaybackStatus {
+                source,
                 destination,
                 id,
                 error,
@@ -577,13 +839,127 @@ impl Packet {
                 timestamp,
             } => {
                 writer.write_u8(0xb6)?;
+                writer.write_u8(source)?;
                 writer.write_u8(destination)?;
                 writer.write_u32(id)?;
                 writer.write_u8(error)?;
                 writer.write_u32(channel)?;
                 writer.write_u64(timestamp)?;
             }
+
+            Packet::SubkernelAddDataRequest {
+                destination,
+                id,
+                status,
+                data,
+                length,
+            } => {
+                writer.write_u8(0xc0)?;
+                writer.write_u8(destination)?;
+                writer.write_u32(id)?;
+                writer.write_u8(status as u8)?;
+                writer.write_u16(length)?;
+                writer.write_all(&data[0..length as usize])?;
+            }
+            Packet::SubkernelAddDataReply { succeeded } => {
+                writer.write_u8(0xc1)?;
+                writer.write_bool(succeeded)?;
+            }
+            Packet::SubkernelLoadRunRequest {
+                source,
+                destination,
+                id,
+                run,
+            } => {
+                writer.write_u8(0xc4)?;
+                writer.write_u8(source)?;
+                writer.write_u8(destination)?;
+                writer.write_u32(id)?;
+                writer.write_bool(run)?;
+            }
+            Packet::SubkernelLoadRunReply { destination, succeeded } => {
+                writer.write_u8(0xc5)?;
+                writer.write_u8(destination)?;
+                writer.write_bool(succeeded)?;
+            }
+            Packet::SubkernelFinished {
+                destination,
+                id,
+                with_exception,
+                exception_src,
+            } => {
+                writer.write_u8(0xc8)?;
+                writer.write_u8(destination)?;
+                writer.write_u32(id)?;
+                writer.write_bool(with_exception)?;
+                writer.write_u8(exception_src)?;
+            }
+            Packet::SubkernelExceptionRequest { destination } => {
+                writer.write_u8(0xc9)?;
+                writer.write_u8(destination)?;
+            }
+            Packet::SubkernelException { last, length, data } => {
+                writer.write_u8(0xca)?;
+                writer.write_bool(last)?;
+                writer.write_u16(length)?;
+                writer.write_all(&data[0..length as usize])?;
+            }
+            Packet::SubkernelMessage {
+                source,
+                destination,
+                id,
+                status,
+                data,
+                length,
+            } => {
+                writer.write_u8(0xcb)?;
+                writer.write_u8(source)?;
+                writer.write_u8(destination)?;
+                writer.write_u32(id)?;
+                writer.write_u8(status as u8)?;
+                writer.write_u16(length)?;
+                writer.write_all(&data[0..length as usize])?;
+            }
+            Packet::SubkernelMessageAck { destination } => {
+                writer.write_u8(0xcc)?;
+                writer.write_u8(destination)?;
+            }
         }
         Ok(())
     }
+
+    pub fn routable_destination(&self) -> Option<u8> {
+        // only for packets that could be re-routed, not only forwarded
+        match self {
+            Packet::DmaAddTraceRequest { destination, .. } => Some(*destination),
+            Packet::DmaAddTraceReply { destination, .. } => Some(*destination),
+            Packet::DmaRemoveTraceRequest { destination, .. } => Some(*destination),
+            Packet::DmaRemoveTraceReply { destination, .. } => Some(*destination),
+            Packet::DmaPlaybackRequest { destination, .. } => Some(*destination),
+            Packet::DmaPlaybackReply { destination, .. } => Some(*destination),
+            Packet::SubkernelLoadRunRequest { destination, .. } => Some(*destination),
+            Packet::SubkernelLoadRunReply { destination, .. } => Some(*destination),
+            Packet::SubkernelMessage { destination, .. } => Some(*destination),
+            Packet::SubkernelMessageAck { destination } => Some(*destination),
+            Packet::DmaPlaybackStatus { destination, .. } => Some(*destination),
+            Packet::SubkernelFinished { destination, .. } => Some(*destination),
+            _ => None,
+        }
+    }
+
+    pub fn expects_response(&self) -> bool {
+        // returns true if the routable packet should elicit a response
+        // e.g. reply, ACK packets end a conversation,
+        // and firmware should not wait for response
+        match self {
+            Packet::DmaAddTraceReply { .. }
+            | Packet::DmaRemoveTraceReply { .. }
+            | Packet::DmaPlaybackReply { .. }
+            | Packet::SubkernelLoadRunReply { .. }
+            | Packet::SubkernelMessageAck { .. }
+            | Packet::DmaPlaybackStatus { .. }
+            | Packet::SubkernelFinished { .. } => false,
+            _ => true,
+        }
+    }
 }

src/libboard_artiq/src/fiq.rs

@@ -0,0 +1,22 @@
+use libboard_zynq::{println, stdio};
+use libcortex_a9::{interrupt_handler, regs::MPIDR};
+use libregister::RegisterR;
+
+#[cfg(has_si549)]
+use crate::si549;
+
+interrupt_handler!(FIQ, fiq, __irq_stack0_start, __irq_stack1_start, {
+    match MPIDR.read().cpu_id() {
+        0 => {
+            // nFIQ is driven directly and bypass GIC
+            #[cfg(has_si549)]
+            si549::wrpll::interrupt_handler();
+            return;
+        }
+        _ => {}
+    };
+
+    stdio::drop_uart();
+    println!("FIQ");
+    loop {}
+});

src/libboard_artiq/src/grabber.rs

@@ -0,0 +1,163 @@
+use log::info;
+
+use crate::pl::csr;
+
+#[derive(PartialEq, Clone, Copy)]
+enum State {
+    Reset,
+    ExitReset,
+    Lock,
+    Align,
+    Watch,
+}
+
+#[derive(Clone, Copy)]
+struct Info {
+    state: State,
+    frame_size: (u16, u16),
+}
+
+static mut INFO: [Info; csr::GRABBER_LEN] = [Info {
+    state: State::Reset,
+    frame_size: (0, 0),
+}; csr::GRABBER_LEN];
+
+fn get_pll_reset(g: usize) -> bool {
+    unsafe { (csr::GRABBER[g].pll_reset_read)() != 0 }
+}
+
+fn set_pll_reset(g: usize, reset: bool) {
+    let val = if reset { 1 } else { 0 };
+    unsafe { (csr::GRABBER[g].pll_reset_write)(val) }
+}
+
+fn pll_locked(g: usize) -> bool {
+    unsafe { (csr::GRABBER[g].pll_locked_read)() != 0 }
+}
+
+fn clock_pattern_ok(g: usize) -> bool {
+    unsafe { (csr::GRABBER[g].clk_sampled_read)() == 0b1100011 }
+}
+
+fn clock_pattern_ok_filter(g: usize) -> bool {
+    for _ in 0..128 {
+        if !clock_pattern_ok(g) {
+            return false;
+        }
+    }
+    true
+}
+
+fn phase_shift(g: usize, direction: u8) {
+    unsafe {
+        (csr::GRABBER[g].phase_shift_write)(direction);
+        while (csr::GRABBER[g].phase_shift_done_read)() == 0 {}
+    }
+}
+
+fn clock_align(g: usize) -> bool {
+    while clock_pattern_ok_filter(g) {
+        phase_shift(g, 1);
+    }
+    phase_shift(g, 1);
+
+    let mut count = 0;
+    while !clock_pattern_ok_filter(g) {
+        phase_shift(g, 1);
+        count += 1;
+        if count > 1024 {
+            return false;
+        }
+    }
+
+    let mut window = 1;
+    phase_shift(g, 1);
+    while clock_pattern_ok_filter(g) {
+        phase_shift(g, 1);
+        window += 1;
+    }
+
+    for _ in 0..window / 2 {
+        phase_shift(g, 0);
+    }
+
+    true
+}
+
+fn get_last_pixels(g: usize) -> (u16, u16) {
+    unsafe { ((csr::GRABBER[g].last_x_read)(), (csr::GRABBER[g].last_y_read)()) }
+}
+
+fn get_video_clock(g: usize) -> u32 {
+    let freq_count = unsafe { (csr::GRABBER[g].freq_count_read)() } as u32;
+    2 * freq_count * (csr::CONFIG_CLOCK_FREQUENCY / 1000) / (511 * 1000)
+}
+
+pub fn tick() {
+    for g in 0..csr::GRABBER.len() {
+        let next = match unsafe { INFO[g].state } {
+            State::Reset => {
+                set_pll_reset(g, true);
+                unsafe {
+                    INFO[g].frame_size = (0, 0);
+                }
+                State::ExitReset
+            }
+            State::ExitReset => {
+                if get_pll_reset(g) {
+                    set_pll_reset(g, false);
+                    State::Lock
+                } else {
+                    State::ExitReset
+                }
+            }
+            State::Lock => {
+                if pll_locked(g) {
+                    info!("grabber{} locked: {}MHz", g, get_video_clock(g));
+                    State::Align
+                } else {
+                    State::Lock
+                }
+            }
+            State::Align => {
+                if pll_locked(g) {
+                    if clock_align(g) {
+                        info!("grabber{} alignment success", g);
+                        State::Watch
+                    } else {
+                        info!("grabber{} alignment failure", g);
+                        State::Reset
+                    }
+                } else {
+                    info!("grabber{} lock lost", g);
+                    State::Reset
+                }
+            }
+            State::Watch => {
+                if pll_locked(g) {
+                    if clock_pattern_ok(g) {
+                        let last_xy = get_last_pixels(g);
+                        if last_xy != unsafe { INFO[g].frame_size } {
+                            // x capture is on ~LVAL which is after
+                            // the last increment on DVAL
+                            // y capture is on ~FVAL which coincides with the
+                            // last increment on ~LVAL
+                            info!("grabber{} frame size: {}x{}", g, last_xy.0, last_xy.1 + 1);
+                            unsafe { INFO[g].frame_size = last_xy }
+                        }
+                        State::Watch
+                    } else {
+                        info!("grabber{} alignment lost", g);
+                        State::Reset
+                    }
+                } else {
+                    info!("grabber{} lock lost", g);
+                    State::Reset
+                }
+            }
+        };
+        unsafe {
+            INFO[g].state = next;
+        }
+    }
+}

src/libboard_artiq/src/io_expander.rs

@@ -1,6 +1,9 @@
 use libboard_zynq::i2c;
 use log::info;
 
+#[cfg(has_virtual_leds)]
+use crate::pl::csr;
+
 // Only the bare minimum registers. Bits/IO connections equivalent between IC types.
 struct Registers {
     // PCA9539 equivalent register names in comments
@@ -10,23 +13,55 @@ struct Registers {
     gpiob: u8,  // Output Port 1
 }
 
-pub struct IoExpander<'a> {
-    i2c: &'a mut i2c::I2c,
+//IO expanders pins
+const IODIR_OUT_SFP_TX_DISABLE: u8 = 0x02;
+const IODIR_OUT_SFP_LED: u8 = 0x40;
+#[cfg(hw_rev = "v1.0")]
+const IODIR_OUT_SFP0_LED: u8 = 0x40;
+#[cfg(hw_rev = "v1.1")]
+const IODIR_OUT_SFP0_LED: u8 = 0x80;
+#[cfg(has_si549)]
+const IODIR_CLK_SEL: u8 = 0x80; // out
+#[cfg(has_si5324)]
+const IODIR_CLK_SEL: u8 = 0x00; // in
+
+//IO expander port direction
+const IODIR0: [u8; 2] = [
+    0xFF & !IODIR_OUT_SFP_TX_DISABLE & !IODIR_OUT_SFP0_LED,
+    0xFF & !IODIR_OUT_SFP_TX_DISABLE & !IODIR_OUT_SFP_LED & !IODIR_CLK_SEL,
+];
+
+const IODIR1: [u8; 2] = [
+    0xFF & !IODIR_OUT_SFP_TX_DISABLE & !IODIR_OUT_SFP_LED,
+    0xFF & !IODIR_OUT_SFP_TX_DISABLE & !IODIR_OUT_SFP_LED,
+];
+
+pub struct IoExpander {
     address: u8,
+    #[cfg(has_virtual_leds)]
+    virtual_led_mapping: &'static [(u8, u8, u8)],
     iodir: [u8; 2],
     out_current: [u8; 2],
     out_target: [u8; 2],
     registers: Registers,
 }
 
-impl<'a> IoExpander<'a> {
-    pub fn new(i2c: &'a mut i2c::I2c, index: u8) -> Result<Self, &'static str> {
+impl IoExpander {
+    pub fn new(i2c: &mut i2c::I2c, index: u8) -> Result<Self, &'static str> {
+        #[cfg(all(hw_rev = "v1.0", has_virtual_leds))]
+        const VIRTUAL_LED_MAPPING0: [(u8, u8, u8); 2] = [(0, 0, 6), (1, 1, 6)];
+        #[cfg(all(hw_rev = "v1.1", has_virtual_leds))]
+        const VIRTUAL_LED_MAPPING0: [(u8, u8, u8); 2] = [(0, 0, 7), (1, 1, 6)];
+        #[cfg(has_virtual_leds)]
+        const VIRTUAL_LED_MAPPING1: [(u8, u8, u8); 2] = [(2, 0, 6), (3, 1, 6)];
+
         // Both expanders on SHARED I2C bus
         let mut io_expander = match index {
             0 => IoExpander {
-                i2c,
                 address: 0x40,
-                iodir: [0xff; 2],
+                #[cfg(has_virtual_leds)]
+                virtual_led_mapping: &VIRTUAL_LED_MAPPING0,
+                iodir: IODIR0,
                 out_current: [0; 2],
                 out_target: [0; 2],
                 registers: Registers {
@@ -37,9 +72,10 @@ impl<'a> IoExpander<'a> {
                 },
             },
             1 => IoExpander {
-                i2c,
                 address: 0x42,
-                iodir: [0xff; 2],
+                #[cfg(has_virtual_leds)]
+                virtual_led_mapping: &VIRTUAL_LED_MAPPING1,
+                iodir: IODIR1,
                 out_current: [0; 2],
                 out_target: [0; 2],
                 registers: Registers {
@@ -51,7 +87,7 @@ impl<'a> IoExpander<'a> {
             },
             _ => return Err("incorrect I/O expander index"),
         };
-        if !io_expander.check_ack()? {
+        if !io_expander.check_ack(i2c)? {
             info!("MCP23017 io expander {} not found. Checking for PCA9539.", index);
             io_expander.address += 0xa8; // translate to PCA9539 addresses (see schematic)
             io_expander.registers = Registers {
@@ -60,57 +96,58 @@ impl<'a> IoExpander<'a> {
                 gpioa: 0x02,
                 gpiob: 0x03,
             };
-            if !io_expander.check_ack()? {
+            if !io_expander.check_ack(i2c)? {
                 return Err("Neither MCP23017 nor PCA9539 io expander found.");
             };
         }
         Ok(io_expander)
     }
 
-    fn select(&mut self) -> Result<(), &'static str> {
-        self.i2c.pca954x_select(0x70, None)?;
-        self.i2c.pca954x_select(0x71, Some(3))?;
+    fn select(&self, i2c: &mut i2c::I2c) -> Result<(), &'static str> {
+        i2c.pca954x_select(0x70, None)?;
+        i2c.pca954x_select(0x71, Some(3))?;
         Ok(())
     }
 
-    fn write(&mut self, addr: u8, value: u8) -> Result<(), &'static str> {
-        self.i2c.start()?;
-        self.i2c.write(self.address)?;
-        self.i2c.write(addr)?;
-        self.i2c.write(value)?;
-        self.i2c.stop()?;
+    fn write(&self, i2c: &mut i2c::I2c, addr: u8, value: u8) -> Result<(), &'static str> {
+        i2c.start()?;
+        i2c.write(self.address)?;
+        i2c.write(addr)?;
+        i2c.write(value)?;
+        i2c.stop()?;
         Ok(())
     }
 
-    fn check_ack(&mut self) -> Result<bool, &'static str> {
+    fn check_ack(&self, i2c: &mut i2c::I2c) -> Result<bool, &'static str> {
         // Check for ack from io expander
-        self.select()?;
-        self.i2c.start()?;
-        let ack = self.i2c.write(self.address)?;
-        self.i2c.stop()?;
+        self.select(i2c)?;
+        i2c.start()?;
+        let ack = i2c.write(self.address)?;
+        i2c.stop()?;
         Ok(ack)
     }
 
-    fn update_iodir(&mut self) -> Result<(), &'static str> {
-        self.write(self.registers.iodira, self.iodir[0])?;
-        self.write(self.registers.iodirb, self.iodir[1])?;
+    fn update_iodir(&self, i2c: &mut i2c::I2c) -> Result<(), &'static str> {
+        self.write(i2c, self.registers.iodira, self.iodir[0])?;
+        self.write(i2c, self.registers.iodirb, self.iodir[1])?;
         Ok(())
     }
 
-    pub fn init(&mut self) -> Result<(), &'static str> {
-        self.select()?;
-        self.update_iodir()?;
+    pub fn init(&mut self, i2c: &mut i2c::I2c) -> Result<(), &'static str> {
+        self.select(i2c)?;
+
+        self.update_iodir(i2c)?;
 
         self.out_current[0] = 0x00;
-        self.write(self.registers.gpioa, 0x00)?;
+        self.write(i2c, self.registers.gpioa, 0x00)?;
         self.out_current[1] = 0x00;
-        self.write(self.registers.gpiob, 0x00)?;
+        self.write(i2c, self.registers.gpiob, 0x00)?;
         Ok(())
     }
 
-    pub fn set_oe(&mut self, port: u8, outputs: u8) -> Result<(), &'static str> {
+    pub fn set_oe(&mut self, i2c: &mut i2c::I2c, port: u8, outputs: u8) -> Result<(), &'static str> {
         self.iodir[port as usize] &= !outputs;
-        self.update_iodir()?;
+        self.update_iodir(i2c)?;
         Ok(())
     }
 
@@ -122,15 +159,21 @@ impl<'a> IoExpander<'a> {
         }
     }
 
-    pub fn service(&mut self) -> Result<(), &'static str> {
+    pub fn service(&mut self, i2c: &mut i2c::I2c) -> Result<(), &'static str> {
+        #[cfg(has_virtual_leds)]
+        for (led, port, bit) in self.virtual_led_mapping.iter() {
+            let level = unsafe { csr::virtual_leds::status_read() >> led & 1 };
+            self.set(*port, *bit, level != 0);
+        }
+
         if self.out_target != self.out_current {
-            self.select()?;
+            self.select(i2c)?;
             if self.out_target[0] != self.out_current[0] {
-                self.write(self.registers.gpioa, self.out_target[0])?;
+                self.write(i2c, self.registers.gpioa, self.out_target[0])?;
                 self.out_current[0] = self.out_target[0];
             }
             if self.out_target[1] != self.out_current[1] {
-                self.write(self.registers.gpiob, self.out_target[1])?;
+                self.write(i2c, self.registers.gpiob, self.out_target[1])?;
                 self.out_current[1] = self.out_target[1];
             }
         }

src/libboard_artiq/src/lib.rs

@@ -1,5 +1,7 @@
 #![no_std]
 #![feature(never_type)]
+#![feature(naked_functions)]
+#![feature(asm)]
 
 extern crate core_io;
 extern crate crc;
@@ -19,6 +21,7 @@ pub mod drtioaux;
 #[cfg(has_drtio)]
 pub mod drtioaux_async;
 pub mod drtioaux_proto;
+pub mod fiq;
 #[cfg(feature = "target_kasli_soc")]
 pub mod io_expander;
 pub mod logger;
@@ -29,9 +32,14 @@ pub mod mem;
 #[rustfmt::skip]
 #[path = "../../../build/pl.rs"]
 pub mod pl;
+#[cfg(has_drtio_eem)]
+pub mod drtio_eem;
+#[cfg(has_grabber)]
+pub mod grabber;
 #[cfg(has_si5324)]
 pub mod si5324;
-
+#[cfg(has_si549)]
+pub mod si549;
 use core::{cmp, str};
 
 pub fn identifier_read(buf: &mut [u8]) -> &str {

src/libboard_artiq/src/si549.rs

@@ -0,0 +1,816 @@
+use embedded_hal::prelude::_embedded_hal_blocking_delay_DelayUs;
+use libboard_zynq::timer::GlobalTimer;
+use log::info;
+
+use crate::pl::csr;
+
+#[cfg(feature = "target_kasli_soc")]
+const ADDRESS: u8 = 0x67;
+
+const ADPLL_MAX: i32 = (950.0 / 0.0001164) as i32;
+
+pub struct DividerConfig {
+    pub hsdiv: u16,
+    pub lsdiv: u8,
+    pub fbdiv: u64,
+}
+
+pub struct FrequencySetting {
+    pub main: DividerConfig,
+    pub helper: DividerConfig,
+}
+
+mod i2c {
+    use super::*;
+
+    #[derive(Clone, Copy)]
+    pub enum DCXO {
+        Main,
+        Helper,
+    }
+
+    fn half_period(timer: &mut GlobalTimer) {
+        timer.delay_us(1)
+    }
+
+    fn sda_i(dcxo: DCXO) -> bool {
+        match dcxo {
+            DCXO::Main => unsafe { csr::wrpll::main_dcxo_sda_in_read() == 1 },
+            DCXO::Helper => unsafe { csr::wrpll::helper_dcxo_sda_in_read() == 1 },
+        }
+    }
+
+    fn sda_oe(dcxo: DCXO, oe: bool) {
+        let val = if oe { 1 } else { 0 };
+        match dcxo {
+            DCXO::Main => unsafe { csr::wrpll::main_dcxo_sda_oe_write(val) },
+            DCXO::Helper => unsafe { csr::wrpll::helper_dcxo_sda_oe_write(val) },
+        };
+    }
+
+    fn sda_o(dcxo: DCXO, o: bool) {
+        let val = if o { 1 } else { 0 };
+        match dcxo {
+            DCXO::Main => unsafe { csr::wrpll::main_dcxo_sda_out_write(val) },
+            DCXO::Helper => unsafe { csr::wrpll::helper_dcxo_sda_out_write(val) },
+        };
+    }
+
+    fn scl_oe(dcxo: DCXO, oe: bool) {
+        let val = if oe { 1 } else { 0 };
+        match dcxo {
+            DCXO::Main => unsafe { csr::wrpll::main_dcxo_scl_oe_write(val) },
+            DCXO::Helper => unsafe { csr::wrpll::helper_dcxo_scl_oe_write(val) },
+        };
+    }
+
+    fn scl_o(dcxo: DCXO, o: bool) {
+        let val = if o { 1 } else { 0 };
+        match dcxo {
+            DCXO::Main => unsafe { csr::wrpll::main_dcxo_scl_out_write(val) },
+            DCXO::Helper => unsafe { csr::wrpll::helper_dcxo_scl_out_write(val) },
+        };
+    }
+
+    pub fn init(dcxo: DCXO, timer: &mut GlobalTimer) -> Result<(), &'static str> {
+        // Set SCL as output, and high level
+        scl_o(dcxo, true);
+        scl_oe(dcxo, true);
+        // Prepare a zero level on SDA so that sda_oe pulls it down
+        sda_o(dcxo, false);
+        // Release SDA
+        sda_oe(dcxo, false);
+
+        // Check the I2C bus is ready
+        half_period(timer);
+        half_period(timer);
+        if !sda_i(dcxo) {
+            // Try toggling SCL a few times
+            for _bit in 0..8 {
+                scl_o(dcxo, false);
+                half_period(timer);
+                scl_o(dcxo, true);
+                half_period(timer);
+            }
+        }
+
+        if !sda_i(dcxo) {
+            return Err("SDA is stuck low and doesn't get unstuck");
+        }
+        Ok(())
+    }
+
+    pub fn start(dcxo: DCXO, timer: &mut GlobalTimer) {
+        // Set SCL high then SDA low
+        scl_o(dcxo, true);
+        half_period(timer);
+        sda_oe(dcxo, true);
+        half_period(timer);
+    }
+
+    pub fn stop(dcxo: DCXO, timer: &mut GlobalTimer) {
+        // First, make sure SCL is low, so that the target releases the SDA line
+        scl_o(dcxo, false);
+        half_period(timer);
+        // Set SCL high then SDA high
+        sda_oe(dcxo, true);
+        scl_o(dcxo, true);
+        half_period(timer);
+        sda_oe(dcxo, false);
+        half_period(timer);
+    }
+
+    pub fn write(dcxo: DCXO, data: u8, timer: &mut GlobalTimer) -> bool {
+        // MSB first
+        for bit in (0..8).rev() {
+            // Set SCL low and set our bit on SDA
+            scl_o(dcxo, false);
+            sda_oe(dcxo, data & (1 << bit) == 0);
+            half_period(timer);
+            // Set SCL high ; data is shifted on the rising edge of SCL
+            scl_o(dcxo, true);
+            half_period(timer);
+        }
+        // Check ack
+        // Set SCL low, then release SDA so that the I2C target can respond
+        scl_o(dcxo, false);
+        half_period(timer);
+        sda_oe(dcxo, false);
+        // Set SCL high and check for ack
+        scl_o(dcxo, true);
+        half_period(timer);
+        // returns true if acked (I2C target pulled SDA low)
+        !sda_i(dcxo)
+    }
+
+    pub fn read(dcxo: DCXO, ack: bool, timer: &mut GlobalTimer) -> u8 {
+        // Set SCL low first, otherwise setting SDA as input may cause a transition
+        // on SDA with SCL high which will be interpreted as START/STOP condition.
+        scl_o(dcxo, false);
+        half_period(timer); // make sure SCL has settled low
+        sda_oe(dcxo, false);
+
+        let mut data: u8 = 0;
+
+        // MSB first
+        for bit in (0..8).rev() {
+            scl_o(dcxo, false);
+            half_period(timer);
+            // Set SCL high and shift data
+            scl_o(dcxo, true);
+            half_period(timer);
+            if sda_i(dcxo) {
+                data |= 1 << bit
+            }
+        }
+        // Send ack
+        // Set SCL low and pull SDA low when acking
+        scl_o(dcxo, false);
+        if ack {
+            sda_oe(dcxo, true)
+        }
+        half_period(timer);
+        // then set SCL high
+        scl_o(dcxo, true);
+        half_period(timer);
+
+        data
+    }
+}
+
+fn write(dcxo: i2c::DCXO, reg: u8, val: u8, timer: &mut GlobalTimer) -> Result<(), &'static str> {
+    i2c::start(dcxo, timer);
+    if !i2c::write(dcxo, ADDRESS << 1, timer) {
+        return Err("Si549 failed to ack write address");
+    }
+    if !i2c::write(dcxo, reg, timer) {
+        return Err("Si549 failed to ack register");
+    }
+    if !i2c::write(dcxo, val, timer) {
+        return Err("Si549 failed to ack value");
+    }
+    i2c::stop(dcxo, timer);
+    Ok(())
+}
+
+fn read(dcxo: i2c::DCXO, reg: u8, timer: &mut GlobalTimer) -> Result<u8, &'static str> {
+    i2c::start(dcxo, timer);
+    if !i2c::write(dcxo, ADDRESS << 1, timer) {
+        return Err("Si549 failed to ack write address");
+    }
+    if !i2c::write(dcxo, reg, timer) {
+        return Err("Si549 failed to ack register");
+    }
+    i2c::stop(dcxo, timer);
+
+    i2c::start(dcxo, timer);
+    if !i2c::write(dcxo, (ADDRESS << 1) | 1, timer) {
+        return Err("Si549 failed to ack read address");
+    }
+    let val = i2c::read(dcxo, false, timer);
+    i2c::stop(dcxo, timer);
+    Ok(val)
+}
+
+fn setup(dcxo: i2c::DCXO, config: &DividerConfig, timer: &mut GlobalTimer) -> Result<(), &'static str> {
+    i2c::init(dcxo, timer)?;
+
+    write(dcxo, 255, 0x00, timer)?; // PAGE
+    write(dcxo, 69, 0x00, timer)?; // Disable FCAL override.
+    write(dcxo, 17, 0x00, timer)?; // Synchronously disable output
+
+    // The Si549 has no ID register, so we check that it responds correctly
+    // by writing values to a RAM-like register and reading them back.
+    for test_value in 0..255 {
+        write(dcxo, 23, test_value, timer)?;
+        let readback = read(dcxo, 23, timer)?;
+        if readback != test_value {
+            return Err("Si549 detection failed");
+        }
+    }
+
+    write(dcxo, 23, config.hsdiv as u8, timer)?;
+    write(dcxo, 24, (config.hsdiv >> 8) as u8 | (config.lsdiv << 4), timer)?;
+    write(dcxo, 26, config.fbdiv as u8, timer)?;
+    write(dcxo, 27, (config.fbdiv >> 8) as u8, timer)?;
+    write(dcxo, 28, (config.fbdiv >> 16) as u8, timer)?;
+    write(dcxo, 29, (config.fbdiv >> 24) as u8, timer)?;
+    write(dcxo, 30, (config.fbdiv >> 32) as u8, timer)?;
+    write(dcxo, 31, (config.fbdiv >> 40) as u8, timer)?;
+
+    write(dcxo, 7, 0x08, timer)?; // Start FCAL
+    timer.delay_us(30_000); // Internal FCAL VCO calibration
+    write(dcxo, 17, 0x01, timer)?; // Synchronously enable output
+
+    Ok(())
+}
+
+pub fn main_setup(timer: &mut GlobalTimer, settings: &FrequencySetting) -> Result<(), &'static str> {
+    unsafe {
+        csr::wrpll::main_dcxo_bitbang_enable_write(1);
+        csr::wrpll::main_dcxo_i2c_address_write(ADDRESS);
+    }
+
+    setup(i2c::DCXO::Main, &settings.main, timer)?;
+
+    // Si549 maximum settling time for large frequency change.
+    timer.delay_us(40_000);
+
+    unsafe {
+        csr::wrpll::main_dcxo_bitbang_enable_write(0);
+    }
+
+    info!("Main Si549 started");
+    Ok(())
+}
+
+pub fn helper_setup(timer: &mut GlobalTimer, settings: &FrequencySetting) -> Result<(), &'static str> {
+    unsafe {
+        csr::wrpll::helper_reset_write(1);
+        csr::wrpll::helper_dcxo_bitbang_enable_write(1);
+        csr::wrpll::helper_dcxo_i2c_address_write(ADDRESS);
+    }
+
+    setup(i2c::DCXO::Helper, &settings.helper, timer)?;
+
+    // Si549 maximum settling time for large frequency change.
+    timer.delay_us(40_000);
+
+    unsafe {
+        csr::wrpll::helper_reset_write(0);
+        csr::wrpll::helper_dcxo_bitbang_enable_write(0);
+    }
+    info!("Helper Si549 started");
+    Ok(())
+}
+
+fn set_adpll(dcxo: i2c::DCXO, adpll: i32) -> Result<(), &'static str> {
+    if adpll.abs() > ADPLL_MAX {
+        return Err("adpll is too large");
+    }
+
+    match dcxo {
+        i2c::DCXO::Main => unsafe {
+            if csr::wrpll::main_dcxo_bitbang_enable_read() == 1 {
+                return Err("Main si549 bitbang mode is active when using gateware i2c");
+            }
+
+            while csr::wrpll::main_dcxo_adpll_busy_read() == 1 {}
+            if csr::wrpll::main_dcxo_nack_read() == 1 {
+                return Err("Main si549 failed to ack adpll write");
+            }
+
+            csr::wrpll::main_dcxo_i2c_address_write(ADDRESS);
+            csr::wrpll::main_dcxo_adpll_write(adpll as u32);
+
+            csr::wrpll::main_dcxo_adpll_stb_write(1);
+        },
+        i2c::DCXO::Helper => unsafe {
+            if csr::wrpll::helper_dcxo_bitbang_enable_read() == 1 {
+                return Err("Helper si549 bitbang mode is active when using gateware i2c");
+            }
+
+            while csr::wrpll::helper_dcxo_adpll_busy_read() == 1 {}
+            if csr::wrpll::helper_dcxo_nack_read() == 1 {
+                return Err("Helper si549 failed to ack adpll write");
+            }
+
+            csr::wrpll::helper_dcxo_i2c_address_write(ADDRESS);
+            csr::wrpll::helper_dcxo_adpll_write(adpll as u32);
+
+            csr::wrpll::helper_dcxo_adpll_stb_write(1);
+        },
+    };
+
+    Ok(())
+}
+
+#[cfg(has_wrpll)]
+pub mod wrpll {
+
+    use super::*;
+
+    const BEATING_PERIOD: i32 = 0x8000;
+    const BEATING_HALFPERIOD: i32 = 0x4000;
+    const COUNTER_WIDTH: u32 = 24;
+    const DIV_WIDTH: u32 = 2;
+
+    const KP: i32 = 6;
+    const KI: i32 = 2;
+    // 4 ppm capture range
+    const ADPLL_LIM: i32 = (4.0 / 0.0001164) as i32;
+
+    static mut BASE_ADPLL: i32 = 0;
+    static mut H_LAST_ADPLL: i32 = 0;
+    static mut LAST_PERIOD_ERR: i32 = 0;
+    static mut M_LAST_ADPLL: i32 = 0;
+    static mut LAST_PHASE_ERR: i32 = 0;
+
+    #[derive(Clone, Copy)]
+    pub enum ISR {
+        RefTag,
+        MainTag,
+    }
+
+    mod tag_collector {
+        use super::*;
+
+        #[cfg(wrpll_ref_clk = "GT_CDR")]
+        static mut TAG_OFFSET: u32 = 8382;
+        #[cfg(wrpll_ref_clk = "SMA_CLKIN")]
+        static mut TAG_OFFSET: u32 = 0;
+        static mut REF_TAG: u32 = 0;
+        static mut REF_TAG_READY: bool = false;
+        static mut MAIN_TAG: u32 = 0;
+        static mut MAIN_TAG_READY: bool = false;
+
+        pub fn reset() {
+            clear_phase_diff_ready();
+            unsafe {
+                REF_TAG = 0;
+                MAIN_TAG = 0;
+            }
+        }
+
+        pub fn clear_phase_diff_ready() {
+            unsafe {
+                REF_TAG_READY = false;
+                MAIN_TAG_READY = false;
+            }
+        }
+
+        pub fn collect_tags(interrupt: ISR) {
+            match interrupt {
+                ISR::RefTag => unsafe {
+                    REF_TAG = csr::wrpll::ref_tag_read();
+                    REF_TAG_READY = true;
+                },
+                ISR::MainTag => unsafe {
+                    MAIN_TAG = csr::wrpll::main_tag_read();
+                    MAIN_TAG_READY = true;
+                },
+            }
+        }
+
+        pub fn phase_diff_ready() -> bool {
+            unsafe { REF_TAG_READY && MAIN_TAG_READY }
+        }
+
+        #[cfg(feature = "calibrate_wrpll_skew")]
+        pub fn set_tag_offset(offset: u32) {
+            unsafe {
+                TAG_OFFSET = offset;
+            }
+        }
+
+        #[cfg(feature = "calibrate_wrpll_skew")]
+        pub fn get_tag_offset() -> u32 {
+            unsafe { TAG_OFFSET }
+        }
+
+        pub fn get_period_error() -> i32 {
+            // n * BEATING_PERIOD - REF_TAG(n) mod BEATING_PERIOD
+            let mut period_error = unsafe { REF_TAG.overflowing_neg().0.rem_euclid(BEATING_PERIOD as u32) as i32 };
+            // mapping tags from [0, 2π] -> [-π, π]
+            if period_error > BEATING_HALFPERIOD {
+                period_error -= BEATING_PERIOD
+            }
+            period_error
+        }
+
+        pub fn get_phase_error() -> i32 {
+            // MAIN_TAG(n) - REF_TAG(n) - TAG_OFFSET mod BEATING_PERIOD
+            let mut phase_error = unsafe {
+                MAIN_TAG
+                    .overflowing_sub(REF_TAG + TAG_OFFSET)
+                    .0
+                    .rem_euclid(BEATING_PERIOD as u32) as i32
+            };
+
+            // mapping tags from [0, 2π] -> [-π, π]
+            if phase_error > BEATING_HALFPERIOD {
+                phase_error -= BEATING_PERIOD
+            }
+            phase_error
+        }
+    }
+
+    fn set_isr(en: bool) {
+        let val = if en { 1 } else { 0 };
+        unsafe {
+            csr::wrpll::ref_tag_ev_enable_write(val);
+            csr::wrpll::main_tag_ev_enable_write(val);
+        }
+    }
+
+    fn set_base_adpll() -> Result<(), &'static str> {
+        let count2adpll =
+            |error: i32| ((error as f64 * 1e6) / (0.0001164 * (1 << (COUNTER_WIDTH - DIV_WIDTH)) as f64)) as i32;
+
+        let (ref_count, main_count) = get_freq_counts();
+        unsafe {
+            BASE_ADPLL = count2adpll(ref_count as i32 - main_count as i32);
+            set_adpll(i2c::DCXO::Main, BASE_ADPLL)?;
+            set_adpll(i2c::DCXO::Helper, BASE_ADPLL)?;
+        }
+        Ok(())
+    }
+
+    fn get_freq_counts() -> (u32, u32) {
+        unsafe {
+            csr::wrpll::frequency_counter_update_write(1);
+            while csr::wrpll::frequency_counter_busy_read() == 1 {}
+            #[cfg(wrpll_ref_clk = "GT_CDR")]
+            let ref_count = csr::wrpll::frequency_counter_counter_rtio_rx0_read();
+            #[cfg(wrpll_ref_clk = "SMA_CLKIN")]
+            let ref_count = csr::wrpll::frequency_counter_counter_ref_read();
+            let main_count = csr::wrpll::frequency_counter_counter_sys_read();
+
+            (ref_count, main_count)
+        }
+    }
+
+    fn reset_plls(timer: &mut GlobalTimer) -> Result<(), &'static str> {
+        unsafe {
+            H_LAST_ADPLL = 0;
+            LAST_PERIOD_ERR = 0;
+            M_LAST_ADPLL = 0;
+            LAST_PHASE_ERR = 0;
+        }
+        set_adpll(i2c::DCXO::Main, 0)?;
+        set_adpll(i2c::DCXO::Helper, 0)?;
+        // wait for adpll to transfer and DCXO to settle
+        timer.delay_us(200);
+        Ok(())
+    }
+
+    fn clear_pending(interrupt: ISR) {
+        match interrupt {
+            ISR::RefTag => unsafe { csr::wrpll::ref_tag_ev_pending_write(1) },
+            ISR::MainTag => unsafe { csr::wrpll::main_tag_ev_pending_write(1) },
+        };
+    }
+
+    fn is_pending(interrupt: ISR) -> bool {
+        match interrupt {
+            ISR::RefTag => unsafe { csr::wrpll::ref_tag_ev_pending_read() == 1 },
+            ISR::MainTag => unsafe { csr::wrpll::main_tag_ev_pending_read() == 1 },
+        }
+    }
+
+    pub fn interrupt_handler() {
+        if is_pending(ISR::RefTag) {
+            tag_collector::collect_tags(ISR::RefTag);
+            clear_pending(ISR::RefTag);
+            helper_pll().expect("failed to run helper DCXO PLL");
+        }
+
+        if is_pending(ISR::MainTag) {
+            tag_collector::collect_tags(ISR::MainTag);
+            clear_pending(ISR::MainTag);
+        }
+
+        if tag_collector::phase_diff_ready() {
+            main_pll().expect("failed to run main DCXO PLL");
+            tag_collector::clear_phase_diff_ready();
+        }
+    }
+
+    fn helper_pll() -> Result<(), &'static str> {
+        let period_err = tag_collector::get_period_error();
+        unsafe {
+            // Based on https://hackmd.io/IACbwcOTSt6Adj3_F9bKuw?view#Integral-wind-up-and-output-limiting
+            let adpll = (H_LAST_ADPLL + (KP + KI) * period_err - KP * LAST_PERIOD_ERR).clamp(-ADPLL_LIM, ADPLL_LIM);
+            set_adpll(i2c::DCXO::Helper, BASE_ADPLL + adpll)?;
+            H_LAST_ADPLL = adpll;
+            LAST_PERIOD_ERR = period_err;
+        };
+        Ok(())
+    }
+
+    fn main_pll() -> Result<(), &'static str> {
+        let phase_err = tag_collector::get_phase_error();
+        unsafe {
+            // Based on https://hackmd.io/IACbwcOTSt6Adj3_F9bKuw?view#Integral-wind-up-and-output-limiting
+            let adpll = (M_LAST_ADPLL + (KP + KI) * phase_err - KP * LAST_PHASE_ERR).clamp(-ADPLL_LIM, ADPLL_LIM);
+            set_adpll(i2c::DCXO::Main, BASE_ADPLL + adpll)?;
+            M_LAST_ADPLL = adpll;
+            LAST_PHASE_ERR = phase_err;
+        };
+        Ok(())
+    }
+
+    #[cfg(wrpll_ref_clk = "GT_CDR")]
+    fn test_skew(timer: &mut GlobalTimer) -> Result<(), &'static str> {
+        // wait for PLL to stabilize
+        timer.delay_us(20_000);
+
+        info!("testing the skew of SYS CLK...");
+        if has_timing_error(timer) {
+            return Err("the skew cannot satisfy setup/hold time constraint of RX synchronizer");
+        }
+        info!("the skew of SYS CLK met the timing constraint");
+        Ok(())
+    }
+
+    #[cfg(wrpll_ref_clk = "GT_CDR")]
+    fn has_timing_error(timer: &mut GlobalTimer) -> bool {
+        unsafe {
+            csr::wrpll_skewtester::error_write(1);
+        }
+        timer.delay_us(5_000);
+        unsafe { csr::wrpll_skewtester::error_read() == 1 }
+    }
+
+    #[cfg(feature = "calibrate_wrpll_skew")]
+    fn find_edge(target: bool, timer: &mut GlobalTimer) -> Result<u32, &'static str> {
+        const STEP: u32 = 8;
+        const STABLE_THRESHOLD: u32 = 10;
+
+        enum FSM {
+            Init,
+            WaitEdge,
+            GotEdge,
+        }
+
+        let mut state: FSM = FSM::Init;
+        let mut offset: u32 = tag_collector::get_tag_offset();
+        let mut median_edge: u32 = 0;
+        let mut stable_counter: u32 = 0;
+
+        for _ in 0..(BEATING_PERIOD as u32 / STEP) as usize {
+            tag_collector::set_tag_offset(offset);
+            offset += STEP;
+            // wait for PLL to stabilize
+            timer.delay_us(20_000);
+
+            let error = has_timing_error(timer);
+            // A median edge deglitcher
+            match state {
+                FSM::Init => {
+                    if error != target {
+                        stable_counter += 1;
+                    } else {
+                        stable_counter = 0;
+                    }
+
+                    if stable_counter >= STABLE_THRESHOLD {
+                        state = FSM::WaitEdge;
+                        stable_counter = 0;
+                    }
+                }
+                FSM::WaitEdge => {
+                    if error == target {
+                        state = FSM::GotEdge;
+                        median_edge = offset;
+                    }
+                }
+                FSM::GotEdge => {
+                    if error != target {
+                        median_edge += STEP;
+                        stable_counter = 0;
+                    } else {
+                        stable_counter += 1;
+                    }
+
+                    if stable_counter >= STABLE_THRESHOLD {
+                        return Ok(median_edge);
+                    }
+                }
+            }
+        }
+        return Err("failed to find timing error edge");
+    }
+
+    #[cfg(feature = "calibrate_wrpll_skew")]
+    fn calibrate_skew(timer: &mut GlobalTimer) -> Result<(), &'static str> {
+        info!("calibrating skew to meet timing constraint...");
+
+        // clear calibrated value
+        tag_collector::set_tag_offset(0);
+        let rising = find_edge(true, timer)? as i32;
+        let falling = find_edge(false, timer)? as i32;
+
+        let width = BEATING_PERIOD - (falling - rising);
+        let result = falling + width / 2;
+        tag_collector::set_tag_offset(result as u32);
+
+        info!(
+            "calibration successful, error zone: {} -> {}, width: {} ({}deg), middle of working region: {}",
+            rising,
+            falling,
+            width,
+            360 * width / BEATING_PERIOD,
+            result,
+        );
+
+        Ok(())
+    }
+
+    pub fn select_recovered_clock(rc: bool, timer: &mut GlobalTimer) {
+        set_isr(false);
+
+        if rc {
+            tag_collector::reset();
+            reset_plls(timer).expect("failed to reset main and helper PLL");
+
+            // get within capture range
+            set_base_adpll().expect("failed to set base adpll");
+
+            // clear gateware pending flag
+            clear_pending(ISR::RefTag);
+            clear_pending(ISR::MainTag);
+
+            // use nFIQ to avoid IRQ being disabled by mutex lock and mess up PLL
+            set_isr(true);
+            info!("WRPLL interrupt enabled");
+
+            #[cfg(feature = "calibrate_wrpll_skew")]
+            calibrate_skew(timer).expect("failed to set the correct skew");
+
+            #[cfg(wrpll_ref_clk = "GT_CDR")]
+            test_skew(timer).expect("skew test failed");
+        }
+    }
+}
+
+#[cfg(has_wrpll_refclk)]
+pub mod wrpll_refclk {
+    use super::*;
+
+    pub struct MmcmSetting {
+        pub clkout0_reg1: u16,  //0x08
+        pub clkout0_reg2: u16,  //0x09
+        pub clkfbout_reg1: u16, //0x14
+        pub clkfbout_reg2: u16, //0x15
+        pub div_reg: u16,       //0x16
+        pub lock_reg1: u16,     //0x18
+        pub lock_reg2: u16,     //0x19
+        pub lock_reg3: u16,     //0x1A
+        pub power_reg: u16,     //0x28
+        pub filt_reg1: u16,     //0x4E
+        pub filt_reg2: u16,     //0x4F
+    }
+
+    fn one_clock_cycle() {
+        unsafe {
+            csr::wrpll_refclk::mmcm_dclk_write(1);
+            csr::wrpll_refclk::mmcm_dclk_write(0);
+        }
+    }
+
+    fn set_addr(address: u8) {
+        unsafe {
+            csr::wrpll_refclk::mmcm_daddr_write(address);
+        }
+    }
+
+    fn set_data(value: u16) {
+        unsafe {
+            csr::wrpll_refclk::mmcm_din_write(value);
+        }
+    }
+
+    fn set_enable(en: bool) {
+        unsafe {
+            let val = if en { 1 } else { 0 };
+            csr::wrpll_refclk::mmcm_den_write(val);
+        }
+    }
+
+    fn set_write_enable(en: bool) {
+        unsafe {
+            let val = if en { 1 } else { 0 };
+            csr::wrpll_refclk::mmcm_dwen_write(val);
+        }
+    }
+
+    fn get_data() -> u16 {
+        unsafe { csr::wrpll_refclk::mmcm_dout_read() }
+    }
+
+    fn drp_ready() -> bool {
+        unsafe { csr::wrpll_refclk::mmcm_dready_read() == 1 }
+    }
+
+    #[allow(dead_code)]
+    fn read(address: u8) -> u16 {
+        set_addr(address);
+        set_enable(true);
+        // Set DADDR on the mmcm and assert DEN for one clock cycle
+        one_clock_cycle();
+
+        set_enable(false);
+        while !drp_ready() {
+            // keep the clock signal until data is ready
+            one_clock_cycle();
+        }
+        get_data()
+    }
+
+    fn write(address: u8, value: u16) {
+        set_addr(address);
+        set_data(value);
+        set_write_enable(true);
+        set_enable(true);
+        // Set DADDR, DI on the mmcm and assert DWE, DEN for one clock cycle
+        one_clock_cycle();
+
+        set_write_enable(false);
+        set_enable(false);
+        while !drp_ready() {
+            // keep the clock signal until write is finished
+            one_clock_cycle();
+        }
+    }
+
+    fn reset(rst: bool) {
+        unsafe {
+            let val = if rst { 1 } else { 0 };
+            csr::wrpll_refclk::mmcm_reset_write(val)
+        }
+    }
+
+    pub fn setup(timer: &mut GlobalTimer, settings: MmcmSetting, mmcm_bypass: bool) -> Result<(), &'static str> {
+        unsafe {
+            csr::wrpll_refclk::refclk_reset_write(1);
+        }
+
+        if mmcm_bypass {
+            info!("Bypassing mmcm");
+            unsafe {
+                csr::wrpll_refclk::mmcm_bypass_write(1);
+            }
+        } else {
+            // Based on "DRP State Machine" from XAPP888
+            // hold reset HIGH during mmcm config
+            reset(true);
+            write(0x08, settings.clkout0_reg1);
+            write(0x09, settings.clkout0_reg2);
+            write(0x14, settings.clkfbout_reg1);
+            write(0x15, settings.clkfbout_reg2);
+            write(0x16, settings.div_reg);
+            write(0x18, settings.lock_reg1);
+            write(0x19, settings.lock_reg2);
+            write(0x1A, settings.lock_reg3);
+            write(0x28, settings.power_reg);
+            write(0x4E, settings.filt_reg1);
+            write(0x4F, settings.filt_reg2);
+            reset(false);
+
+            // wait for the mmcm to lock
+            timer.delay_us(100);
+
+            let locked = unsafe { csr::wrpll_refclk::mmcm_locked_read() == 1 };
+            if !locked {
+                return Err("mmcm failed to generate 125MHz ref clock from SMA CLKIN");
+            }
+        }
+
+        unsafe {
+            csr::wrpll_refclk::refclk_reset_write(0);
+        }
+
+        Ok(())
+    }
+}

src/libbuild_zynq/link.x

@@ -10,7 +10,9 @@ SECTIONS
     __text_start = .;
     .text :
     {
+        __exceptions_start = .;
         KEEP(*(.text.exceptions));
+        __exceptions_end = .;
         *(.text.boot);
         *(.text .text.*);
     } > SDRAM

src/libc/Cargo.toml.tpl

@@ -6,7 +6,7 @@ edition = "2018"
 build = "build.rs"
 
 [dependencies]
-libboard_zynq = { git = "https://git.m-labs.hk/M-Labs/zynq-rs.git" }
+libboard_zynq = { path = "@@ZYNQ_RS@@/libboard_zynq" }
 
 [build-dependencies]
 cc = { version = "1.0.1" }

src/libdyld/Cargo.toml.tpl

@@ -8,4 +8,4 @@ name = "dyld"
 
 [dependencies]
 log = "0.4"
-libcortex_a9 = { git = "https://git.m-labs.hk/M-Labs/zynq-rs.git" }
+libcortex_a9 = { path = "@@ZYNQ_RS@@/libcortex_a9" }

src/libio/Cargo.toml.tpl

@@ -11,7 +11,7 @@ path = "lib.rs"
 core_io = { version = "0.1", features = ["collections"] }
 byteorder = { version = "1.0", default-features = false, optional = true }
 
-libsupport_zynq = { default-features = false, features = ["alloc_core"], git = "https://git.m-labs.hk/M-Labs/zynq-rs.git" }
+libsupport_zynq = { path = "@@ZYNQ_RS@@/libsupport_zynq", default-features = false, features = ["alloc_core"] }
 
 [features]
 alloc = []

src/libio/cursor.rs

@@ -1,3 +1,6 @@
+#[cfg(feature = "alloc")]
+use alloc::vec::Vec;
+
 use core_io::{Error as IoError, Read, Write};
 
 #[derive(Debug, Clone)]
@@ -64,7 +67,7 @@ impl Write for Cursor<&mut [u8]> {
 }
 
 #[cfg(feature = "alloc")]
-impl Write for Cursor<::alloc::Vec<u8>> {
+impl Write for Cursor<Vec<u8>> {
     #[inline]
     fn write(&mut self, buf: &[u8]) -> Result<usize, IoError> {
         self.inner.extend_from_slice(buf);

src/libio/lib.rs

@@ -1,13 +1,10 @@
 #![no_std]
 #![feature(never_type)]
-#![cfg_attr(feature = "alloc", feature(alloc))]
 
+#[cfg(feature = "alloc")]
 extern crate alloc;
 extern crate core_io;
 
-#[cfg(feature = "alloc")]
-#[macro_use]
-use alloc;
 #[cfg(feature = "byteorder")]
 extern crate byteorder;
 

src/libio/proto.rs

@@ -1,3 +1,4 @@
+#[cfg(feature = "alloc")]
 use alloc::{string::String, vec};
 use core::str::Utf8Error;
 
@@ -50,7 +51,8 @@ pub trait ProtoRead {
     }
 
     #[inline]
-    fn read_bytes(&mut self) -> Result<::alloc::vec::Vec<u8>, Self::ReadError> {
+    #[cfg(feature = "alloc")]
+    fn read_bytes(&mut self) -> Result<vec::Vec<u8>, Self::ReadError> {
         let length = self.read_u32()?;
         let mut value = vec![0; length as usize];
         self.read_exact(&mut value)?;
@@ -58,7 +60,8 @@ pub trait ProtoRead {
     }
 
     #[inline]
-    fn read_string(&mut self) -> Result<::alloc::string::String, ReadStringError<Self::ReadError>> {
+    #[cfg(feature = "alloc")]
+    fn read_string(&mut self) -> Result<String, ReadStringError<Self::ReadError>> {
         let bytes = self.read_bytes().map_err(ReadStringError::Other)?;
         String::from_utf8(bytes).map_err(|err| ReadStringError::Utf8(err.utf8_error()))
     }
@@ -135,6 +138,7 @@ pub trait ProtoWrite {
     }
 
     #[inline]
+    #[cfg(feature = "alloc")]
     fn write_string(&mut self, value: &str) -> Result<(), Self::WriteError> {
         self.write_bytes(value.as_bytes())
     }

src/libksupport/Cargo.toml.tpl

@@ -0,0 +1,34 @@
+[package]
+name = "ksupport"
+description = "Kernel support for Zynq-based platforms"
+version = "0.1.0"
+authors = ["M-Labs"]
+edition = "2018"
+
+[build-dependencies]
+build_zynq = { path = "../libbuild_zynq" }
+
+[dependencies]
+cslice = "0.3"
+log = "0.4"
+nb = "0.1"
+core_io = { version = "0.1", features = ["collections"] }
+byteorder = { version = "1.3", default-features = false }
+void = { version = "1", default-features = false }
+log_buffer = { version = "1.2" }
+libm = { version = "0.2", features = ["unstable"] }
+vcell = "0.1"
+
+libboard_zynq = { path = "@@ZYNQ_RS@@/libboard_zynq", features = ["ipv6"]}
+libsupport_zynq = { path = "@@ZYNQ_RS@@/libsupport_zynq", default-features = false, features = ["alloc_core"] }
+libcortex_a9 = { path = "@@ZYNQ_RS@@/libcortex_a9" }
+libasync = { path = "@@ZYNQ_RS@@/libasync" }
+libregister = { path = "@@ZYNQ_RS@@/libregister" }
+libconfig = { path = "@@ZYNQ_RS@@/libconfig", features = ["fat_lfn", "ipv6"] }
+
+dyld = { path = "../libdyld" }
+dwarf = { path = "../libdwarf" }
+unwind = { path = "../libunwind" }
+libc = { path = "../libc" }
+io = { path = "../libio" }
+libboard_artiq = { path = "../libboard_artiq" }

src/libksupport/build.rs

@@ -0,0 +1,5 @@
+extern crate build_zynq;
+
+fn main() {
+    build_zynq::cfg();
+}

src/libksupport/src/eh_artiq.rs

@@ -422,7 +422,7 @@ extern "C" fn stop_fn(
 }
 
 // Must be kept in sync with preallocate_runtime_exception_names() in artiq/language/embedding_map.py
-static EXCEPTION_ID_LOOKUP: [(&str, u32); 11] = [
+static EXCEPTION_ID_LOOKUP: [(&str, u32); 12] = [
     ("RuntimeError", 0),
     ("RTIOUnderflow", 1),
     ("RTIOOverflow", 2),
@@ -434,6 +434,7 @@ static EXCEPTION_ID_LOOKUP: [(&str, u32); 11] = [
     ("ZeroDivisionError", 8),
     ("IndexError", 9),
     ("UnwrapNoneError", 10),
+    ("SubkernelError", 11),
 ];
 
 pub fn get_exception_id(name: &str) -> u32 {

src/libksupport/src/kernel/api.rs

@@ -5,6 +5,8 @@ use libc::{c_char, c_int, size_t};
 use libm;
 use log::{info, warn};
 
+#[cfg(has_drtio)]
+use super::subkernel;
 use super::{cache,
             core1::rtio_get_destination_status,
             dma,
@@ -114,6 +116,16 @@ pub fn resolve(required: &[u8]) -> Option<u32> {
         api!(i2c_read = i2c::read),
         api!(i2c_switch_select = i2c::switch_select),
 
+        // subkernel
+        #[cfg(has_drtio)]
+        api!(subkernel_load_run = subkernel::load_run),
+        #[cfg(has_drtio)]
+        api!(subkernel_await_finish = subkernel::await_finish),
+        #[cfg(has_drtio)]
+        api!(subkernel_send_message = subkernel::send_message),
+        #[cfg(has_drtio)]
+        api!(subkernel_await_message = subkernel::await_message),
+
         // Double-precision floating-point arithmetic helper functions
         // RTABI chapter 4.1.2, Table 2
         api!(__aeabi_dadd),

src/libksupport/src/kernel/dma.rs

@@ -10,6 +10,7 @@ use crate::{artiq_raise, pl::csr, rtio};
 pub struct DmaTrace {
     duration: i64,
     address: i32,
+    uses_ddma: bool,
 }
 
 #[derive(Clone, Debug)]
@@ -116,7 +117,7 @@ pub extern "C" fn dma_record_output(target: i32, word: i32) {
     }
 }
 
-pub extern "C" fn dma_record_output_wide(target: i32, words: CSlice<i32>) {
+pub extern "C" fn dma_record_output_wide(target: i32, words: &CSlice<i32>) {
     assert!(words.len() <= 16); // enforce the hardware limit
 
     unsafe {
@@ -151,8 +152,12 @@ pub extern "C" fn dma_retrieve(name: CSlice<u8>) -> DmaTrace {
     }
     match unsafe { KERNEL_CHANNEL_0TO1.as_mut().unwrap() }.recv() {
         Message::DmaGetReply(None) => (),
-        Message::DmaGetReply(Some((address, duration))) => {
-            return DmaTrace { address, duration };
+        Message::DmaGetReply(Some((address, duration, uses_ddma))) => {
+            return DmaTrace {
+                address,
+                duration,
+                uses_ddma,
+            };
         }
         _ => panic!("Expected DmaGetReply after DmaGetRequest!"),
     }
@@ -160,23 +165,26 @@ pub extern "C" fn dma_retrieve(name: CSlice<u8>) -> DmaTrace {
     artiq_raise!("DMAError", "DMA trace not found");
 }
 
-pub extern "C" fn dma_playback(timestamp: i64, ptr: i32) {
+pub extern "C" fn dma_playback(timestamp: i64, ptr: i32, _uses_ddma: bool) {
     unsafe {
         csr::rtio_dma::base_address_write(ptr as u32);
         csr::rtio_dma::time_offset_write(timestamp as u64);
 
+        let old_cri_master = csr::cri_con::selected_read();
         csr::cri_con::selected_write(1);
         csr::rtio_dma::enable_write(1);
         #[cfg(has_drtio)]
-        KERNEL_CHANNEL_1TO0
-            .as_mut()
-            .unwrap()
-            .send(Message::DmaStartRemoteRequest {
-                id: ptr,
-                timestamp: timestamp,
-            });
+        if _uses_ddma {
+            KERNEL_CHANNEL_1TO0
+                .as_mut()
+                .unwrap()
+                .send(Message::DmaStartRemoteRequest {
+                    id: ptr,
+                    timestamp: timestamp,
+                });
+        }
         while csr::rtio_dma::enable_read() != 0 {}
-        csr::cri_con::selected_write(0);
+        csr::cri_con::selected_write(old_cri_master);
 
         let error = csr::rtio_dma::error_read();
         if error != 0 {
@@ -203,7 +211,7 @@ pub extern "C" fn dma_playback(timestamp: i64, ptr: i32) {
             }
         }
         #[cfg(has_drtio)]
-        {
+        if _uses_ddma {
             KERNEL_CHANNEL_1TO0
                 .as_mut()
                 .unwrap()

src/libksupport/src/kernel/mod.rs

@@ -3,7 +3,7 @@ use core::ptr;
 
 use libcortex_a9::{mutex::Mutex, semaphore::Semaphore, sync_channel};
 
-use crate::eh_artiq;
+use crate::{eh_artiq, RPCException};
 
 mod control;
 pub use control::Control;
@@ -13,16 +13,17 @@ mod dma;
 mod rpc;
 pub use dma::DmaRecorder;
 mod cache;
+#[cfg(has_drtio)]
+mod subkernel;
 
+#[cfg(has_drtio)]
 #[derive(Debug, Clone)]
-pub struct RPCException {
-    pub id: u32,
-    pub message: u32,
-    pub param: [i64; 3],
-    pub file: u32,
-    pub line: i32,
-    pub column: i32,
-    pub function: u32,
+pub enum SubkernelStatus {
+    NoError,
+    Timeout,
+    IncorrectState,
+    CommLost,
+    OtherError,
 }
 
 #[derive(Debug, Clone)]
@@ -52,7 +53,7 @@ pub enum Message {
     DmaPutRequest(DmaRecorder),
     DmaEraseRequest(String),
     DmaGetRequest(String),
-    DmaGetReply(Option<(i32, i64)>),
+    DmaGetReply(Option<(i32, i64, bool)>),
     #[cfg(has_drtio)]
     DmaStartRemoteRequest {
         id: i32,
@@ -72,6 +73,45 @@ pub enum Message {
     UpDestinationsRequest(i32),
     #[cfg(has_drtio)]
     UpDestinationsReply(bool),
+
+    #[cfg(has_drtio)]
+    SubkernelLoadRunRequest {
+        id: u32,
+        destination: u8,
+        run: bool,
+    },
+    #[cfg(has_drtio)]
+    SubkernelLoadRunReply {
+        succeeded: bool,
+    },
+    #[cfg(has_drtio)]
+    SubkernelAwaitFinishRequest {
+        id: u32,
+        timeout: i64,
+    },
+    #[cfg(has_drtio)]
+    SubkernelAwaitFinishReply {
+        status: SubkernelStatus,
+    },
+    #[cfg(has_drtio)]
+    SubkernelMsgSend {
+        id: u32,
+        destination: Option<u8>,
+        data: Vec<u8>,
+    },
+    #[cfg(has_drtio)]
+    SubkernelMsgSent,
+    #[cfg(has_drtio)]
+    SubkernelMsgRecvRequest {
+        id: i32,
+        timeout: i64,
+        tags: Vec<u8>,
+    },
+    #[cfg(has_drtio)]
+    SubkernelMsgRecvReply {
+        status: SubkernelStatus,
+        count: u8,
+    },
 }
 
 static CHANNEL_0TO1: Mutex<Option<sync_channel::Sender<'static, Message>>> = Mutex::new(None);

src/libksupport/src/kernel/rpc.rs

@@ -10,7 +10,7 @@ use crate::{eh_artiq, rpc::send_args};
 fn rpc_send_common(is_async: bool, service: u32, tag: &CSlice<u8>, data: *const *const ()) {
     let core1_tx = unsafe { KERNEL_CHANNEL_1TO0.as_mut().unwrap() };
     let mut buffer = Vec::<u8>::new();
-    send_args(&mut buffer, service, tag.as_ref(), data).expect("RPC encoding failed");
+    send_args(&mut buffer, service, tag.as_ref(), data, true).expect("RPC encoding failed");
     core1_tx.send(Message::RpcSend { is_async, data: buffer });
 }
 

src/libksupport/src/kernel/subkernel.rs

@@ -0,0 +1,122 @@
+use alloc::vec::Vec;
+
+use cslice::CSlice;
+
+use super::{Message, SubkernelStatus, KERNEL_CHANNEL_0TO1, KERNEL_CHANNEL_1TO0};
+use crate::{artiq_raise, rpc::send_args};
+
+pub extern "C" fn load_run(id: u32, destination: u8, run: bool) {
+    unsafe {
+        KERNEL_CHANNEL_1TO0
+            .as_mut()
+            .unwrap()
+            .send(Message::SubkernelLoadRunRequest {
+                id: id,
+                destination: destination,
+                run: run,
+            });
+    }
+    match unsafe { KERNEL_CHANNEL_0TO1.as_mut().unwrap() }.recv() {
+        Message::SubkernelLoadRunReply { succeeded: true } => (),
+        Message::SubkernelLoadRunReply { succeeded: false } => {
+            artiq_raise!("SubkernelError", "Error loading or running the subkernel")
+        }
+        _ => panic!("Expected SubkernelLoadRunReply after SubkernelLoadRunRequest!"),
+    }
+}
+
+pub extern "C" fn await_finish(id: u32, timeout: i64) {
+    unsafe {
+        KERNEL_CHANNEL_1TO0
+            .as_mut()
+            .unwrap()
+            .send(Message::SubkernelAwaitFinishRequest {
+                id: id,
+                timeout: timeout,
+            });
+    }
+    match unsafe { KERNEL_CHANNEL_0TO1.as_mut().unwrap() }.recv() {
+        Message::SubkernelAwaitFinishReply {
+            status: SubkernelStatus::NoError,
+        } => (),
+        Message::SubkernelAwaitFinishReply {
+            status: SubkernelStatus::IncorrectState,
+        } => artiq_raise!("SubkernelError", "Subkernel not running"),
+        Message::SubkernelAwaitFinishReply {
+            status: SubkernelStatus::Timeout,
+        } => artiq_raise!("SubkernelError", "Subkernel timed out"),
+        Message::SubkernelAwaitFinishReply {
+            status: SubkernelStatus::CommLost,
+        } => artiq_raise!("SubkernelError", "Lost communication with satellite"),
+        Message::SubkernelAwaitFinishReply {
+            status: SubkernelStatus::OtherError,
+        } => artiq_raise!("SubkernelError", "An error occurred during subkernel operation"),
+        _ => panic!("expected SubkernelAwaitFinishReply after SubkernelAwaitFinishRequest"),
+    }
+}
+
+pub extern "C" fn send_message(
+    id: u32,
+    is_return: bool,
+    destination: u8,
+    count: u8,
+    tag: &CSlice<u8>,
+    data: *const *const (),
+) {
+    let mut buffer = Vec::<u8>::new();
+    send_args(&mut buffer, 0, tag.as_ref(), data, false).expect("RPC encoding failed");
+    // overwrite service tag, include how many tags are in the message
+    buffer[3] = count;
+    unsafe {
+        KERNEL_CHANNEL_1TO0.as_mut().unwrap().send(Message::SubkernelMsgSend {
+            id: id,
+            destination: if is_return { None } else { Some(destination) },
+            data: buffer[3..].to_vec(),
+        });
+    }
+    match unsafe { KERNEL_CHANNEL_0TO1.as_mut().unwrap() }.recv() {
+        Message::SubkernelMsgSent => (),
+        _ => panic!("expected SubkernelMsgSent after SubkernelMsgSend"),
+    }
+}
+
+pub extern "C" fn await_message(id: i32, timeout: i64, tags: &CSlice<u8>, min: u8, max: u8) {
+    unsafe {
+        KERNEL_CHANNEL_1TO0
+            .as_mut()
+            .unwrap()
+            .send(Message::SubkernelMsgRecvRequest {
+                id: id,
+                timeout: timeout,
+                tags: tags.as_ref().to_vec(),
+            });
+    }
+    match unsafe { KERNEL_CHANNEL_0TO1.as_mut().unwrap() }.recv() {
+        Message::SubkernelMsgRecvReply {
+            status: SubkernelStatus::NoError,
+            count,
+        } => {
+            if min > count || count > max {
+                artiq_raise!("SubkernelError", "Received more or less arguments than required")
+            }
+        }
+        Message::SubkernelMsgRecvReply {
+            status: SubkernelStatus::IncorrectState,
+            ..
+        } => artiq_raise!("SubkernelError", "Subkernel not running"),
+        Message::SubkernelMsgRecvReply {
+            status: SubkernelStatus::Timeout,
+            ..
+        } => artiq_raise!("SubkernelError", "Subkernel timed out"),
+        Message::SubkernelMsgRecvReply {
+            status: SubkernelStatus::CommLost,
+            ..
+        } => artiq_raise!("SubkernelError", "Lost communication with satellite"),
+        Message::SubkernelMsgRecvReply {
+            status: SubkernelStatus::OtherError,
+            ..
+        } => artiq_raise!("SubkernelError", "An error occurred during subkernel operation"),
+        _ => panic!("expected SubkernelMsgRecvReply after SubkernelMsgRecvRequest"),
+    }
+    // RpcRecvRequest should be called after this to receive message data
+}

src/libksupport/src/lib.rs

@@ -0,0 +1,163 @@
+#![no_std]
+#![feature(c_variadic)]
+#![feature(const_btree_new)]
+#![feature(const_in_array_repeat_expressions)]
+#![feature(naked_functions)]
+#![feature(asm)]
+
+#[macro_use]
+extern crate alloc;
+
+use alloc::{collections::BTreeMap, string::String};
+
+use io::{Cursor, ProtoRead};
+use libasync::block_async;
+use libconfig::Config;
+use log::{error, warn};
+#[cfg(has_drtiosat)]
+pub use pl::csr::drtiosat as rtio_core;
+#[cfg(has_rtio_core)]
+pub use pl::csr::rtio_core;
+use void::Void;
+
+pub mod eh_artiq;
+pub mod i2c;
+pub mod irq;
+pub mod kernel;
+pub mod rpc;
+#[cfg(ki_impl = "csr")]
+#[path = "rtio_csr.rs"]
+pub mod rtio;
+#[cfg(ki_impl = "acp")]
+#[path = "rtio_acp.rs"]
+pub mod rtio;
+#[rustfmt::skip]
+#[path = "../../../build/pl.rs"]
+pub mod pl;
+
+
+#[derive(Debug, Clone)]
+pub struct RPCException {
+    pub id: u32,
+    pub message: u32,
+    pub param: [i64; 3],
+    pub file: u32,
+    pub line: i32,
+    pub column: i32,
+    pub function: u32,
+}
+
+pub static mut SEEN_ASYNC_ERRORS: u8 = 0;
+
+pub const ASYNC_ERROR_COLLISION: u8 = 1 << 0;
+pub const ASYNC_ERROR_BUSY: u8 = 1 << 1;
+pub const ASYNC_ERROR_SEQUENCE_ERROR: u8 = 1 << 2;
+
+pub unsafe fn get_async_errors() -> u8 {
+    let errors = SEEN_ASYNC_ERRORS;
+    SEEN_ASYNC_ERRORS = 0;
+    errors
+}
+
+fn wait_for_async_rtio_error() -> nb::Result<(), Void> {
+    unsafe {
+        #[cfg(has_rtio_core)]
+        let errors = rtio_core::async_error_read();
+        #[cfg(has_drtiosat)]
+        let errors = rtio_core::protocol_error_read();
+        if errors != 0 {
+            Ok(())
+        } else {
+            Err(nb::Error::WouldBlock)
+        }
+    }
+}
+
+pub async fn report_async_rtio_errors() {
+    loop {
+        let _ = block_async!(wait_for_async_rtio_error()).await;
+        unsafe {
+            #[cfg(has_rtio_core)]
+            let errors = rtio_core::async_error_read();
+            #[cfg(has_drtiosat)]
+            let errors = rtio_core::protocol_error_read();
+            if errors & ASYNC_ERROR_COLLISION != 0 {
+                let channel = rtio_core::collision_channel_read();
+                error!(
+                    "RTIO collision involving channel 0x{:04x}:{}",
+                    channel,
+                    resolve_channel_name(channel as u32)
+                );
+            }
+            if errors & ASYNC_ERROR_BUSY != 0 {
+                let channel = rtio_core::busy_channel_read();
+                error!(
+                    "RTIO busy error involving channel 0x{:04x}:{}",
+                    channel,
+                    resolve_channel_name(channel as u32)
+                );
+            }
+            if errors & ASYNC_ERROR_SEQUENCE_ERROR != 0 {
+                let channel = rtio_core::sequence_error_channel_read();
+                error!(
+                    "RTIO sequence error involving channel 0x{:04x}:{}",
+                    channel,
+                    resolve_channel_name(channel as u32)
+                );
+            }
+            SEEN_ASYNC_ERRORS = errors;
+            #[cfg(has_rtio_core)]
+            rtio_core::async_error_write(errors);
+            #[cfg(has_drtiosat)]
+            rtio_core::protocol_error_write(errors);
+        }
+    }
+}
+
+static mut RTIO_DEVICE_MAP: BTreeMap<u32, String> = BTreeMap::new();
+
+fn read_device_map(cfg: &Config) -> BTreeMap<u32, String> {
+    let mut device_map: BTreeMap<u32, String> = BTreeMap::new();
+    let _ = cfg
+        .read("device_map")
+        .and_then(|raw_bytes| {
+            let mut bytes_cr = Cursor::new(raw_bytes);
+            let size = bytes_cr.read_u32().unwrap();
+            for _ in 0..size {
+                let channel = bytes_cr.read_u32().unwrap();
+                let device_name = bytes_cr.read_string().unwrap();
+                if let Some(old_entry) = device_map.insert(channel, device_name.clone()) {
+                    warn!(
+                        "conflicting device map entries for RTIO channel {}: '{}' and '{}'",
+                        channel, old_entry, device_name
+                    );
+                }
+            }
+            Ok(())
+        })
+        .or_else(|err| {
+            warn!(
+                "error reading device map ({}), device names will not be available in RTIO error messages",
+                err
+            );
+            Err(err)
+        });
+    device_map
+}
+
+fn _resolve_channel_name(channel: u32, device_map: &BTreeMap<u32, String>) -> String {
+    match device_map.get(&channel) {
+        Some(val) => val.clone(),
+        None => String::from("unknown"),
+    }
+}
+
+pub fn resolve_channel_name(channel: u32) -> String {
+    _resolve_channel_name(channel, unsafe { &RTIO_DEVICE_MAP })
+}
+
+pub fn setup_device_map(cfg: &Config) {
+    unsafe {
+        RTIO_DEVICE_MAP = read_device_map(cfg);
+    }
+}

src/libksupport/src/rpc.rs

@@ -1,71 +1,62 @@
-use alloc::boxed::Box;
-use core::{future::Future, str};
+use core::str;
 
-use async_recursion::async_recursion;
 use byteorder::{ByteOrder, NativeEndian};
-use core_io::{Error, Write};
+use core_io::{Error, Read, Write};
 use cslice::{CMutSlice, CSlice};
-use io::proto::ProtoWrite;
-use libasync::smoltcp::TcpStream;
-use libboard_zynq::smoltcp;
+use io::{ProtoRead, ProtoWrite};
 use log::trace;
 
 use self::tag::{split_tag, Tag, TagIterator};
-use crate::proto_async;
 
 #[inline]
-fn round_up(val: usize, power_of_two: usize) -> usize {
+pub fn round_up(val: usize, power_of_two: usize) -> usize {
     assert!(power_of_two.is_power_of_two());
     let max_rem = power_of_two - 1;
     (val + max_rem) & (!max_rem)
 }
 
 #[inline]
-unsafe fn round_up_mut<T>(ptr: *mut T, power_of_two: usize) -> *mut T {
+pub unsafe fn round_up_mut<T>(ptr: *mut T, power_of_two: usize) -> *mut T {
     round_up(ptr as usize, power_of_two) as *mut T
 }
 
 #[inline]
-unsafe fn round_up_const<T>(ptr: *const T, power_of_two: usize) -> *const T {
+pub unsafe fn round_up_const<T>(ptr: *const T, power_of_two: usize) -> *const T {
     round_up(ptr as usize, power_of_two) as *const T
 }
 
 #[inline]
-unsafe fn align_ptr<T>(ptr: *const ()) -> *const T {
+pub unsafe fn align_ptr<T>(ptr: *const ()) -> *const T {
     round_up_const(ptr, core::mem::align_of::<T>()) as *const T
 }
 
 #[inline]
-unsafe fn align_ptr_mut<T>(ptr: *mut ()) -> *mut T {
+pub unsafe fn align_ptr_mut<T>(ptr: *mut ()) -> *mut T {
     round_up_mut(ptr, core::mem::align_of::<T>()) as *mut T
 }
 
-/// Reads (deserializes) `length` array or list elements of type `tag` from `stream`,
-/// writing them into the buffer given by `storage`.
-///
-/// `alloc` is used for nested allocations (if elements themselves contain
-/// lists/arrays), see [recv_value].
-#[async_recursion(?Send)]
-async unsafe fn recv_elements<F>(
-    stream: &TcpStream,
-    elt_tag: Tag<'async_recursion>,
+// versions for reader rather than TcpStream
+// they will be made into sync for satellite subkernels later
+unsafe fn recv_elements<F, R>(
+    reader: &mut R,
+    elt_tag: Tag,
     length: usize,
     storage: *mut (),
-    alloc: &(impl Fn(usize) -> F + 'async_recursion),
-) -> Result<(), smoltcp::Error>
+    alloc: &mut F,
+) -> Result<(), Error>
 where
-    F: Future<Output = *mut ()>,
+    F: FnMut(usize) -> *mut (),
+    R: Read + ?Sized,
 {
-    // List of simple types are special-cased in the protocol for performance.
     match elt_tag {
         Tag::Bool => {
             let dest = core::slice::from_raw_parts_mut(storage as *mut u8, length);
-            proto_async::read_chunk(stream, dest).await?;
+            reader.read_exact(dest)?;
         }
         Tag::Int32 => {
             let ptr = storage as *mut u32;
             let dest = core::slice::from_raw_parts_mut(ptr as *mut u8, length * 4);
-            proto_async::read_chunk(stream, dest).await?;
+            reader.read_exact(dest)?;
             drop(dest);
             let dest = core::slice::from_raw_parts_mut(ptr, length);
             NativeEndian::from_slice_u32(dest);
@@ -73,7 +64,7 @@ where
         Tag::Int64 | Tag::Float64 => {
             let ptr = storage as *mut u64;
             let dest = core::slice::from_raw_parts_mut(ptr as *mut u8, length * 8);
-            proto_async::read_chunk(stream, dest).await?;
+            reader.read_exact(dest)?;
             drop(dest);
             let dest = core::slice::from_raw_parts_mut(ptr, length);
             NativeEndian::from_slice_u64(dest);
@@ -81,27 +72,17 @@ where
         _ => {
             let mut data = storage;
             for _ in 0..length {
-                recv_value(stream, elt_tag, &mut data, alloc).await?
+                recv_value(reader, elt_tag, &mut data, alloc)?
             }
         }
     }
     Ok(())
 }
 
-/// Reads (deserializes) a value of type `tag` from `stream`, writing the results to
-/// the kernel-side buffer `data` (the passed pointer to which is incremented to point
-/// past the just-received data). For nested allocations (lists/arrays), `alloc` is
-/// invoked any number of times with the size of the required allocation as a parameter
-/// (which is assumed to be correctly aligned for all payload types).
-#[async_recursion(?Send)]
-async unsafe fn recv_value<F>(
-    stream: &TcpStream,
-    tag: Tag<'async_recursion>,
-    data: &mut *mut (),
-    alloc: &(impl Fn(usize) -> F + 'async_recursion),
-) -> Result<(), smoltcp::Error>
+unsafe fn recv_value<F, R>(reader: &mut R, tag: Tag, data: &mut *mut (), alloc: &mut F) -> Result<(), Error>
 where
-    F: Future<Output = *mut ()>,
+    F: FnMut(usize) -> *mut (),
+    R: Read + ?Sized,
 {
     macro_rules! consume_value {
         ($ty:ty, | $ptr:ident | $map:expr) => {{
@@ -114,22 +95,22 @@ where
     match tag {
         Tag::None => Ok(()),
         Tag::Bool => consume_value!(i8, |ptr| {
-            *ptr = proto_async::read_i8(stream).await?;
+            *ptr = reader.read_u8()? as i8;
             Ok(())
         }),
         Tag::Int32 => consume_value!(i32, |ptr| {
-            *ptr = proto_async::read_i32(stream).await?;
+            *ptr = reader.read_u32()? as i32;
             Ok(())
         }),
         Tag::Int64 | Tag::Float64 => consume_value!(i64, |ptr| {
-            *ptr = proto_async::read_i64(stream).await?;
+            *ptr = reader.read_u64()? as i64;
             Ok(())
         }),
         Tag::String | Tag::Bytes | Tag::ByteArray => {
             consume_value!(CMutSlice<u8>, |ptr| {
-                let length = proto_async::read_i32(stream).await? as usize;
-                *ptr = CMutSlice::new(alloc(length).await as *mut u8, length);
-                proto_async::read_chunk(stream, (*ptr).as_mut()).await?;
+                let length = reader.read_u32()? as usize;
+                *ptr = CMutSlice::new(alloc(length) as *mut u8, length);
+                reader.read_exact((*ptr).as_mut())?;
                 Ok(())
             })
         }
@@ -139,10 +120,8 @@ where
             let mut it = it.clone();
             for _ in 0..arity {
                 let tag = it.next().expect("truncated tag");
-                recv_value(stream, tag, data, alloc).await?
+                recv_value(reader, tag, data, alloc)?
             }
-            // Take into account any tail padding (if element(s) with largest alignment
-            // are not at the end).
             *data = round_up_mut(*data, alignment);
             Ok(())
         }
@@ -154,50 +133,41 @@ where
             }
             consume_value!(*mut List, |ptr_to_list| {
                 let tag = it.clone().next().expect("truncated tag");
-                let length = proto_async::read_i32(stream).await? as usize;
+                let length = reader.read_u32()? as usize;
 
-                // To avoid multiple kernel CPU roundtrips, use a single allocation for
-                // both the pointer/length List (slice) and the backing storage for the
-                // elements. We can assume that alloc() is aligned suitably, so just
-                // need to take into account any extra padding required.
-                // (Note: At the time of writing, there will never actually be any types
-                // with alignment larger than 8 bytes, so storage_offset == 0 always.)
                 let list_size = 4 + 4;
                 let storage_offset = round_up(list_size, tag.alignment());
                 let storage_size = tag.size() * length;
 
-                let allocation = alloc(storage_offset + storage_size).await as *mut u8;
+                let allocation = alloc(storage_offset + storage_size) as *mut u8;
                 *ptr_to_list = allocation as *mut List;
                 let storage = allocation.offset(storage_offset as isize) as *mut ();
 
                 (**ptr_to_list).length = length;
                 (**ptr_to_list).elements = storage;
-                recv_elements(stream, tag, length, storage, alloc).await
+                recv_elements(reader, tag, length, storage, alloc)
             })
         }
         Tag::Array(it, num_dims) => {
             consume_value!(*mut (), |buffer| {
-                // Deserialize length along each dimension and compute total number of
-                // elements.
                 let mut total_len: usize = 1;
                 for _ in 0..num_dims {
-                    let len = proto_async::read_i32(stream).await? as usize;
+                    let len = reader.read_u32()? as usize;
                     total_len *= len;
                     consume_value!(usize, |ptr| *ptr = len)
                 }
 
-                // Allocate backing storage for elements; deserialize them.
                 let elt_tag = it.clone().next().expect("truncated tag");
-                *buffer = alloc(elt_tag.size() * total_len).await;
-                recv_elements(stream, elt_tag, total_len, *buffer, alloc).await
+                *buffer = alloc(elt_tag.size() * total_len);
+                recv_elements(reader, elt_tag, total_len, *buffer, alloc)
             })
         }
         Tag::Range(it) => {
             *data = round_up_mut(*data, tag.alignment());
             let tag = it.clone().next().expect("truncated tag");
-            recv_value(stream, tag, data, alloc).await?;
-            recv_value(stream, tag, data, alloc).await?;
-            recv_value(stream, tag, data, alloc).await?;
+            recv_value(reader, tag, data, alloc)?;
+            recv_value(reader, tag, data, alloc)?;
+            recv_value(reader, tag, data, alloc)?;
             Ok(())
         }
         Tag::Keyword(_) => unreachable!(),
@@ -205,28 +175,39 @@ where
     }
 }
 
-pub async fn recv_return<F>(
-    stream: &TcpStream,
-    tag_bytes: &[u8],
+pub fn recv_return<'a, F, R>(
+    reader: &mut R,
+    tag_bytes: &'a [u8],
     data: *mut (),
-    alloc: &impl Fn(usize) -> F,
-) -> Result<(), smoltcp::Error>
+    alloc: &mut F,
+) -> Result<&'a [u8], Error>
 where
-    F: Future<Output = *mut ()>,
+    F: FnMut(usize) -> *mut (),
+    R: Read + ?Sized,
 {
     let mut it = TagIterator::new(tag_bytes);
     trace!("recv ...->{}", it);
 
     let tag = it.next().expect("truncated tag");
     let mut data = data;
-    unsafe { recv_value(stream, tag, &mut data, alloc).await? };
+    unsafe { recv_value(reader, tag, &mut data, alloc)? };
 
-    Ok(())
+    Ok(it.data)
 }
 
-unsafe fn send_elements<W>(writer: &mut W, elt_tag: Tag, length: usize, data: *const ()) -> Result<(), Error>
-where W: Write + ?Sized {
-    writer.write_u8(elt_tag.as_u8())?;
+unsafe fn send_elements<W>(
+    writer: &mut W,
+    elt_tag: Tag,
+    length: usize,
+    data: *const (),
+    write_tags: bool,
+) -> Result<(), Error>
+where
+    W: Write + ?Sized,
+{
+    if write_tags {
+        writer.write_u8(elt_tag.as_u8())?;
+    }
     match elt_tag {
         // we cannot use NativeEndian::from_slice_i32 as the data is not mutable,
         // and that is not needed as the data is already in native endian
@@ -245,14 +226,14 @@ where W: Write + ?Sized {
         _ => {
             let mut data = data;
             for _ in 0..length {
-                send_value(writer, elt_tag, &mut data)?;
+                send_value(writer, elt_tag, &mut data, write_tags)?;
             }
         }
     }
     Ok(())
 }
 
-unsafe fn send_value<W>(writer: &mut W, tag: Tag, data: &mut *const ()) -> Result<(), Error>
+unsafe fn send_value<W>(writer: &mut W, tag: Tag, data: &mut *const (), write_tags: bool) -> Result<(), Error>
 where W: Write + ?Sized {
     macro_rules! consume_value {
         ($ty:ty, | $ptr:ident | $map:expr) => {{
@@ -262,7 +243,9 @@ where W: Write + ?Sized {
         }};
     }
 
-    writer.write_u8(tag.as_u8())?;
+    if write_tags {
+        writer.write_u8(tag.as_u8())?;
+    }
     match tag {
         Tag::None => Ok(()),
         Tag::Bool => consume_value!(u8, |ptr| writer.write_u8(*ptr)),
@@ -274,12 +257,14 @@ where W: Write + ?Sized {
         Tag::Bytes | Tag::ByteArray => consume_value!(CSlice<u8>, |ptr| writer.write_bytes((*ptr).as_ref())),
         Tag::Tuple(it, arity) => {
             let mut it = it.clone();
-            writer.write_u8(arity)?;
+            if write_tags {
+                writer.write_u8(arity)?;
+            }
             let mut max_alignment = 0;
             for _ in 0..arity {
                 let tag = it.next().expect("truncated tag");
                 max_alignment = core::cmp::max(max_alignment, tag.alignment());
-                send_value(writer, tag, data)?
+                send_value(writer, tag, data, write_tags)?
             }
             *data = round_up_const(*data, max_alignment);
             Ok(())
@@ -294,11 +279,13 @@ where W: Write + ?Sized {
                 let length = (**ptr).length as usize;
                 writer.write_u32((*ptr).length)?;
                 let tag = it.clone().next().expect("truncated tag");
-                send_elements(writer, tag, length, (**ptr).elements)
+                send_elements(writer, tag, length, (**ptr).elements, write_tags)
             })
         }
         Tag::Array(it, num_dims) => {
-            writer.write_u8(num_dims)?;
+            if write_tags {
+                writer.write_u8(num_dims)?;
+            }
             consume_value!(*const (), |buffer| {
                 let elt_tag = it.clone().next().expect("truncated tag");
 
@@ -310,14 +297,14 @@ where W: Write + ?Sized {
                     })
                 }
                 let length = total_len as usize;
-                send_elements(writer, elt_tag, length, *buffer)
+                send_elements(writer, elt_tag, length, *buffer, write_tags)
             })
         }
         Tag::Range(it) => {
             let tag = it.clone().next().expect("truncated tag");
-            send_value(writer, tag, data)?;
-            send_value(writer, tag, data)?;
-            send_value(writer, tag, data)?;
+            send_value(writer, tag, data, write_tags)?;
+            send_value(writer, tag, data, write_tags)?;
+            send_value(writer, tag, data, write_tags)?;
             Ok(())
         }
         Tag::Keyword(it) => {
@@ -329,7 +316,7 @@ where W: Write + ?Sized {
                 writer.write_string(str::from_utf8((*ptr).name.as_ref()).unwrap())?;
                 let tag = it.clone().next().expect("truncated tag");
                 let mut data = ptr.offset(1) as *const ();
-                send_value(writer, tag, &mut data)
+                send_value(writer, tag, &mut data, write_tags)
             })
             // Tag::Keyword never appears in composite types, so we don't have
             // to accurately advance data.
@@ -344,8 +331,16 @@ where W: Write + ?Sized {
     }
 }
 
-pub fn send_args<W>(writer: &mut W, service: u32, tag_bytes: &[u8], data: *const *const ()) -> Result<(), Error>
-where W: Write + ?Sized {
+pub fn send_args<W>(
+    writer: &mut W,
+    service: u32,
+    tag_bytes: &[u8],
+    data: *const *const (),
+    write_tags: bool,
+) -> Result<(), Error>
+where
+    W: Write + ?Sized,
+{
     let (arg_tags_bytes, return_tag_bytes) = split_tag(tag_bytes);
 
     let mut args_it = TagIterator::new(arg_tags_bytes);
@@ -356,7 +351,7 @@ where W: Write + ?Sized {
     for index in 0.. {
         if let Some(arg_tag) = args_it.next() {
             let mut data = unsafe { *data.offset(index) };
-            unsafe { send_value(writer, arg_tag, &mut data)? };
+            unsafe { send_value(writer, arg_tag, &mut data, write_tags)? };
         } else {
             break;
         }
@@ -367,7 +362,7 @@ where W: Write + ?Sized {
     Ok(())
 }
 
-mod tag {
+pub mod tag {
     use core::fmt;
 
     pub fn split_tag(tag_bytes: &[u8]) -> (&[u8], &[u8]) {
@@ -484,7 +479,7 @@ mod tag {
 
     #[derive(Debug, Clone, Copy)]
     pub struct TagIterator<'a> {
-        data: &'a [u8],
+        pub data: &'a [u8],
     }
 
     impl<'a> TagIterator<'a> {

src/libksupport/src/rtio_acp.rs

@@ -4,13 +4,14 @@ use cslice::CSlice;
 use libcortex_a9::asm;
 use vcell::VolatileCell;
 
-use crate::{artiq_raise, pl::csr, rtio_mgt::resolve_channel_name};
+use crate::{artiq_raise, pl::csr, resolve_channel_name, rtio_core};
 
 pub const RTIO_O_STATUS_WAIT: i32 = 1;
 pub const RTIO_O_STATUS_UNDERFLOW: i32 = 2;
 pub const RTIO_O_STATUS_DESTINATION_UNREACHABLE: i32 = 4;
 pub const RTIO_I_STATUS_WAIT_EVENT: i32 = 1;
 pub const RTIO_I_STATUS_OVERFLOW: i32 = 2;
+#[allow(unused)]
 pub const RTIO_I_STATUS_WAIT_STATUS: i32 = 4; // TODO
 pub const RTIO_I_STATUS_DESTINATION_UNREACHABLE: i32 = 8;
 
@@ -51,7 +52,7 @@ static mut TRANSACTION_BUFFER: Transaction = Transaction {
 
 pub extern "C" fn init() {
     unsafe {
-        csr::rtio_core::reset_write(1);
+        rtio_core::reset_write(1);
         csr::rtio::engine_addr_base_write(&TRANSACTION_BUFFER as *const Transaction as u32);
         csr::rtio::enable_write(1);
     }

src/libksupport/src/rtio_csr.rs

@@ -2,7 +2,7 @@ use core::ptr::{read_volatile, write_volatile};
 
 use cslice::CSlice;
 
-use crate::{artiq_raise, pl::csr, rtio_mgt::resolve_channel_name};
+use crate::{artiq_raise, pl::csr, resolve_channel_name, rtio_core};
 
 pub const RTIO_O_STATUS_WAIT: u8 = 1;
 pub const RTIO_O_STATUS_UNDERFLOW: u8 = 2;
@@ -20,7 +20,7 @@ pub struct TimestampedData {
 
 pub extern "C" fn init() {
     unsafe {
-        csr::rtio_core::reset_write(1);
+        rtio_core::reset_write(1);
     }
 }
 
@@ -107,7 +107,7 @@ pub extern "C" fn output(target: i32, data: i32) {
     }
 }
 
-pub extern "C" fn output_wide(target: i32, data: CSlice<i32>) {
+pub extern "C" fn output_wide(target: i32, data: &CSlice<i32>) {
     unsafe {
         csr::rtio::target_write(target as u32);
         // writing target clears o_data

src/runtime/Cargo.toml.tpl

@@ -18,7 +18,6 @@ num-traits = { version = "0.2", default-features = false }
 num-derive = "0.3"
 cslice = "0.3"
 log = "0.4"
-nb = "0.1"
 embedded-hal = "0.2"
 core_io = { version = "0.1", features = ["collections"] }
 byteorder = { version = "1.3", default-features = false }
@@ -26,19 +25,23 @@ void = { version = "1", default-features = false }
 futures = { version = "0.3", default-features = false, features = ["async-await"] }
 async-recursion = "0.3"
 log_buffer = { version = "1.2" }
-libm = { version = "0.2", features = ["unstable"] }
 vcell = "0.1"
 
-libboard_zynq = { git = "https://git.m-labs.hk/M-Labs/zynq-rs.git", features = ["ipv6"]}
-libsupport_zynq = { default-features = false, features = ["alloc_core"], git = "https://git.m-labs.hk/M-Labs/zynq-rs.git" }
-libcortex_a9 = { git = "https://git.m-labs.hk/M-Labs/zynq-rs.git" }
-libasync = { git = "https://git.m-labs.hk/M-Labs/zynq-rs.git" }
-libregister = { git = "https://git.m-labs.hk/M-Labs/zynq-rs.git" }
-libconfig = { git = "https://git.m-labs.hk/M-Labs/zynq-rs.git", features = ["fat_lfn", "ipv6"] }
+libboard_zynq = { path = "@@ZYNQ_RS@@/libboard_zynq", features = ["ipv6"]}
+libsupport_zynq = { path = "@@ZYNQ_RS@@/libsupport_zynq", default-features = false, features = ["alloc_core"] }
+libcortex_a9 = { path = "@@ZYNQ_RS@@/libcortex_a9" }
+libasync = { path = "@@ZYNQ_RS@@/libasync" }
+libregister = { path = "@@ZYNQ_RS@@/libregister" }
+libconfig = { path = "@@ZYNQ_RS@@/libconfig", features = ["fat_lfn", "ipv6"] }
 
 dyld = { path = "../libdyld" }
 dwarf = { path = "../libdwarf" }
 unwind = { path = "../libunwind" }
 libc = { path = "../libc" }
-io = { path = "../libio" }
-libboard_artiq = { path = "../libboard_artiq" }
+io = { path = "../libio", features = ["alloc"] }
+ksupport = { path = "../libksupport" }
+libboard_artiq = { path = "../libboard_artiq" }
+
+[dependencies.tar-no-std]
+git = "https://git.m-labs.hk/M-Labs/tar-no-std"
+rev = "2ab6dc5"

src/runtime/src/analyzer.rs

@@ -1,6 +1,12 @@
+use alloc::rc::Rc;
+#[cfg(has_drtio)]
+use alloc::vec::Vec;
+use core::cell::RefCell;
+
 use libasync::{smoltcp::TcpStream, task};
-use libboard_zynq::smoltcp::Error;
-use libcortex_a9::cache;
+use libboard_artiq::drtio_routing;
+use libboard_zynq::{smoltcp::Error, timer::GlobalTimer};
+use libcortex_a9::{cache, mutex::Mutex};
 use log::{debug, info, warn};
 
 use crate::{pl, proto_async::*};
@@ -37,6 +43,50 @@ fn disarm() {
     debug!("RTIO analyzer disarmed");
 }
 
+#[cfg(has_drtio)]
+pub mod remote_analyzer {
+    use super::*;
+    use crate::rtio_mgt::drtio;
+
+    pub struct RemoteBuffer {
+        pub total_byte_count: u64,
+        pub sent_bytes: u32,
+        pub error: bool,
+        pub data: Vec<u8>,
+    }
+
+    pub async fn get_data(
+        aux_mutex: &Rc<Mutex<bool>>,
+        routing_table: &drtio_routing::RoutingTable,
+        up_destinations: &Rc<RefCell<[bool; drtio_routing::DEST_COUNT]>>,
+        timer: GlobalTimer,
+    ) -> Result<RemoteBuffer, drtio::Error> {
+        // gets data from satellites and returns consolidated data
+        let mut remote_data: Vec<u8> = Vec::new();
+        let mut remote_error = false;
+        let mut remote_sent_bytes = 0;
+        let mut remote_total_bytes = 0;
+
+        let data_vec = match drtio::analyzer_query(aux_mutex, routing_table, up_destinations, timer).await {
+            Ok(data_vec) => data_vec,
+            Err(e) => return Err(e),
+        };
+        for data in data_vec {
+            remote_total_bytes += data.total_byte_count;
+            remote_sent_bytes += data.sent_bytes;
+            remote_error |= data.error;
+            remote_data.extend(data.data);
+        }
+
+        Ok(RemoteBuffer {
+            total_byte_count: remote_total_bytes,
+            sent_bytes: remote_sent_bytes,
+            error: remote_error,
+            data: remote_data,
+        })
+    }
+}
+
 #[derive(Debug)]
 struct Header {
     sent_bytes: u32,
@@ -56,7 +106,13 @@ async fn write_header(stream: &mut TcpStream, header: &Header) -> Result<(), Err
     Ok(())
 }
 
-async fn handle_connection(stream: &mut TcpStream) -> Result<(), Error> {
+async fn handle_connection(
+    stream: &mut TcpStream,
+    _aux_mutex: &Rc<Mutex<bool>>,
+    _routing_table: &drtio_routing::RoutingTable,
+    _up_destinations: &Rc<RefCell<[bool; drtio_routing::DEST_COUNT]>>,
+    _timer: GlobalTimer,
+) -> Result<(), Error> {
     info!("received connection");
 
     let data = unsafe { &BUFFER.data[..] };
@@ -65,6 +121,11 @@ async fn handle_connection(stream: &mut TcpStream) -> Result<(), Error> {
     let total_byte_count = unsafe { pl::csr::rtio_analyzer::dma_byte_count_read() as u64 };
     let pointer = (total_byte_count % BUFFER_SIZE as u64) as usize;
     let wraparound = total_byte_count >= BUFFER_SIZE as u64;
+    let sent_bytes = if wraparound {
+        BUFFER_SIZE as u32
+    } else {
+        total_byte_count as u32
+    };
 
     if overflow_occurred {
         warn!("overflow occured");
@@ -73,13 +134,39 @@ async fn handle_connection(stream: &mut TcpStream) -> Result<(), Error> {
         warn!("bus error occured");
     }
 
+    #[cfg(has_drtio)]
+    let remote = remote_analyzer::get_data(_aux_mutex, _routing_table, _up_destinations, _timer).await;
+    #[cfg(has_drtio)]
+    let (header, remote_data) = match remote {
+        Ok(remote) => (
+            Header {
+                total_byte_count: total_byte_count + remote.total_byte_count,
+                sent_bytes: sent_bytes + remote.sent_bytes,
+                error_occurred: overflow_occurred | bus_error_occurred | remote.error,
+                log_channel: pl::csr::CONFIG_RTIO_LOG_CHANNEL as u8,
+                dds_onehot_sel: true,
+            },
+            remote.data,
+        ),
+        Err(e) => {
+            warn!("Error getting remote analyzer data: {}", e);
+            (
+                Header {
+                    total_byte_count: total_byte_count,
+                    sent_bytes: sent_bytes,
+                    error_occurred: true,
+                    log_channel: pl::csr::CONFIG_RTIO_LOG_CHANNEL as u8,
+                    dds_onehot_sel: true,
+                },
+                Vec::new(),
+            )
+        }
+    };
+
+    #[cfg(not(has_drtio))]
     let header = Header {
         total_byte_count: total_byte_count,
-        sent_bytes: if wraparound {
-            BUFFER_SIZE as u32
-        } else {
-            total_byte_count as u32
-        },
+        sent_bytes: sent_bytes,
         error_occurred: overflow_occurred | bus_error_occurred,
         log_channel: pl::csr::CONFIG_RTIO_LOG_CHANNEL as u8,
         dds_onehot_sel: true, // kept for backward compatibility of analyzer dumps
@@ -93,17 +180,28 @@ async fn handle_connection(stream: &mut TcpStream) -> Result<(), Error> {
     } else {
         stream.send(data[..pointer].iter().copied()).await?;
     }
+    #[cfg(has_drtio)]
+    stream.send(remote_data.iter().copied()).await?;
 
     Ok(())
 }
 
-pub fn start() {
+pub fn start(
+    aux_mutex: &Rc<Mutex<bool>>,
+    routing_table: &Rc<RefCell<drtio_routing::RoutingTable>>,
+    up_destinations: &Rc<RefCell<[bool; drtio_routing::DEST_COUNT]>>,
+    timer: GlobalTimer,
+) {
+    let aux_mutex = aux_mutex.clone();
+    let routing_table = routing_table.clone();
+    let up_destinations = up_destinations.clone();
     task::spawn(async move {
         loop {
             arm();
             let mut stream = TcpStream::accept(1382, 2048, 2048).await.unwrap();
             disarm();
-            let _ = handle_connection(&mut stream)
+            let routing_table = routing_table.borrow();
+            let _ = handle_connection(&mut stream, &aux_mutex, &routing_table, &up_destinations, timer)
                 .await
                 .map_err(|e| warn!("connection terminated: {:?}", e));
             let _ = stream.flush().await;

src/runtime/src/comms.rs

@@ -1,13 +1,24 @@
 use alloc::{collections::BTreeMap, rc::Rc, string::String, vec, vec::Vec};
 use core::{cell::RefCell, fmt, slice, str};
 
+use core_io::Error as IoError;
 use cslice::CSlice;
+use dyld::elf;
 use futures::{future::FutureExt, select_biased};
+#[cfg(has_drtio)]
+use io::Cursor;
+#[cfg(has_drtio)]
+use ksupport::rpc;
+use ksupport::{kernel, resolve_channel_name};
+#[cfg(has_drtio)]
+use libasync::delay;
 use libasync::{smoltcp::{Sockets, TcpStream},
                task};
 use libboard_artiq::drtio_routing;
 #[cfg(feature = "target_kasli_soc")]
 use libboard_zynq::error_led::ErrorLED;
+#[cfg(has_drtio)]
+use libboard_zynq::time::Milliseconds;
 use libboard_zynq::{self as zynq,
                     smoltcp::{self,
                               iface::{EthernetInterfaceBuilder, NeighborCache},
@@ -21,20 +32,26 @@ use libcortex_a9::{mutex::Mutex,
 use log::{error, info, warn};
 use num_derive::{FromPrimitive, ToPrimitive};
 use num_traits::{FromPrimitive, ToPrimitive};
+#[cfg(has_drtio)]
+use tar_no_std::TarArchiveRef;
 
 #[cfg(has_drtio)]
 use crate::pl;
-use crate::{analyzer, kernel, mgmt, moninj,
-            proto_async::*,
-            rpc, rtio_dma,
-            rtio_mgt::{self, resolve_channel_name}};
+use crate::{analyzer, mgmt, moninj, proto_async::*, rpc_async, rtio_dma, rtio_mgt};
+#[cfg(has_drtio)]
+use crate::{subkernel, subkernel::Error as SubkernelError};
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub enum Error {
     NetworkError(smoltcp::Error),
+    IoError,
     UnexpectedPattern,
     UnrecognizedPacket,
     BufferExhausted,
+    #[cfg(has_drtio)]
+    SubkernelError(subkernel::Error),
+    #[cfg(has_drtio)]
+    DestinationDown,
 }
 
 pub type Result<T> = core::result::Result<T, Error>;
@@ -43,9 +60,14 @@ impl fmt::Display for Error {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         match self {
             Error::NetworkError(error) => write!(f, "network error: {}", error),
+            Error::IoError => write!(f, "io error"),
             Error::UnexpectedPattern => write!(f, "unexpected pattern"),
             Error::UnrecognizedPacket => write!(f, "unrecognized packet"),
             Error::BufferExhausted => write!(f, "buffer exhausted"),
+            #[cfg(has_drtio)]
+            Error::SubkernelError(error) => write!(f, "subkernel error: {:?}", error),
+            #[cfg(has_drtio)]
+            Error::DestinationDown => write!(f, "subkernel destination down"),
         }
     }
 }
@@ -56,6 +78,19 @@ impl From<smoltcp::Error> for Error {
     }
 }
 
+impl From<IoError> for Error {
+    fn from(_error: IoError) -> Self {
+        Error::IoError
+    }
+}
+
+#[cfg(has_drtio)]
+impl From<subkernel::Error> for Error {
+    fn from(error: subkernel::Error) -> Self {
+        Error::SubkernelError(error)
+    }
+}
+
 #[derive(Debug, FromPrimitive, ToPrimitive)]
 enum Request {
     SystemInfo = 3,
@@ -63,6 +98,7 @@ enum Request {
     RunKernel = 6,
     RPCReply = 7,
     RPCException = 8,
+    UploadSubkernel = 9,
 }
 
 #[derive(Debug, FromPrimitive, ToPrimitive)]
@@ -182,7 +218,7 @@ async fn handle_run_kernel(
                                 kernel::Message::RpcRecvRequest(slot) => slot,
                                 other => panic!("expected root value slot from core1, not {:?}", other),
                             };
-                            rpc::recv_return(stream, &tag, slot, &|size| {
+                            rpc_async::recv_return(stream, &tag, slot, &|size| {
                                 let control = control.clone();
                                 async move {
                                     if size == 0 {
@@ -227,7 +263,7 @@ async fn handle_run_kernel(
                             let function = read_i32(stream).await? as u32;
                             control
                                 .tx
-                                .async_send(kernel::Message::RpcRecvReply(Err(kernel::RPCException {
+                                .async_send(kernel::Message::RpcRecvReply(Err(ksupport::RPCException {
                                     id,
                                     message,
                                     param,
@@ -330,7 +366,7 @@ async fn handle_run_kernel(
                 rtio_dma::erase(name, aux_mutex, routing_table, timer).await;
             }
             kernel::Message::DmaGetRequest(name) => {
-                let result = rtio_dma::retrieve(name);
+                let result = rtio_dma::retrieve(name).await;
                 control
                     .borrow_mut()
                     .tx
@@ -365,6 +401,143 @@ async fn handle_run_kernel(
                 control.borrow_mut().tx.async_send(reply).await;
             }
             #[cfg(has_drtio)]
+            kernel::Message::SubkernelLoadRunRequest {
+                id,
+                destination: _,
+                run,
+            } => {
+                let succeeded = match subkernel::load(aux_mutex, routing_table, timer, id, run).await {
+                    Ok(()) => true,
+                    Err(e) => {
+                        error!("Error loading subkernel: {:?}", e);
+                        false
+                    }
+                };
+                control
+                    .borrow_mut()
+                    .tx
+                    .async_send(kernel::Message::SubkernelLoadRunReply { succeeded: succeeded })
+                    .await;
+            }
+            #[cfg(has_drtio)]
+            kernel::Message::SubkernelAwaitFinishRequest { id, timeout } => {
+                let res = subkernel::await_finish(aux_mutex, routing_table, timer, id, timeout).await;
+                let status = match res {
+                    Ok(ref res) => {
+                        if res.status == subkernel::FinishStatus::CommLost {
+                            kernel::SubkernelStatus::CommLost
+                        } else if let Some(exception) = &res.exception {
+                            error!("Exception in subkernel");
+                            match stream {
+                                None => (),
+                                Some(stream) => {
+                                    write_chunk(stream, exception).await?;
+                                }
+                            }
+                            // will not be called after exception is served
+                            kernel::SubkernelStatus::OtherError
+                        } else {
+                            kernel::SubkernelStatus::NoError
+                        }
+                    }
+                    Err(SubkernelError::Timeout) => kernel::SubkernelStatus::Timeout,
+                    Err(SubkernelError::IncorrectState) => kernel::SubkernelStatus::IncorrectState,
+                    Err(_) => kernel::SubkernelStatus::OtherError,
+                };
+                control
+                    .borrow_mut()
+                    .tx
+                    .async_send(kernel::Message::SubkernelAwaitFinishReply { status: status })
+                    .await;
+            }
+            #[cfg(has_drtio)]
+            kernel::Message::SubkernelMsgSend { id, destination, data } => {
+                let res =
+                    subkernel::message_send(aux_mutex, routing_table, timer, id, destination.unwrap(), data).await;
+                match res {
+                    Ok(_) => (),
+                    Err(e) => {
+                        error!("error sending subkernel message: {:?}", e)
+                    }
+                };
+                control
+                    .borrow_mut()
+                    .tx
+                    .async_send(kernel::Message::SubkernelMsgSent)
+                    .await;
+            }
+            #[cfg(has_drtio)]
+            kernel::Message::SubkernelMsgRecvRequest { id, timeout, tags } => {
+                let message_received = subkernel::message_await(id as u32, timeout, timer).await;
+                let (status, count) = match message_received {
+                    Ok(ref message) => (kernel::SubkernelStatus::NoError, message.count),
+                    Err(SubkernelError::Timeout) => (kernel::SubkernelStatus::Timeout, 0),
+                    Err(SubkernelError::IncorrectState) => (kernel::SubkernelStatus::IncorrectState, 0),
+                    Err(SubkernelError::CommLost) => (kernel::SubkernelStatus::CommLost, 0),
+                    Err(SubkernelError::SubkernelException) => {
+                        error!("Exception in subkernel");
+                        // just retrieve the exception
+                        let status = subkernel::await_finish(aux_mutex, routing_table, timer, id as u32, timeout)
+                            .await
+                            .unwrap();
+                        match stream {
+                            None => (),
+                            Some(stream) => {
+                                write_chunk(stream, &status.exception.unwrap()).await?;
+                            }
+                        }
+                        (kernel::SubkernelStatus::OtherError, 0)
+                    }
+                    Err(_) => (kernel::SubkernelStatus::OtherError, 0),
+                };
+                control
+                    .borrow_mut()
+                    .tx
+                    .async_send(kernel::Message::SubkernelMsgRecvReply {
+                        status: status,
+                        count: count,
+                    })
+                    .await;
+                if let Ok(message) = message_received {
+                    // receive code almost identical to RPC recv, except we are not reading from a stream
+                    let mut reader = Cursor::new(message.data);
+                    let mut current_tags: &[u8] = &tags;
+                    let mut i = 0;
+                    loop {
+                        // kernel has to consume all arguments in the whole message
+                        let slot = match fast_recv(&mut control.borrow_mut().rx).await {
+                            kernel::Message::RpcRecvRequest(slot) => slot,
+                            other => panic!("expected root value slot from core1, not {:?}", other),
+                        };
+                        let remaining_tags = rpc::recv_return(&mut reader, &current_tags, slot, &mut |size| {
+                            if size == 0 {
+                                0 as *mut ()
+                            } else {
+                                let mut control = control.borrow_mut();
+                                control.tx.send(kernel::Message::RpcRecvReply(Ok(size)));
+                                match control.rx.recv() {
+                                    kernel::Message::RpcRecvRequest(slot) => slot,
+                                    other => {
+                                        panic!("expected nested value slot from kernel CPU, not {:?}", other)
+                                    }
+                                }
+                            }
+                        })?;
+                        control
+                            .borrow_mut()
+                            .tx
+                            .async_send(kernel::Message::RpcRecvReply(Ok(0)))
+                            .await;
+                        i += 1;
+                        if i < count {
+                            current_tags = remaining_tags;
+                        } else {
+                            break;
+                        }
+                    }
+                }
+            }
+            #[cfg(has_drtio)]
             kernel::Message::UpDestinationsRequest(destination) => {
                 let result = _up_destinations.borrow()[destination as usize];
                 control
@@ -381,6 +554,56 @@ async fn handle_run_kernel(
     Ok(())
 }
 
+async fn handle_flash_kernel(
+    buffer: &Vec<u8>,
+    control: &Rc<RefCell<kernel::Control>>,
+    _up_destinations: &Rc<RefCell<[bool; drtio_routing::DEST_COUNT]>>,
+    _aux_mutex: &Rc<Mutex<bool>>,
+    _routing_table: &drtio_routing::RoutingTable,
+    _timer: GlobalTimer,
+) -> Result<()> {
+    if buffer[0] == elf::ELFMAG0 && buffer[1] == elf::ELFMAG1 && buffer[2] == elf::ELFMAG2 && buffer[3] == elf::ELFMAG3
+    {
+        // assume ELF file, proceed as before
+        load_kernel(buffer, control, None).await
+    } else {
+        #[cfg(has_drtio)]
+        {
+            let archive = TarArchiveRef::new(buffer.as_ref());
+            let entries = archive.entries();
+            let mut main_lib: Vec<u8> = Vec::new();
+            for entry in entries {
+                if entry.filename().as_str() == "main.elf" {
+                    main_lib = entry.data().to_vec();
+                } else {
+                    // subkernel filename must be in format:
+                    // "<subkernel id> <destination>.elf"
+                    let filename = entry.filename();
+                    let mut iter = filename.as_str().split_whitespace();
+                    let sid: u32 = iter.next().unwrap().parse().unwrap();
+                    let dest: u8 = iter.next().unwrap().strip_suffix(".elf").unwrap().parse().unwrap();
+                    let up = _up_destinations.borrow()[dest as usize];
+                    if up {
+                        let subkernel_lib = entry.data().to_vec();
+                        subkernel::add_subkernel(sid, dest, subkernel_lib).await;
+                        match subkernel::upload(_aux_mutex, _routing_table, _timer, sid).await {
+                            Ok(_) => (),
+                            Err(_) => return Err(Error::UnexpectedPattern),
+                        }
+                    } else {
+                        return Err(Error::DestinationDown);
+                    }
+                }
+            }
+            load_kernel(&main_lib, control, None).await
+        }
+        #[cfg(not(has_drtio))]
+        {
+            panic!("multi-kernel libraries are not supported in standalone systems");
+        }
+    }
+}
+
 async fn load_kernel(
     buffer: &Vec<u8>,
     control: &Rc<RefCell<kernel::Control>>,
@@ -434,9 +657,13 @@ async fn handle_connection(
         return Err(Error::UnexpectedPattern);
     }
     stream.send_slice("e".as_bytes()).await?;
+    #[cfg(has_drtio)]
+    subkernel::clear_subkernels().await;
     loop {
         let request = read_request(stream, true).await?;
         if request.is_none() {
+            #[cfg(has_drtio)]
+            subkernel::clear_subkernels().await;
             return Ok(());
         }
         let request = request.unwrap();
@@ -460,6 +687,29 @@ async fn handle_connection(
                 )
                 .await?;
             }
+            Request::UploadSubkernel => {
+                #[cfg(has_drtio)]
+                {
+                    let id = read_i32(stream).await? as u32;
+                    let destination = read_i8(stream).await? as u8;
+                    let buffer = read_bytes(stream, 1024 * 1024).await?;
+                    subkernel::add_subkernel(id, destination, buffer).await;
+                    match subkernel::upload(aux_mutex, routing_table, timer, id).await {
+                        Ok(_) => write_header(stream, Reply::LoadCompleted).await?,
+                        Err(_) => {
+                            write_header(stream, Reply::LoadFailed).await?;
+                            write_chunk(stream, b"subkernel failed to load").await?;
+                            return Err(Error::UnexpectedPattern);
+                        }
+                    }
+                }
+                #[cfg(not(has_drtio))]
+                {
+                    write_header(stream, Reply::LoadFailed).await?;
+                    write_chunk(stream, b"No DRTIO on this system, subkernels are not supported").await?;
+                    return Err(Error::UnexpectedPattern);
+                }
+            }
             _ => {
                 error!("unexpected request from host: {:?}", request);
                 return Err(Error::UnrecognizedPacket);
@@ -509,7 +759,6 @@ pub fn main(timer: GlobalTimer, cfg: Config) {
 
     Sockets::init(32);
 
-    // before, mutex was on io, but now that io isn't used...?
     let aux_mutex: Rc<Mutex<bool>> = Rc::new(Mutex::new(false));
     #[cfg(has_drtio)]
     let drtio_routing_table = Rc::new(RefCell::new(drtio_routing::config_routing_table(
@@ -522,18 +771,26 @@ pub fn main(timer: GlobalTimer, cfg: Config) {
     #[cfg(has_drtio_routing)]
     drtio_routing::interconnect_disable_all();
 
-    rtio_mgt::startup(&aux_mutex, &drtio_routing_table, &up_destinations, timer, &cfg);
+    rtio_mgt::startup(&aux_mutex, &drtio_routing_table, &up_destinations, timer);
+    ksupport::setup_device_map(&cfg);
 
-    analyzer::start();
+    analyzer::start(&aux_mutex, &drtio_routing_table, &up_destinations, timer);
     moninj::start(timer, &aux_mutex, &drtio_routing_table);
 
     let control: Rc<RefCell<kernel::Control>> = Rc::new(RefCell::new(kernel::Control::start()));
     let idle_kernel = Rc::new(cfg.read("idle_kernel").ok());
     if let Ok(buffer) = cfg.read("startup_kernel") {
         info!("Loading startup kernel...");
-        if let Ok(()) = task::block_on(load_kernel(&buffer, &control, None)) {
+        let routing_table = drtio_routing_table.borrow();
+        if let Ok(()) = task::block_on(handle_flash_kernel(
+            &buffer,
+            &control,
+            &up_destinations,
+            &aux_mutex,
+            &routing_table,
+            timer,
+        )) {
             info!("Starting startup kernel...");
-            let routing_table = drtio_routing_table.borrow();
             let _ = task::block_on(handle_run_kernel(
                 None,
                 &control,
@@ -581,8 +838,17 @@ pub fn main(timer: GlobalTimer, cfg: Config) {
                             .map_err(|e| warn!("connection terminated: {}", e));
                         if let Some(buffer) = &*idle_kernel {
                             info!("Loading idle kernel");
-                            let _ = load_kernel(&buffer, &control, None)
-                                .await.map_err(|_| warn!("error loading idle kernel"));
+                            let res = handle_flash_kernel(&buffer, &control, &up_destinations,  &aux_mutex, &routing_table, timer)
+                                .await;
+                            match res {
+                                #[cfg(has_drtio)]
+                                Err(Error::DestinationDown) => {
+                                    let mut countdown = timer.countdown();
+                                    delay(&mut countdown, Milliseconds(500)).await;
+                                }
+                                Err(_) => warn!("error loading idle kernel"),
+                                _ => (),
+                            }
                             info!("Running idle kernel");
                             let _ = handle_run_kernel(None, &control, &up_destinations, &aux_mutex, &routing_table, timer)
                                 .await.map_err(|_| warn!("error running idle kernel"));

src/runtime/src/main.rs

@@ -2,102 +2,78 @@
 #![no_main]
 #![recursion_limit = "1024"] // for futures_util::select!
 #![feature(alloc_error_handler)]
-#![feature(panic_info_message)]
-#![feature(c_variadic)]
 #![feature(const_btree_new)]
-#![feature(const_in_array_repeat_expressions)]
-#![feature(naked_functions)]
-#![feature(asm)]
+#![feature(panic_info_message)]
 
 #[macro_use]
 extern crate alloc;
 
-use libasync::{block_async, task};
+#[cfg(all(feature = "target_kasli_soc", has_virtual_leds))]
+use core::cell::RefCell;
+
+use ksupport;
+use libasync::task;
+#[cfg(has_drtio_eem)]
+use libboard_artiq::drtio_eem;
 #[cfg(feature = "target_kasli_soc")]
 use libboard_artiq::io_expander;
 use libboard_artiq::{identifier_read, logger, pl};
 use libboard_zynq::{gic, mpcore, timer::GlobalTimer};
 use libconfig::Config;
 use libcortex_a9::l2c::enable_l2_cache;
-use libsupport_zynq::ram;
-use log::{error, info, warn};
-use nb;
-use void::Void;
-
-const ASYNC_ERROR_COLLISION: u8 = 1 << 0;
-const ASYNC_ERROR_BUSY: u8 = 1 << 1;
-const ASYNC_ERROR_SEQUENCE_ERROR: u8 = 1 << 2;
+use libsupport_zynq::{exception_vectors, ram};
+use log::{info, warn};
 
 mod analyzer;
 mod comms;
-mod eh_artiq;
-mod i2c;
-mod irq;
-mod kernel;
+
 mod mgmt;
 mod moninj;
 mod panic;
 mod proto_async;
-mod rpc;
-#[cfg(ki_impl = "csr")]
-#[path = "rtio_csr.rs"]
-mod rtio;
-#[cfg(ki_impl = "acp")]
-#[path = "rtio_acp.rs"]
-mod rtio;
+mod rpc_async;
 mod rtio_clocking;
 mod rtio_dma;
 mod rtio_mgt;
+#[cfg(has_drtio)]
+mod subkernel;
 
-static mut SEEN_ASYNC_ERRORS: u8 = 0;
-
-pub unsafe fn get_async_errors() -> u8 {
-    let errors = SEEN_ASYNC_ERRORS;
-    SEEN_ASYNC_ERRORS = 0;
-    errors
+// linker symbols
+extern "C" {
+    static __exceptions_start: u32;
 }
 
-fn wait_for_async_rtio_error() -> nb::Result<(), Void> {
-    unsafe {
-        if pl::csr::rtio_core::async_error_read() != 0 {
-            Ok(())
-        } else {
-            Err(nb::Error::WouldBlock)
-        }
+#[cfg(all(feature = "target_kasli_soc", has_virtual_leds))]
+async fn io_expanders_service(
+    i2c_bus: RefCell<&mut libboard_zynq::i2c::I2c>,
+    io_expander0: RefCell<io_expander::IoExpander>,
+    io_expander1: RefCell<io_expander::IoExpander>,
+) {
+    loop {
+        task::r#yield().await;
+        io_expander0
+            .borrow_mut()
+            .service(&mut i2c_bus.borrow_mut())
+            .expect("I2C I/O expander #0 service failed");
+        io_expander1
+            .borrow_mut()
+            .service(&mut i2c_bus.borrow_mut())
+            .expect("I2C I/O expander #1 service failed");
     }
 }
 
-async fn report_async_rtio_errors() {
-    loop {
-        let _ = block_async!(wait_for_async_rtio_error()).await;
-        unsafe {
-            let errors = pl::csr::rtio_core::async_error_read();
-            if errors & ASYNC_ERROR_COLLISION != 0 {
-                let channel = pl::csr::rtio_core::collision_channel_read();
-                error!(
-                    "RTIO collision involving channel 0x{:04x}:{}",
-                    channel,
-                    rtio_mgt::resolve_channel_name(channel as u32)
-                );
-            }
-            if errors & ASYNC_ERROR_BUSY != 0 {
-                let channel = pl::csr::rtio_core::busy_channel_read();
-                error!(
-                    "RTIO busy error involving channel 0x{:04x}:{}",
-                    channel,
-                    rtio_mgt::resolve_channel_name(channel as u32)
-                );
-            }
-            if errors & ASYNC_ERROR_SEQUENCE_ERROR != 0 {
-                let channel = pl::csr::rtio_core::sequence_error_channel_read();
-                error!(
-                    "RTIO sequence error involving channel 0x{:04x}:{}",
-                    channel,
-                    rtio_mgt::resolve_channel_name(channel as u32)
-                );
-            }
-            SEEN_ASYNC_ERRORS = errors;
-            pl::csr::rtio_core::async_error_write(errors);
+#[cfg(has_grabber)]
+mod grabber {
+    use libasync::delay;
+    use libboard_artiq::grabber;
+    use libboard_zynq::time::Milliseconds;
+
+    use crate::GlobalTimer;
+    pub async fn grabber_thread(timer: GlobalTimer) {
+        let mut countdown = timer.countdown();
+        loop {
+            grabber::tick();
+            delay(&mut countdown, Milliseconds(200)).await;
         }
     }
 }
@@ -106,6 +82,9 @@ static mut LOG_BUFFER: [u8; 1 << 17] = [0; 1 << 17];
 
 #[no_mangle]
 pub fn main_core0() {
+    unsafe {
+        exception_vectors::set_vector_table(&__exceptions_start as *const u32 as u32);
+    }
     enable_l2_cache(0x8);
     let mut timer = GlobalTimer::start();
 
@@ -121,21 +100,36 @@ pub fn main_core0() {
 
     info!("gateware ident: {}", identifier_read(&mut [0; 64]));
 
-    i2c::init();
-
+    ksupport::i2c::init();
     #[cfg(feature = "target_kasli_soc")]
     {
-        let i2c = unsafe { (&mut i2c::I2C_BUS).as_mut().unwrap() };
-        for expander_i in 0..=1 {
-            let mut io_expander = io_expander::IoExpander::new(i2c, expander_i).unwrap();
-            io_expander.init().expect("I2C I/O expander #0 initialization failed");
-            // Actively drive TX_DISABLE to false on SFP0..3
-            io_expander.set_oe(0, 1 << 1).unwrap();
-            io_expander.set_oe(1, 1 << 1).unwrap();
-            io_expander.set(0, 1, false);
-            io_expander.set(1, 1, false);
-            io_expander.service().unwrap();
-        }
+        let i2c_bus = unsafe { (ksupport::i2c::I2C_BUS).as_mut().unwrap() };
+        let mut io_expander0 = io_expander::IoExpander::new(i2c_bus, 0).unwrap();
+        let mut io_expander1 = io_expander::IoExpander::new(i2c_bus, 1).unwrap();
+        io_expander0
+            .init(i2c_bus)
+            .expect("I2C I/O expander #0 initialization failed");
+        io_expander1
+            .init(i2c_bus)
+            .expect("I2C I/O expander #1 initialization failed");
+
+        // Drive CLK_SEL to true
+        #[cfg(has_si549)]
+        io_expander0.set(1, 7, true);
+
+        // Drive TX_DISABLE to false on SFP0..3
+        io_expander0.set(0, 1, false);
+        io_expander1.set(0, 1, false);
+        io_expander0.set(1, 1, false);
+        io_expander1.set(1, 1, false);
+        io_expander0.service(i2c_bus).unwrap();
+        io_expander1.service(i2c_bus).unwrap();
+        #[cfg(has_virtual_leds)]
+        task::spawn(io_expanders_service(
+            RefCell::new(i2c_bus),
+            RefCell::new(io_expander0),
+            RefCell::new(io_expander1),
+        ));
     }
 
     let cfg = match Config::new() {
@@ -148,7 +142,13 @@ pub fn main_core0() {
 
     rtio_clocking::init(&mut timer, &cfg);
 
-    task::spawn(report_async_rtio_errors());
+    #[cfg(has_drtio_eem)]
+    drtio_eem::init(&mut timer, &cfg);
+
+    #[cfg(has_grabber)]
+    task::spawn(grabber::grabber_thread(timer));
+
+    task::spawn(ksupport::report_async_rtio_errors());
 
     comms::main(timer, cfg);
 }

src/runtime/src/moninj.rs

@@ -58,10 +58,11 @@ mod remote_moninj {
     use log::error;
 
     use super::*;
-    use crate::rtio_mgt::drtio;
+    use crate::rtio_mgt::{drtio, drtio::Error as DrtioError};
 
     pub async fn read_probe(
         aux_mutex: &Rc<Mutex<bool>>,
+        routing_table: &drtio_routing::RoutingTable,
         timer: GlobalTimer,
         linkno: u8,
         destination: u8,
@@ -71,6 +72,7 @@ mod remote_moninj {
         let reply = drtio::aux_transact(
             aux_mutex,
             linkno,
+            routing_table,
             &drtioaux_async::Packet::MonitorRequest {
                 destination: destination,
                 channel: channel as _,
@@ -82,8 +84,8 @@ mod remote_moninj {
         match reply {
             Ok(drtioaux_async::Packet::MonitorReply { value }) => return value as i64,
             Ok(packet) => error!("received unexpected aux packet: {:?}", packet),
-            Err("link went down") => {
-                debug!("link is down");
+            Err(DrtioError::LinkDown) => {
+                warn!("link is down");
             }
             Err(e) => error!("aux packet error ({})", e),
         }
@@ -92,6 +94,7 @@ mod remote_moninj {
 
     pub async fn inject(
         aux_mutex: &Rc<Mutex<bool>>,
+        _routing_table: &drtio_routing::RoutingTable,
         _timer: GlobalTimer,
         linkno: u8,
         destination: u8,
@@ -115,6 +118,7 @@ mod remote_moninj {
 
     pub async fn read_injection_status(
         aux_mutex: &Rc<Mutex<bool>>,
+        routing_table: &drtio_routing::RoutingTable,
         timer: GlobalTimer,
         linkno: u8,
         destination: u8,
@@ -124,6 +128,7 @@ mod remote_moninj {
         let reply = drtio::aux_transact(
             aux_mutex,
             linkno,
+            routing_table,
             &drtioaux_async::Packet::InjectionStatusRequest {
                 destination: destination,
                 channel: channel as _,
@@ -135,8 +140,8 @@ mod remote_moninj {
         match reply {
             Ok(drtioaux_async::Packet::InjectionStatusReply { value }) => return value as i8,
             Ok(packet) => error!("received unexpected aux packet: {:?}", packet),
-            Err("link went down") => {
-                debug!("link is down");
+            Err(DrtioError::LinkDown) => {
+                warn!("link is down");
             }
             Err(e) => error!("aux packet error ({})", e),
         }
@@ -183,7 +188,7 @@ macro_rules! dispatch {
             local_moninj::$func(channel.into(), $($param, )*)
         } else {
             let linkno = hop - 1 as u8;
-            remote_moninj::$func($aux_mutex, $timer, linkno, destination, channel, $($param, )*).await
+            remote_moninj::$func($aux_mutex, $routing_table, $timer, linkno, destination, channel, $($param, )*).await
         }
     }}
 }

src/runtime/src/rpc_async.rs

@@ -0,0 +1,197 @@
+use alloc::boxed::Box;
+use core::future::Future;
+
+use async_recursion::async_recursion;
+use byteorder::{ByteOrder, NativeEndian};
+use cslice::CMutSlice;
+use ksupport::rpc::{tag::{Tag, TagIterator},
+                    *};
+use libasync::smoltcp::TcpStream;
+use libboard_zynq::smoltcp;
+use log::trace;
+
+use crate::proto_async;
+
+/// Reads (deserializes) `length` array or list elements of type `tag` from `stream`,
+/// writing them into the buffer given by `storage`.
+///
+/// `alloc` is used for nested allocations (if elements themselves contain
+/// lists/arrays), see [recv_value].
+#[async_recursion(?Send)]
+async unsafe fn recv_elements<F>(
+    stream: &TcpStream,
+    elt_tag: Tag<'async_recursion>,
+    length: usize,
+    storage: *mut (),
+    alloc: &(impl Fn(usize) -> F + 'async_recursion),
+) -> Result<(), smoltcp::Error>
+where
+    F: Future<Output = *mut ()>,
+{
+    // List of simple types are special-cased in the protocol for performance.
+    match elt_tag {
+        Tag::Bool => {
+            let dest = core::slice::from_raw_parts_mut(storage as *mut u8, length);
+            proto_async::read_chunk(stream, dest).await?;
+        }
+        Tag::Int32 => {
+            let ptr = storage as *mut u32;
+            let dest = core::slice::from_raw_parts_mut(ptr as *mut u8, length * 4);
+            proto_async::read_chunk(stream, dest).await?;
+            drop(dest);
+            let dest = core::slice::from_raw_parts_mut(ptr, length);
+            NativeEndian::from_slice_u32(dest);
+        }
+        Tag::Int64 | Tag::Float64 => {
+            let ptr = storage as *mut u64;
+            let dest = core::slice::from_raw_parts_mut(ptr as *mut u8, length * 8);
+            proto_async::read_chunk(stream, dest).await?;
+            drop(dest);
+            let dest = core::slice::from_raw_parts_mut(ptr, length);
+            NativeEndian::from_slice_u64(dest);
+        }
+        _ => {
+            let mut data = storage;
+            for _ in 0..length {
+                recv_value(stream, elt_tag, &mut data, alloc).await?
+            }
+        }
+    }
+    Ok(())
+}
+
+/// Reads (deserializes) a value of type `tag` from `stream`, writing the results to
+/// the kernel-side buffer `data` (the passed pointer to which is incremented to point
+/// past the just-received data). For nested allocations (lists/arrays), `alloc` is
+/// invoked any number of times with the size of the required allocation as a parameter
+/// (which is assumed to be correctly aligned for all payload types).
+#[async_recursion(?Send)]
+async unsafe fn recv_value<F>(
+    stream: &TcpStream,
+    tag: Tag<'async_recursion>,
+    data: &mut *mut (),
+    alloc: &(impl Fn(usize) -> F + 'async_recursion),
+) -> Result<(), smoltcp::Error>
+where
+    F: Future<Output = *mut ()>,
+{
+    macro_rules! consume_value {
+        ($ty:ty, | $ptr:ident | $map:expr) => {{
+            let $ptr = align_ptr_mut::<$ty>(*data);
+            *data = $ptr.offset(1) as *mut ();
+            $map
+        }};
+    }
+
+    match tag {
+        Tag::None => Ok(()),
+        Tag::Bool => consume_value!(i8, |ptr| {
+            *ptr = proto_async::read_i8(stream).await?;
+            Ok(())
+        }),
+        Tag::Int32 => consume_value!(i32, |ptr| {
+            *ptr = proto_async::read_i32(stream).await?;
+            Ok(())
+        }),
+        Tag::Int64 | Tag::Float64 => consume_value!(i64, |ptr| {
+            *ptr = proto_async::read_i64(stream).await?;
+            Ok(())
+        }),
+        Tag::String | Tag::Bytes | Tag::ByteArray => {
+            consume_value!(CMutSlice<u8>, |ptr| {
+                let length = proto_async::read_i32(stream).await? as usize;
+                *ptr = CMutSlice::new(alloc(length).await as *mut u8, length);
+                proto_async::read_chunk(stream, (*ptr).as_mut()).await?;
+                Ok(())
+            })
+        }
+        Tag::Tuple(it, arity) => {
+            let alignment = tag.alignment();
+            *data = round_up_mut(*data, alignment);
+            let mut it = it.clone();
+            for _ in 0..arity {
+                let tag = it.next().expect("truncated tag");
+                recv_value(stream, tag, data, alloc).await?
+            }
+            // Take into account any tail padding (if element(s) with largest alignment
+            // are not at the end).
+            *data = round_up_mut(*data, alignment);
+            Ok(())
+        }
+        Tag::List(it) => {
+            #[repr(C)]
+            struct List {
+                elements: *mut (),
+                length: usize,
+            }
+            consume_value!(*mut List, |ptr_to_list| {
+                let tag = it.clone().next().expect("truncated tag");
+                let length = proto_async::read_i32(stream).await? as usize;
+
+                // To avoid multiple kernel CPU roundtrips, use a single allocation for
+                // both the pointer/length List (slice) and the backing storage for the
+                // elements. We can assume that alloc() is aligned suitably, so just
+                // need to take into account any extra padding required.
+                // (Note: At the time of writing, there will never actually be any types
+                // with alignment larger than 8 bytes, so storage_offset == 0 always.)
+                let list_size = 4 + 4;
+                let storage_offset = round_up(list_size, tag.alignment());
+                let storage_size = tag.size() * length;
+
+                let allocation = alloc(storage_offset + storage_size).await as *mut u8;
+                *ptr_to_list = allocation as *mut List;
+                let storage = allocation.offset(storage_offset as isize) as *mut ();
+
+                (**ptr_to_list).length = length;
+                (**ptr_to_list).elements = storage;
+                recv_elements(stream, tag, length, storage, alloc).await
+            })
+        }
+        Tag::Array(it, num_dims) => {
+            consume_value!(*mut (), |buffer| {
+                // Deserialize length along each dimension and compute total number of
+                // elements.
+                let mut total_len: usize = 1;
+                for _ in 0..num_dims {
+                    let len = proto_async::read_i32(stream).await? as usize;
+                    total_len *= len;
+                    consume_value!(usize, |ptr| *ptr = len)
+                }
+
+                // Allocate backing storage for elements; deserialize them.
+                let elt_tag = it.clone().next().expect("truncated tag");
+                *buffer = alloc(elt_tag.size() * total_len).await;
+                recv_elements(stream, elt_tag, total_len, *buffer, alloc).await
+            })
+        }
+        Tag::Range(it) => {
+            *data = round_up_mut(*data, tag.alignment());
+            let tag = it.clone().next().expect("truncated tag");
+            recv_value(stream, tag, data, alloc).await?;
+            recv_value(stream, tag, data, alloc).await?;
+            recv_value(stream, tag, data, alloc).await?;
+            Ok(())
+        }
+        Tag::Keyword(_) => unreachable!(),
+        Tag::Object => unreachable!(),
+    }
+}
+
+pub async fn recv_return<F>(
+    stream: &TcpStream,
+    tag_bytes: &[u8],
+    data: *mut (),
+    alloc: &impl Fn(usize) -> F,
+) -> Result<(), smoltcp::Error>
+where
+    F: Future<Output = *mut ()>,
+{
+    let mut it = TagIterator::new(tag_bytes);
+    trace!("recv ...->{}", it);
+
+    let tag = it.next().expect("truncated tag");
+    let mut data = data;
+    unsafe { recv_value(stream, tag, &mut data, alloc).await? };
+
+    Ok(())
+}

src/runtime/src/rtio_clocking.rs

@@ -1,16 +1,17 @@
 use embedded_hal::blocking::delay::DelayMs;
+#[cfg(has_si5324)]
+use ksupport::i2c;
 use libboard_artiq::pl;
 #[cfg(has_si5324)]
 use libboard_artiq::si5324;
+#[cfg(has_si549)]
+use libboard_artiq::si549;
 #[cfg(has_si5324)]
 use libboard_zynq::i2c::I2c;
 use libboard_zynq::timer::GlobalTimer;
 use libconfig::Config;
 use log::{info, warn};
 
-#[cfg(has_si5324)]
-use crate::i2c;
-
 #[derive(Debug, PartialEq, Copy, Clone)]
 #[allow(non_camel_case_types)]
 pub enum RtioClock {
@@ -20,6 +21,7 @@ pub enum RtioClock {
     Int_150,
     Ext0_Bypass,
     Ext0_Synth0_10to125,
+    Ext0_Synth0_80to125,
     Ext0_Synth0_100to125,
     Ext0_Synth0_125to125,
 }
@@ -36,6 +38,7 @@ fn get_rtio_clock_cfg(cfg: &Config) -> RtioClock {
             "ext0_bypass_125" => RtioClock::Ext0_Bypass,
             "ext0_bypass_100" => RtioClock::Ext0_Bypass,
             "ext0_synth0_10to125" => RtioClock::Ext0_Synth0_10to125,
+            "ext0_synth0_80to125" => RtioClock::Ext0_Synth0_80to125,
             "ext0_synth0_100to125" => RtioClock::Ext0_Synth0_100to125,
             "ext0_synth0_125to125" => RtioClock::Ext0_Synth0_125to125,
             _ => {
@@ -74,7 +77,7 @@ fn init_rtio(timer: &mut GlobalTimer) {
     }
     // if it's not locked, it will hang at the CSR.
 
-    timer.delay_ms(20); // wait for CPLL/QPLL/SYS PLL lock
+    timer.delay_ms(50); // wait for CPLL/QPLL/SYS PLL lock
     let clk = unsafe { pl::csr::sys_crg::current_clock_read() };
     if clk == 1 {
         info!("SYS CLK switched successfully");
@@ -90,10 +93,10 @@ fn init_rtio(timer: &mut GlobalTimer) {
 #[cfg(has_drtio)]
 fn init_drtio(timer: &mut GlobalTimer) {
     unsafe {
-        pl::csr::drtio_transceiver::stable_clkin_write(1);
+        pl::csr::gt_drtio::stable_clkin_write(1);
     }
 
-    timer.delay_ms(20); // wait for CPLL/QPLL/SYS PLL lock
+    timer.delay_ms(50); // wait for CPLL/QPLL/SYS PLL lock
     let clk = unsafe { pl::csr::sys_crg::current_clock_read() };
     if clk == 1 {
         info!("SYS CLK switched successfully");
@@ -102,7 +105,10 @@ fn init_drtio(timer: &mut GlobalTimer) {
     }
     unsafe {
         pl::csr::rtio_core::reset_phy_write(1);
-        pl::csr::drtio_transceiver::txenable_write(0xffffffffu32 as _);
+        pl::csr::gt_drtio::txenable_write(0xffffffffu32 as _);
+
+        #[cfg(has_drtio_eem)]
+        pl::csr::eem_transceiver::txenable_write(0xffffffffu32 as _);
     }
 }
 
@@ -130,6 +136,23 @@ fn setup_si5324(i2c: &mut I2c, timer: &mut GlobalTimer, clk: RtioClock) {
                 SI5324_EXT_INPUT,
             )
         }
+        RtioClock::Ext0_Synth0_80to125 => {
+            // 125 MHz output from 80 MHz CLKINx reference, 611 Hz BW
+            info!("using 80MHz reference to make 125MHz RTIO clock with PLL");
+            (
+                si5324::FrequencySettings {
+                    n1_hs: 4,
+                    nc1_ls: 10,
+                    n2_hs: 10,
+                    n2_ls: 250,
+                    n31: 40,
+                    n32: 40,
+                    bwsel: 4,
+                    crystal_as_ckin2: false,
+                },
+                SI5324_EXT_INPUT,
+            )
+        }
         RtioClock::Ext0_Synth0_100to125 => {
             // 125MHz output, from 100MHz CLKINx reference, 586 Hz loop bandwidth
             info!("using 100MHz reference to make 125MHz RTIO clock with PLL");
@@ -239,20 +262,187 @@ fn setup_si5324(i2c: &mut I2c, timer: &mut GlobalTimer, clk: RtioClock) {
     si5324::setup(i2c, &si5324_settings, si5324_ref_input, timer).expect("cannot initialize Si5324");
 }
 
+#[cfg(all(has_si549, has_wrpll))]
+fn wrpll_setup(timer: &mut GlobalTimer, clk: RtioClock, si549_settings: &si549::FrequencySetting) {
+    // register values are directly copied from preconfigured mmcm
+    let (mmcm_setting, mmcm_bypass) = match clk {
+        RtioClock::Ext0_Synth0_10to125 => (
+            si549::wrpll_refclk::MmcmSetting {
+                // CLKFBOUT_MULT = 62.5, DIVCLK_DIVIDE = 1 , CLKOUT0_DIVIDE = 5
+                clkout0_reg1: 0x1083,
+                clkout0_reg2: 0x0080,
+                clkfbout_reg1: 0x179e,
+                clkfbout_reg2: 0x4c00,
+                div_reg: 0x1041,
+                lock_reg1: 0x00fa,
+                lock_reg2: 0x7c01,
+                lock_reg3: 0xffe9,
+                power_reg: 0x9900,
+                filt_reg1: 0x1008,
+                filt_reg2: 0x8800,
+            },
+            false,
+        ),
+        RtioClock::Ext0_Synth0_80to125 => (
+            si549::wrpll_refclk::MmcmSetting {
+                // CLKFBOUT_MULT = 15.625, DIVCLK_DIVIDE = 1 , CLKOUT0_DIVIDE = 10
+                clkout0_reg1: 0x1145,
+                clkout0_reg2: 0x0000,
+                clkfbout_reg1: 0x11c7,
+                clkfbout_reg2: 0x5880,
+                div_reg: 0x1041,
+                lock_reg1: 0x028a,
+                lock_reg2: 0x7c01,
+                lock_reg3: 0xffe9,
+                power_reg: 0x9900,
+                filt_reg1: 0x9908,
+                filt_reg2: 0x8100,
+            },
+            false,
+        ),
+        RtioClock::Ext0_Synth0_100to125 => (
+            si549::wrpll_refclk::MmcmSetting {
+                // CLKFBOUT_MULT = 12.5, DIVCLK_DIVIDE = 1 , CLKOUT0_DIVIDE = 10
+                clkout0_reg1: 0x1145,
+                clkout0_reg2: 0x0000,
+                clkfbout_reg1: 0x1145,
+                clkfbout_reg2: 0x4c00,
+                div_reg: 0x1041,
+                lock_reg1: 0x0339,
+                lock_reg2: 0x7c01,
+                lock_reg3: 0xffe9,
+                power_reg: 0x9900,
+                filt_reg1: 0x9108,
+                filt_reg2: 0x0100,
+            },
+            false,
+        ),
+        RtioClock::Ext0_Synth0_125to125 => (
+            si549::wrpll_refclk::MmcmSetting {
+                // CLKFBOUT_MULT = 10, DIVCLK_DIVIDE = 1 , CLKOUT0_DIVIDE = 10
+                clkout0_reg1: 0x1145,
+                clkout0_reg2: 0x0000,
+                clkfbout_reg1: 0x1145,
+                clkfbout_reg2: 0x0000,
+                div_reg: 0x1041,
+                lock_reg1: 0x03e8,
+                lock_reg2: 0x7001,
+                lock_reg3: 0xf3e9,
+                power_reg: 0x0100,
+                filt_reg1: 0x9908,
+                filt_reg2: 0x1100,
+            },
+            true,
+        ),
+        _ => unreachable!(),
+    };
+
+    si549::helper_setup(timer, &si549_settings).expect("cannot initialize helper Si549");
+    si549::wrpll_refclk::setup(timer, mmcm_setting, mmcm_bypass).expect("cannot initialize ref clk for wrpll");
+    si549::wrpll::select_recovered_clock(true, timer);
+}
+
+#[cfg(has_si549)]
+fn get_si549_setting(clk: RtioClock) -> si549::FrequencySetting {
+    match clk {
+        RtioClock::Ext0_Synth0_10to125 => {
+            info!("using 10MHz reference to make 125MHz RTIO clock with WRPLL");
+        }
+        RtioClock::Ext0_Synth0_80to125 => {
+            info!("using 80MHz reference to make 125MHz RTIO clock with WRPLL");
+        }
+        RtioClock::Ext0_Synth0_100to125 => {
+            info!("using 100MHz reference to make 125MHz RTIO clock with WRPLL");
+        }
+        RtioClock::Ext0_Synth0_125to125 => {
+            info!("using 125MHz reference to make 125MHz RTIO clock with WRPLL");
+        }
+        RtioClock::Int_100 => {
+            info!("using internal 100MHz RTIO clock");
+        }
+        RtioClock::Int_125 => {
+            info!("using internal 125MHz RTIO clock");
+        }
+        _ => {
+            warn!(
+                "rtio_clock setting '{:?}' is unsupported. Falling back to default internal 125MHz RTIO clock.",
+                clk
+            );
+        }
+    };
+
+    match clk {
+        RtioClock::Int_100 => {
+            si549::FrequencySetting {
+                main: si549::DividerConfig {
+                    hsdiv: 0x06C,
+                    lsdiv: 0,
+                    fbdiv: 0x046C5F49797,
+                },
+                helper: si549::DividerConfig {
+                    // 100MHz*32767/32768
+                    hsdiv: 0x06C,
+                    lsdiv: 0,
+                    fbdiv: 0x046C5670BBD,
+                },
+            }
+        }
+        _ => {
+            // Everything else use 125MHz
+            si549::FrequencySetting {
+                main: si549::DividerConfig {
+                    hsdiv: 0x058,
+                    lsdiv: 0,
+                    fbdiv: 0x04815791F25,
+                },
+                helper: si549::DividerConfig {
+                    // 125MHz*32767/32768
+                    hsdiv: 0x058,
+                    lsdiv: 0,
+                    fbdiv: 0x04814E8F442,
+                },
+            }
+        }
+    }
+}
+
 pub fn init(timer: &mut GlobalTimer, cfg: &Config) {
     let clk = get_rtio_clock_cfg(cfg);
     #[cfg(has_si5324)]
     {
         let i2c = unsafe { (&mut i2c::I2C_BUS).as_mut().unwrap() };
         match clk {
-            RtioClock::Ext0_Bypass => si5324::bypass(i2c, SI5324_EXT_INPUT, timer).expect("cannot bypass Si5324"),
+            RtioClock::Ext0_Bypass => {
+                info!("bypassing the PLL for RTIO clock");
+                si5324::bypass(i2c, SI5324_EXT_INPUT, timer).expect("cannot bypass Si5324")
+            }
             _ => setup_si5324(i2c, timer, clk),
         }
     }
 
+    #[cfg(has_si549)]
+    let si549_settings = get_si549_setting(clk);
+
+    #[cfg(has_si549)]
+    si549::main_setup(timer, &si549_settings).expect("cannot initialize main Si549");
+
     #[cfg(has_drtio)]
     init_drtio(timer);
 
     #[cfg(not(has_drtio))]
     init_rtio(timer);
+
+    #[cfg(all(has_si549, has_wrpll))]
+    {
+        // SYS CLK switch will reset CSRs that are used by WRPLL
+        match clk {
+            RtioClock::Ext0_Synth0_10to125
+            | RtioClock::Ext0_Synth0_80to125
+            | RtioClock::Ext0_Synth0_100to125
+            | RtioClock::Ext0_Synth0_125to125 => {
+                wrpll_setup(timer, clk, &si549_settings);
+            }
+            _ => {}
+        }
+    }
 }

src/runtime/src/rtio_dma.rs

@@ -2,14 +2,13 @@ use alloc::{collections::BTreeMap, rc::Rc, string::String, vec::Vec};
 #[cfg(has_drtio)]
 use core::mem;
 
+use ksupport::kernel::DmaRecorder;
 #[cfg(has_drtio)]
 use libasync::task;
 use libboard_artiq::drtio_routing::RoutingTable;
 use libboard_zynq::timer::GlobalTimer;
 use libcortex_a9::{cache::dcci_slice, mutex::Mutex};
 
-use crate::kernel::DmaRecorder;
-
 const ALIGNMENT: usize = 16 * 8;
 
 static DMA_RECORD_STORE: Mutex<BTreeMap<String, (u32, Vec<u8>, i64)>> = Mutex::new(BTreeMap::new());
@@ -143,9 +142,9 @@ pub mod remote_dma {
             }
         }
 
-        pub async fn playback_done(&mut self, destination: u8, error: u8, channel: u32, timestamp: u64) {
+        pub async fn playback_done(&mut self, source: u8, error: u8, channel: u32, timestamp: u64) {
             let mut traces_locked = self.traces.async_lock().await;
-            let mut trace = traces_locked.get_mut(&destination).unwrap();
+            let mut trace = traces_locked.get_mut(&source).unwrap();
             trace.state = RemoteState::PlaybackEnded {
                 error: error,
                 channel: channel,
@@ -193,7 +192,7 @@ pub mod remote_dma {
             up: bool,
         ) {
             // update state of the destination, resend traces if it's up
-            if let Some(trace) = self.traces.lock().get_mut(&destination) {
+            if let Some(trace) = self.traces.async_lock().await.get_mut(&destination) {
                 if up {
                     match drtio::ddma_upload_trace(
                         aux_mutex,
@@ -213,6 +212,10 @@ pub mod remote_dma {
                 }
             }
         }
+
+        pub async fn is_empty(&self) -> bool {
+            self.traces.async_lock().await.is_empty()
+        }
     }
 
     static mut TRACES: BTreeMap<u32, TraceSet> = BTreeMap::new();
@@ -269,6 +272,11 @@ pub mod remote_dma {
                 .await;
         }
     }
+
+    pub async fn has_remote_traces(id: u32) -> bool {
+        let trace_set = unsafe { TRACES.get_mut(&id).unwrap() };
+        !(trace_set.is_empty().await)
+    }
 }
 
 pub async fn put_record(
@@ -343,7 +351,11 @@ pub async fn erase(name: String, _aux_mutex: &Rc<Mutex<bool>>, _routing_table: &
     }
 }
 
-pub fn retrieve(name: String) -> Option<(i32, i64)> {
+pub async fn retrieve(name: String) -> Option<(i32, i64, bool)> {
     let (ptr, _v, duration) = DMA_RECORD_STORE.lock().get(&name)?.clone();
-    Some((ptr as i32, duration))
+    #[cfg(has_drtio)]
+    let uses_ddma = remote_dma::has_remote_traces(ptr).await;
+    #[cfg(not(has_drtio))]
+    let uses_ddma = false;
+    Some((ptr as i32, duration, uses_ddma))
 }

src/runtime/src/rtio_mgt.rs

@@ -1,32 +1,67 @@
-use alloc::{collections::BTreeMap, rc::Rc, string::String};
+use alloc::rc::Rc;
 use core::cell::RefCell;
 
-use io::{Cursor, ProtoRead};
-use libboard_artiq::{drtio_routing, pl::csr};
+use libboard_artiq::{drtio_routing, drtio_routing::RoutingTable, pl::csr};
 use libboard_zynq::timer::GlobalTimer;
-use libconfig::Config;
 use libcortex_a9::mutex::Mutex;
-use log::warn;
-
-static mut RTIO_DEVICE_MAP: BTreeMap<u32, String> = BTreeMap::new();
 
 #[cfg(has_drtio)]
 pub mod drtio {
     use alloc::vec::Vec;
+    use core::fmt;
 
     use embedded_hal::blocking::delay::DelayMs;
+    use ksupport::{resolve_channel_name, ASYNC_ERROR_BUSY, ASYNC_ERROR_COLLISION, ASYNC_ERROR_SEQUENCE_ERROR,
+                   SEEN_ASYNC_ERRORS};
     use libasync::{delay, task};
-    use libboard_artiq::{drtioaux::Error, drtioaux_async, drtioaux_async::Packet, drtioaux_proto::DMA_TRACE_MAX_SIZE};
+    use libboard_artiq::{drtioaux::Error as DrtioError,
+                         drtioaux_async,
+                         drtioaux_async::Packet,
+                         drtioaux_proto::{PayloadStatus, MASTER_PAYLOAD_MAX_SIZE}};
     use libboard_zynq::time::Milliseconds;
     use log::{error, info, warn};
 
     use super::*;
-    use crate::{rtio_dma::remote_dma, ASYNC_ERROR_BUSY, ASYNC_ERROR_COLLISION, ASYNC_ERROR_SEQUENCE_ERROR,
-                SEEN_ASYNC_ERRORS};
+    use crate::{analyzer::remote_analyzer::RemoteBuffer, rtio_dma::remote_dma, subkernel};
+
+    #[derive(Debug, PartialEq, Eq, Clone, Copy)]
+    pub enum Error {
+        Timeout,
+        AuxError,
+        LinkDown,
+        UnexpectedReply,
+        DmaAddTraceFail(u8),
+        DmaEraseFail(u8),
+        DmaPlaybackFail(u8),
+        SubkernelAddFail(u8),
+        SubkernelRunFail(u8),
+    }
+
+    impl fmt::Display for Error {
+        fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+            match self {
+                Error::Timeout => write!(f, "timed out"),
+                Error::AuxError => write!(f, "aux packet error"),
+                Error::LinkDown => write!(f, "link down"),
+                Error::UnexpectedReply => write!(f, "unexpected reply"),
+                Error::DmaAddTraceFail(dest) => write!(f, "error adding DMA trace on satellite #{}", dest),
+                Error::DmaEraseFail(dest) => write!(f, "error erasing DMA trace on satellite #{}", dest),
+                Error::DmaPlaybackFail(dest) => write!(f, "error playing back DMA trace on satellite #{}", dest),
+                Error::SubkernelAddFail(dest) => write!(f, "error adding subkernel on satellite #{}", dest),
+                Error::SubkernelRunFail(dest) => write!(f, "error on subkernel run request on satellite #{}", dest),
+            }
+        }
+    }
+
+    impl From<DrtioError> for Error {
+        fn from(_error: DrtioError) -> Self {
+            Error::AuxError
+        }
+    }
 
     pub fn startup(
         aux_mutex: &Rc<Mutex<bool>>,
-        routing_table: &Rc<RefCell<drtio_routing::RoutingTable>>,
+        routing_table: &Rc<RefCell<RoutingTable>>,
         up_destinations: &Rc<RefCell<[bool; drtio_routing::DEST_COUNT]>>,
         timer: GlobalTimer,
     ) {
@@ -44,42 +79,102 @@ pub mod drtio {
         unsafe { (csr::DRTIO[linkno].rx_up_read)() == 1 }
     }
 
-    async fn recv_aux_timeout(linkno: u8, timeout: u64, timer: GlobalTimer) -> Result<Packet, &'static str> {
+    async fn process_async_packets(linkno: u8, routing_table: &RoutingTable, packet: Packet) -> Option<Packet> {
+        match packet {
+            Packet::DmaPlaybackStatus {
+                id,
+                source,
+                destination: 0,
+                error,
+                channel,
+                timestamp,
+            } => {
+                remote_dma::playback_done(id, source, error, channel, timestamp).await;
+                None
+            }
+            Packet::SubkernelFinished {
+                id,
+                destination: 0,
+                with_exception,
+                exception_src,
+            } => {
+                subkernel::subkernel_finished(id, with_exception, exception_src).await;
+                None
+            }
+            Packet::SubkernelMessage {
+                id,
+                source,
+                destination: 0,
+                status,
+                length,
+                data,
+            } => {
+                subkernel::message_handle_incoming(id, status, length as usize, &data).await;
+                // acknowledge receiving part of the message
+                drtioaux_async::send(linkno, &Packet::SubkernelMessageAck { destination: source })
+                    .await
+                    .unwrap();
+                None
+            }
+            // routable packets
+            Packet::DmaAddTraceRequest { destination, .. }
+            | Packet::DmaAddTraceReply { destination, .. }
+            | Packet::DmaRemoveTraceRequest { destination, .. }
+            | Packet::DmaRemoveTraceReply { destination, .. }
+            | Packet::DmaPlaybackRequest { destination, .. }
+            | Packet::DmaPlaybackReply { destination, .. }
+            | Packet::SubkernelLoadRunRequest { destination, .. }
+            | Packet::SubkernelLoadRunReply { destination, .. }
+            | Packet::SubkernelMessage { destination, .. }
+            | Packet::SubkernelMessageAck { destination, .. }
+            | Packet::DmaPlaybackStatus { destination, .. }
+            | Packet::SubkernelFinished { destination, .. } => {
+                if destination == 0 {
+                    Some(packet)
+                } else {
+                    let dest_link = routing_table.0[destination as usize][0] - 1;
+                    if dest_link == linkno {
+                        warn!(
+                            "[LINK#{}] Re-routed packet would return to the same link, dropping: {:?}",
+                            linkno, packet
+                        );
+                    } else {
+                        drtioaux_async::send(dest_link, &packet).await.unwrap();
+                    }
+                    None
+                }
+            }
+            other => Some(other),
+        }
+    }
+
+    async fn recv_aux_timeout(linkno: u8, timeout: u64, timer: GlobalTimer) -> Result<Packet, Error> {
         if !link_rx_up(linkno).await {
-            return Err("link went down");
+            return Err(Error::LinkDown);
         }
         match drtioaux_async::recv_timeout(linkno, Some(timeout), timer).await {
             Ok(packet) => return Ok(packet),
-            Err(Error::TimedOut) => return Err("timed out"),
-            Err(_) => return Err("aux packet error"),
+            Err(DrtioError::TimedOut) => return Err(Error::Timeout),
+            Err(_) => return Err(Error::AuxError),
         }
     }
 
     pub async fn aux_transact(
         aux_mutex: &Mutex<bool>,
         linkno: u8,
+        routing_table: &RoutingTable,
         request: &Packet,
         timer: GlobalTimer,
-    ) -> Result<Packet, &'static str> {
+    ) -> Result<Packet, Error> {
         if !link_rx_up(linkno).await {
-            return Err("link went down");
+            return Err(Error::LinkDown);
         }
         let _lock = aux_mutex.async_lock().await;
         drtioaux_async::send(linkno, request).await.unwrap();
         loop {
-            let reply = recv_aux_timeout(linkno, 200, timer).await;
-            match reply {
-                Ok(Packet::DmaPlaybackStatus {
-                    id,
-                    destination,
-                    error,
-                    channel,
-                    timestamp,
-                }) => {
-                    remote_dma::playback_done(id, destination, error, channel, timestamp).await;
-                }
-                Ok(packet) => return Ok(packet),
-                Err(e) => return Err(e),
+            let packet = recv_aux_timeout(linkno, 200, timer).await?;
+            if let Some(packet) = process_async_packets(linkno, routing_table, packet).await {
+                return Ok(packet);
             }
         }
     }
@@ -89,12 +184,17 @@ pub mod drtio {
         loop {
             if timer.get_time() > max_time {
                 return;
-            } //could this be cut short?
+            }
             let _ = drtioaux_async::recv(linkno).await;
         }
     }
 
-    async fn ping_remote(aux_mutex: &Rc<Mutex<bool>>, linkno: u8, timer: GlobalTimer) -> u32 {
+    async fn ping_remote(
+        aux_mutex: &Rc<Mutex<bool>>,
+        linkno: u8,
+        routing_table: &RoutingTable,
+        timer: GlobalTimer,
+    ) -> u32 {
         let mut count = 0;
         loop {
             if !link_rx_up(linkno).await {
@@ -104,7 +204,7 @@ pub mod drtio {
             if count > 100 {
                 return 0;
             }
-            let reply = aux_transact(aux_mutex, linkno, &Packet::EchoRequest, timer).await;
+            let reply = aux_transact(aux_mutex, linkno, routing_table, &Packet::EchoRequest, timer).await;
             match reply {
                 Ok(Packet::EchoReply) => {
                     // make sure receive buffer is drained
@@ -117,7 +217,7 @@ pub mod drtio {
         }
     }
 
-    async fn sync_tsc(aux_mutex: &Rc<Mutex<bool>>, linkno: u8, timer: GlobalTimer) -> Result<(), &'static str> {
+    async fn sync_tsc(aux_mutex: &Rc<Mutex<bool>>, linkno: u8, timer: GlobalTimer) -> Result<(), Error> {
         let _lock = aux_mutex.async_lock().await;
 
         unsafe {
@@ -128,22 +228,23 @@ pub mod drtio {
         // by the satellite, in response to a TSC set on the RT link.
         let reply = recv_aux_timeout(linkno, 10000, timer).await?;
         if reply == Packet::TSCAck {
-            return Ok(());
+            Ok(())
         } else {
-            return Err("unexpected reply");
+            Err(Error::UnexpectedReply)
         }
     }
 
     async fn load_routing_table(
         aux_mutex: &Rc<Mutex<bool>>,
         linkno: u8,
-        routing_table: &drtio_routing::RoutingTable,
+        routing_table: &RoutingTable,
         timer: GlobalTimer,
-    ) -> Result<(), &'static str> {
+    ) -> Result<(), Error> {
         for i in 0..drtio_routing::DEST_COUNT {
             let reply = aux_transact(
                 aux_mutex,
                 linkno,
+                routing_table,
                 &Packet::RoutingSetPath {
                     destination: i as u8,
                     hops: routing_table.0[i],
@@ -152,7 +253,7 @@ pub mod drtio {
             )
             .await?;
             if reply != Packet::RoutingAck {
-                return Err("unexpected reply");
+                return Err(Error::UnexpectedReply);
             }
         }
         Ok(())
@@ -162,13 +263,21 @@ pub mod drtio {
         aux_mutex: &Rc<Mutex<bool>>,
         linkno: u8,
         rank: u8,
+        routing_table: &RoutingTable,
         timer: GlobalTimer,
-    ) -> Result<(), &'static str> {
-        let reply = aux_transact(aux_mutex, linkno, &Packet::RoutingSetRank { rank: rank }, timer).await?;
-        if reply != Packet::RoutingAck {
-            return Err("unexpected reply");
+    ) -> Result<(), Error> {
+        let reply = aux_transact(
+            aux_mutex,
+            linkno,
+            routing_table,
+            &Packet::RoutingSetRank { rank: rank },
+            timer,
+        )
+        .await?;
+        match reply {
+            Packet::RoutingAck => Ok(()),
+            _ => Err(Error::UnexpectedReply),
         }
-        Ok(())
     }
 
     async fn init_buffer_space(destination: u8, linkno: u8) {
@@ -187,17 +296,14 @@ pub mod drtio {
         }
     }
 
-    async fn process_unsolicited_aux(aux_mutex: &Rc<Mutex<bool>>, linkno: u8) {
+    async fn process_unsolicited_aux(aux_mutex: &Mutex<bool>, linkno: u8, routing_table: &RoutingTable) {
         let _lock = aux_mutex.async_lock().await;
         match drtioaux_async::recv(linkno).await {
-            Ok(Some(Packet::DmaPlaybackStatus {
-                id,
-                destination,
-                error,
-                channel,
-                timestamp,
-            })) => remote_dma::playback_done(id, destination, error, channel, timestamp).await,
-            Ok(Some(packet)) => warn!("[LINK#{}] unsolicited aux packet: {:?}", linkno, packet),
+            Ok(Some(packet)) => {
+                if let Some(packet) = process_async_packets(linkno, routing_table, packet).await {
+                    warn!("[LINK#{}] unsolicited aux packet: {:?}", linkno, packet);
+                }
+            }
             Ok(None) => (),
             Err(_) => warn!("[LINK#{}] aux packet error", linkno),
         }
@@ -225,7 +331,7 @@ pub mod drtio {
     }
 
     async fn destination_set_up(
-        routing_table: &drtio_routing::RoutingTable,
+        routing_table: &RoutingTable,
         up_destinations: &Rc<RefCell<[bool; drtio_routing::DEST_COUNT]>>,
         destination: u8,
         up: bool,
@@ -248,7 +354,7 @@ pub mod drtio {
 
     async fn destination_survey(
         aux_mutex: &Rc<Mutex<bool>>,
-        routing_table: &drtio_routing::RoutingTable,
+        routing_table: &RoutingTable,
         up_links: &[bool],
         up_destinations: &Rc<RefCell<[bool; drtio_routing::DEST_COUNT]>>,
         timer: GlobalTimer,
@@ -269,6 +375,7 @@ pub mod drtio {
                         let reply = aux_transact(
                             aux_mutex,
                             linkno,
+                            routing_table,
                             &Packet::DestinationStatusRequest {
                                 destination: destination,
                             },
@@ -280,6 +387,8 @@ pub mod drtio {
                                 destination_set_up(routing_table, up_destinations, destination, false).await;
                                 remote_dma::destination_changed(aux_mutex, routing_table, timer, destination, false)
                                     .await;
+                                subkernel::destination_changed(aux_mutex, routing_table, timer, destination, false)
+                                    .await;
                             }
                             Ok(Packet::DestinationOkReply) => (),
                             Ok(Packet::DestinationSequenceErrorReply { channel }) => {
@@ -315,12 +424,14 @@ pub mod drtio {
                     } else {
                         destination_set_up(routing_table, up_destinations, destination, false).await;
                         remote_dma::destination_changed(aux_mutex, routing_table, timer, destination, false).await;
+                        subkernel::destination_changed(aux_mutex, routing_table, timer, destination, false).await;
                     }
                 } else {
                     if up_links[linkno as usize] {
                         let reply = aux_transact(
                             aux_mutex,
                             linkno,
+                            routing_table,
                             &Packet::DestinationStatusRequest {
                                 destination: destination,
                             },
@@ -334,6 +445,8 @@ pub mod drtio {
                                 init_buffer_space(destination as u8, linkno).await;
                                 remote_dma::destination_changed(aux_mutex, routing_table, timer, destination, true)
                                     .await;
+                                subkernel::destination_changed(aux_mutex, routing_table, timer, destination, true)
+                                    .await;
                             }
                             Ok(packet) => error!("[DEST#{}] received unexpected aux packet: {:?}", destination, packet),
                             Err(e) => error!("[DEST#{}] communication failed ({})", destination, e),
@@ -346,7 +459,7 @@ pub mod drtio {
 
     pub async fn link_task(
         aux_mutex: &Rc<Mutex<bool>>,
-        routing_table: &drtio_routing::RoutingTable,
+        routing_table: &RoutingTable,
         up_destinations: &Rc<RefCell<[bool; drtio_routing::DEST_COUNT]>>,
         timer: GlobalTimer,
     ) {
@@ -357,7 +470,7 @@ pub mod drtio {
                 if up_links[linkno as usize] {
                     /* link was previously up */
                     if link_rx_up(linkno).await {
-                        process_unsolicited_aux(aux_mutex, linkno).await;
+                        process_unsolicited_aux(aux_mutex, linkno, routing_table).await;
                         process_local_errors(linkno).await;
                     } else {
                         info!("[LINK#{}] link is down", linkno);
@@ -367,7 +480,7 @@ pub mod drtio {
                     /* link was previously down */
                     if link_rx_up(linkno).await {
                         info!("[LINK#{}] link RX became up, pinging", linkno);
-                        let ping_count = ping_remote(aux_mutex, linkno, timer).await;
+                        let ping_count = ping_remote(aux_mutex, linkno, routing_table, timer).await;
                         if ping_count > 0 {
                             info!("[LINK#{}] remote replied after {} packets", linkno, ping_count);
                             up_links[linkno as usize] = true;
@@ -377,7 +490,7 @@ pub mod drtio {
                             if let Err(e) = load_routing_table(aux_mutex, linkno, routing_table, timer).await {
                                 error!("[LINK#{}] failed to load routing table ({})", linkno, e);
                             }
-                            if let Err(e) = set_rank(aux_mutex, linkno, 1 as u8, timer).await {
+                            if let Err(e) = set_rank(aux_mutex, linkno, 1 as u8, routing_table, timer).await {
                                 error!("[LINK#{}] failed to set rank ({})", linkno, e);
                             }
                             info!("[LINK#{}] link initialization completed", linkno);
@@ -394,7 +507,7 @@ pub mod drtio {
     }
 
     #[allow(dead_code)]
-    pub fn reset(aux_mutex: Rc<Mutex<bool>>, mut timer: GlobalTimer) {
+    pub fn reset(aux_mutex: Rc<Mutex<bool>>, routing_table: &RoutingTable, mut timer: GlobalTimer) {
         for linkno in 0..csr::DRTIO.len() {
             unsafe {
                 (csr::DRTIO[linkno].reset_write)(1);
@@ -410,7 +523,13 @@ pub mod drtio {
         for linkno in 0..csr::DRTIO.len() {
             let linkno = linkno as u8;
             if task::block_on(link_rx_up(linkno)) {
-                let reply = task::block_on(aux_transact(&aux_mutex, linkno, &Packet::ResetRequest, timer));
+                let reply = task::block_on(aux_transact(
+                    &aux_mutex,
+                    linkno,
+                    routing_table,
+                    &Packet::ResetRequest,
+                    timer,
+                ));
                 match reply {
                     Ok(Packet::ResetAck) => (),
                     Ok(_) => error!("[LINK#{}] reset failed, received unexpected aux packet", linkno),
@@ -420,148 +539,347 @@ pub mod drtio {
         }
     }
 
-    pub async fn ddma_upload_trace(
+    async fn partition_data<PacketF, HandlerF>(
+        linkno: u8,
         aux_mutex: &Rc<Mutex<bool>>,
-        routing_table: &drtio_routing::RoutingTable,
+        routing_table: &RoutingTable,
         timer: GlobalTimer,
-        id: u32,
-        destination: u8,
-        trace: &Vec<u8>,
-    ) -> Result<(), &'static str> {
-        let linkno = routing_table.0[destination as usize][0] - 1;
+        data: &[u8],
+        packet_f: PacketF,
+        reply_handler_f: HandlerF,
+    ) -> Result<(), Error>
+    where
+        PacketF: Fn(&[u8; MASTER_PAYLOAD_MAX_SIZE], PayloadStatus, usize) -> Packet,
+        HandlerF: Fn(&Packet) -> Result<(), Error>,
+    {
         let mut i = 0;
-        while i < trace.len() {
-            let mut trace_slice: [u8; DMA_TRACE_MAX_SIZE] = [0; DMA_TRACE_MAX_SIZE];
-            let len: usize = if i + DMA_TRACE_MAX_SIZE < trace.len() {
-                DMA_TRACE_MAX_SIZE
+        while i < data.len() {
+            let mut slice: [u8; MASTER_PAYLOAD_MAX_SIZE] = [0; MASTER_PAYLOAD_MAX_SIZE];
+            let len: usize = if i + MASTER_PAYLOAD_MAX_SIZE < data.len() {
+                MASTER_PAYLOAD_MAX_SIZE
             } else {
-                trace.len() - i
+                data.len() - i
             } as usize;
-            let last = i + len == trace.len();
-            trace_slice[..len].clone_from_slice(&trace[i..i + len]);
+            let first = i == 0;
+            let last = i + len == data.len();
+            slice[..len].clone_from_slice(&data[i..i + len]);
             i += len;
-            let reply = aux_transact(
-                aux_mutex,
-                linkno,
-                &Packet::DmaAddTraceRequest {
-                    id: id,
-                    destination: destination,
-                    last: last,
-                    length: len as u16,
-                    trace: trace_slice,
-                },
-                timer,
-            )
-            .await;
-            match reply {
-                Ok(Packet::DmaAddTraceReply { succeeded: true }) => (),
-                Ok(Packet::DmaAddTraceReply { succeeded: false }) => {
-                    return Err("error adding trace on satellite");
-                }
-                Ok(_) => {
-                    return Err("adding DMA trace failed, unexpected aux packet");
-                }
-                Err(_) => {
-                    return Err("adding DMA trace failed, aux error");
-                }
-            }
+            let status = PayloadStatus::from_status(first, last);
+            let packet = packet_f(&slice, status, len);
+            let reply = aux_transact(aux_mutex, linkno, routing_table, &packet, timer).await?;
+            reply_handler_f(&reply)?;
         }
         Ok(())
     }
 
-    pub async fn ddma_send_erase(
+    pub async fn ddma_upload_trace(
         aux_mutex: &Rc<Mutex<bool>>,
-        routing_table: &drtio_routing::RoutingTable,
+        routing_table: &RoutingTable,
         timer: GlobalTimer,
         id: u32,
         destination: u8,
-    ) -> Result<(), &'static str> {
+        trace: &Vec<u8>,
+    ) -> Result<(), Error> {
+        let linkno = routing_table.0[destination as usize][0] - 1;
+        partition_data(
+            linkno,
+            aux_mutex,
+            routing_table,
+            timer,
+            trace,
+            |slice, status, len| Packet::DmaAddTraceRequest {
+                id: id,
+                source: 0,
+                destination: destination,
+                status: status,
+                length: len as u16,
+                trace: *slice,
+            },
+            |reply| match reply {
+                Packet::DmaAddTraceReply {
+                    destination: 0,
+                    succeeded: true,
+                    ..
+                } => Ok(()),
+                Packet::DmaAddTraceReply {
+                    destination: 0,
+                    succeeded: false,
+                    ..
+                } => Err(Error::DmaAddTraceFail(destination)),
+                _ => Err(Error::UnexpectedReply),
+            },
+        )
+        .await
+    }
+
+    pub async fn ddma_send_erase(
+        aux_mutex: &Rc<Mutex<bool>>,
+        routing_table: &RoutingTable,
+        timer: GlobalTimer,
+        id: u32,
+        destination: u8,
+    ) -> Result<(), Error> {
         let linkno = routing_table.0[destination as usize][0] - 1;
         let reply = aux_transact(
             aux_mutex,
             linkno,
+            routing_table,
             &Packet::DmaRemoveTraceRequest {
                 id: id,
+                source: 0,
                 destination: destination,
             },
             timer,
         )
-        .await;
+        .await?;
         match reply {
-            Ok(Packet::DmaRemoveTraceReply { succeeded: true }) => Ok(()),
-            Ok(Packet::DmaRemoveTraceReply { succeeded: false }) => Err("satellite DMA erase error"),
-            Ok(_) => Err("adding trace failed, unexpected aux packet"),
-            Err(_) => Err("erasing trace failed, aux error"),
+            Packet::DmaRemoveTraceReply {
+                destination: 0,
+                succeeded: true,
+            } => Ok(()),
+            Packet::DmaRemoveTraceReply {
+                destination: 0,
+                succeeded: false,
+            } => Err(Error::DmaEraseFail(destination)),
+            _ => Err(Error::UnexpectedReply),
         }
     }
 
     pub async fn ddma_send_playback(
         aux_mutex: &Rc<Mutex<bool>>,
-        routing_table: &drtio_routing::RoutingTable,
+        routing_table: &RoutingTable,
         timer: GlobalTimer,
         id: u32,
         destination: u8,
         timestamp: u64,
-    ) -> Result<(), &'static str> {
+    ) -> Result<(), Error> {
         let linkno = routing_table.0[destination as usize][0] - 1;
         let reply = aux_transact(
             aux_mutex,
             linkno,
+            routing_table,
             &Packet::DmaPlaybackRequest {
                 id: id,
+                source: 0,
                 destination: destination,
                 timestamp: timestamp,
             },
             timer,
         )
-        .await;
+        .await?;
         match reply {
-            Ok(Packet::DmaPlaybackReply { succeeded: true }) => Ok(()),
-            Ok(Packet::DmaPlaybackReply { succeeded: false }) => Err("error on DMA playback request"),
-            Ok(_) => Err("received unexpected aux packet during DMA playback"),
-            Err(_) => Err("aux error on DMA playback"),
+            Packet::DmaPlaybackReply {
+                destination: 0,
+                succeeded: true,
+            } => Ok(()),
+            Packet::DmaPlaybackReply {
+                destination: 0,
+                succeeded: false,
+            } => Err(Error::DmaPlaybackFail(destination)),
+            _ => Err(Error::UnexpectedReply),
         }
     }
-}
 
-fn read_device_map(cfg: &Config) -> BTreeMap<u32, String> {
-    let mut device_map: BTreeMap<u32, String> = BTreeMap::new();
-    let _ = cfg
-        .read("device_map")
-        .and_then(|raw_bytes| {
-            let mut bytes_cr = Cursor::new(raw_bytes);
-            let size = bytes_cr.read_u32().unwrap();
-            for _ in 0..size {
-                let channel = bytes_cr.read_u32().unwrap();
-                let device_name = bytes_cr.read_string().unwrap();
-                if let Some(old_entry) = device_map.insert(channel, device_name.clone()) {
-                    warn!(
-                        "conflicting device map entries for RTIO channel {}: '{}' and '{}'",
-                        channel, old_entry, device_name
-                    );
+    async fn analyzer_get_data(
+        aux_mutex: &Rc<Mutex<bool>>,
+        routing_table: &RoutingTable,
+        timer: GlobalTimer,
+        destination: u8,
+    ) -> Result<RemoteBuffer, Error> {
+        let linkno = routing_table.0[destination as usize][0] - 1;
+        let reply = aux_transact(
+            aux_mutex,
+            linkno,
+            routing_table,
+            &Packet::AnalyzerHeaderRequest {
+                destination: destination,
+            },
+            timer,
+        )
+        .await?;
+        let (sent, total, overflow) = match reply {
+            Packet::AnalyzerHeader {
+                sent_bytes,
+                total_byte_count,
+                overflow_occurred,
+            } => (sent_bytes, total_byte_count, overflow_occurred),
+            _ => return Err(Error::UnexpectedReply),
+        };
+
+        let mut remote_data: Vec<u8> = Vec::new();
+        if sent > 0 {
+            let mut last_packet = false;
+            while !last_packet {
+                let reply = aux_transact(
+                    aux_mutex,
+                    linkno,
+                    routing_table,
+                    &Packet::AnalyzerDataRequest {
+                        destination: destination,
+                    },
+                    timer,
+                )
+                .await?;
+                match reply {
+                    Packet::AnalyzerData { last, length, data } => {
+                        last_packet = last;
+                        remote_data.extend(&data[0..length as usize]);
+                    }
+                    _ => return Err(Error::UnexpectedReply),
                 }
             }
-            Ok(())
+        }
+
+        Ok(RemoteBuffer {
+            sent_bytes: sent,
+            total_byte_count: total,
+            error: overflow,
+            data: remote_data,
         })
-        .or_else(|err| {
-            warn!(
-                "error reading device map ({}), device names will not be available in RTIO error messages",
-                err
-            );
-            Err(err)
-        });
-    device_map
-}
-
-fn _resolve_channel_name(channel: u32, device_map: &BTreeMap<u32, String>) -> String {
-    match device_map.get(&channel) {
-        Some(val) => val.clone(),
-        None => String::from("unknown"),
     }
-}
 
-pub fn resolve_channel_name(channel: u32) -> String {
-    _resolve_channel_name(channel, unsafe { &RTIO_DEVICE_MAP })
+    pub async fn analyzer_query(
+        aux_mutex: &Rc<Mutex<bool>>,
+        routing_table: &RoutingTable,
+        up_destinations: &Rc<RefCell<[bool; drtio_routing::DEST_COUNT]>>,
+        timer: GlobalTimer,
+    ) -> Result<Vec<RemoteBuffer>, Error> {
+        let mut remote_buffers: Vec<RemoteBuffer> = Vec::new();
+        for i in 1..drtio_routing::DEST_COUNT {
+            if destination_up(up_destinations, i as u8).await {
+                remote_buffers.push(analyzer_get_data(aux_mutex, routing_table, timer, i as u8).await?);
+            }
+        }
+        Ok(remote_buffers)
+    }
+
+    pub async fn subkernel_upload(
+        aux_mutex: &Rc<Mutex<bool>>,
+        routing_table: &RoutingTable,
+        timer: GlobalTimer,
+        id: u32,
+        destination: u8,
+        data: &Vec<u8>,
+    ) -> Result<(), Error> {
+        let linkno = routing_table.0[destination as usize][0] - 1;
+        partition_data(
+            linkno,
+            aux_mutex,
+            routing_table,
+            timer,
+            data,
+            |slice, status, len| Packet::SubkernelAddDataRequest {
+                id: id,
+                destination: destination,
+                status: status,
+                length: len as u16,
+                data: *slice,
+            },
+            |reply| match reply {
+                Packet::SubkernelAddDataReply { succeeded: true } => Ok(()),
+                Packet::SubkernelAddDataReply { succeeded: false } => Err(Error::SubkernelAddFail(destination)),
+                _ => Err(Error::UnexpectedReply),
+            },
+        )
+        .await
+    }
+
+    pub async fn subkernel_load(
+        aux_mutex: &Rc<Mutex<bool>>,
+        routing_table: &RoutingTable,
+        timer: GlobalTimer,
+        id: u32,
+        destination: u8,
+        run: bool,
+    ) -> Result<(), Error> {
+        let linkno = routing_table.0[destination as usize][0] - 1;
+        let reply = aux_transact(
+            aux_mutex,
+            linkno,
+            routing_table,
+            &Packet::SubkernelLoadRunRequest {
+                id: id,
+                source: 0,
+                destination: destination,
+                run: run,
+            },
+            timer,
+        )
+        .await?;
+        match reply {
+            Packet::SubkernelLoadRunReply {
+                destination: 0,
+                succeeded: true,
+            } => return Ok(()),
+            Packet::SubkernelLoadRunReply {
+                destination: 0,
+                succeeded: false,
+            } => return Err(Error::SubkernelRunFail(destination)),
+            _ => Err(Error::UnexpectedReply),
+        }
+    }
+
+    pub async fn subkernel_retrieve_exception(
+        aux_mutex: &Rc<Mutex<bool>>,
+        routing_table: &RoutingTable,
+        timer: GlobalTimer,
+        destination: u8,
+    ) -> Result<Vec<u8>, Error> {
+        let linkno = routing_table.0[destination as usize][0] - 1;
+        let mut remote_data: Vec<u8> = Vec::new();
+        loop {
+            let reply = aux_transact(
+                aux_mutex,
+                linkno,
+                routing_table,
+                &Packet::SubkernelExceptionRequest {
+                    destination: destination,
+                },
+                timer,
+            )
+            .await?;
+            match reply {
+                Packet::SubkernelException { last, length, data } => {
+                    remote_data.extend(&data[0..length as usize]);
+                    if last {
+                        return Ok(remote_data);
+                    }
+                }
+                _ => return Err(Error::UnexpectedReply),
+            }
+        }
+    }
+
+    pub async fn subkernel_send_message(
+        aux_mutex: &Rc<Mutex<bool>>,
+        routing_table: &RoutingTable,
+        timer: GlobalTimer,
+        id: u32,
+        destination: u8,
+        message: &[u8],
+    ) -> Result<(), Error> {
+        let linkno = routing_table.0[destination as usize][0] - 1;
+        partition_data(
+            linkno,
+            aux_mutex,
+            routing_table,
+            timer,
+            message,
+            |slice, status, len| Packet::SubkernelMessage {
+                source: 0,
+                destination: destination,
+                id: id,
+                status: status,
+                length: len as u16,
+                data: *slice,
+            },
+            |reply| match reply {
+                Packet::SubkernelMessageAck { .. } => Ok(()),
+                _ => Err(Error::UnexpectedReply),
+            },
+        )
+        .await
+    }
 }
 
 #[cfg(not(has_drtio))]
@@ -570,26 +888,22 @@ pub mod drtio {
 
     pub fn startup(
         _aux_mutex: &Rc<Mutex<bool>>,
-        _routing_table: &Rc<RefCell<drtio_routing::RoutingTable>>,
+        _routing_table: &Rc<RefCell<RoutingTable>>,
         _up_destinations: &Rc<RefCell<[bool; drtio_routing::DEST_COUNT]>>,
         _timer: GlobalTimer,
     ) {
     }
 
     #[allow(dead_code)]
-    pub fn reset(_aux_mutex: Rc<Mutex<bool>>, mut _timer: GlobalTimer) {}
+    pub fn reset(_aux_mutex: Rc<Mutex<bool>>, _routing_table: &RoutingTable, mut _timer: GlobalTimer) {}
 }
 
 pub fn startup(
     aux_mutex: &Rc<Mutex<bool>>,
-    routing_table: &Rc<RefCell<drtio_routing::RoutingTable>>,
+    routing_table: &Rc<RefCell<RoutingTable>>,
     up_destinations: &Rc<RefCell<[bool; drtio_routing::DEST_COUNT]>>,
     timer: GlobalTimer,
-    cfg: &Config,
 ) {
-    unsafe {
-        RTIO_DEVICE_MAP = read_device_map(cfg);
-    }
     drtio::startup(aux_mutex, routing_table, up_destinations, timer);
     unsafe {
         csr::rtio_core::reset_phy_write(1);
@@ -597,9 +911,9 @@ pub fn startup(
 }
 
 #[allow(dead_code)]
-pub fn reset(aux_mutex: Rc<Mutex<bool>>, timer: GlobalTimer) {
+pub fn reset(aux_mutex: Rc<Mutex<bool>>, routing_table: &RoutingTable, timer: GlobalTimer) {
     unsafe {
         csr::rtio_core::reset_write(1);
     }
-    drtio::reset(aux_mutex, timer)
+    drtio::reset(aux_mutex, routing_table, timer)
 }

src/runtime/src/subkernel.rs

@@ -0,0 +1,325 @@
+use alloc::{collections::BTreeMap, rc::Rc, vec::Vec};
+
+use libasync::task;
+use libboard_artiq::{drtio_routing::RoutingTable,
+                     drtioaux_proto::{PayloadStatus, MASTER_PAYLOAD_MAX_SIZE}};
+use libboard_zynq::{time::Milliseconds, timer::GlobalTimer};
+use libcortex_a9::mutex::Mutex;
+use log::{error, warn};
+
+use crate::rtio_mgt::{drtio, drtio::Error as DrtioError};
+
+#[derive(Debug, PartialEq, Clone, Copy)]
+pub enum FinishStatus {
+    Ok,
+    CommLost,
+    Exception(u8), // exception source
+}
+
+#[derive(Debug, PartialEq, Clone, Copy)]
+pub enum SubkernelState {
+    NotLoaded,
+    Uploaded,
+    Running,
+    Finished { status: FinishStatus },
+}
+
+#[derive(Debug, PartialEq, Eq, Clone, Copy)]
+pub enum Error {
+    Timeout,
+    IncorrectState,
+    SubkernelNotFound,
+    SubkernelException,
+    CommLost,
+    DrtioError(DrtioError),
+}
+
+impl From<DrtioError> for Error {
+    fn from(value: DrtioError) -> Error {
+        Error::DrtioError(value)
+    }
+}
+
+pub struct SubkernelFinished {
+    pub id: u32,
+    pub status: FinishStatus,
+    pub exception: Option<Vec<u8>>,
+}
+
+struct Subkernel {
+    pub destination: u8,
+    pub data: Vec<u8>,
+    pub state: SubkernelState,
+}
+
+impl Subkernel {
+    pub fn new(destination: u8, data: Vec<u8>) -> Self {
+        Subkernel {
+            destination: destination,
+            data: data,
+            state: SubkernelState::NotLoaded,
+        }
+    }
+}
+
+static SUBKERNELS: Mutex<BTreeMap<u32, Subkernel>> = Mutex::new(BTreeMap::new());
+
+pub async fn add_subkernel(id: u32, destination: u8, kernel: Vec<u8>) {
+    SUBKERNELS
+        .async_lock()
+        .await
+        .insert(id, Subkernel::new(destination, kernel));
+}
+
+pub async fn upload(
+    aux_mutex: &Rc<Mutex<bool>>,
+    routing_table: &RoutingTable,
+    timer: GlobalTimer,
+    id: u32,
+) -> Result<(), Error> {
+    if let Some(subkernel) = SUBKERNELS.async_lock().await.get_mut(&id) {
+        drtio::subkernel_upload(
+            aux_mutex,
+            routing_table,
+            timer,
+            id,
+            subkernel.destination,
+            &subkernel.data,
+        )
+        .await?;
+        subkernel.state = SubkernelState::Uploaded;
+        Ok(())
+    } else {
+        Err(Error::SubkernelNotFound)
+    }
+}
+
+pub async fn load(
+    aux_mutex: &Rc<Mutex<bool>>,
+    routing_table: &RoutingTable,
+    timer: GlobalTimer,
+    id: u32,
+    run: bool,
+) -> Result<(), Error> {
+    if let Some(subkernel) = SUBKERNELS.async_lock().await.get_mut(&id) {
+        if subkernel.state != SubkernelState::Uploaded {
+            return Err(Error::IncorrectState);
+        }
+        drtio::subkernel_load(aux_mutex, routing_table, timer, id, subkernel.destination, run).await?;
+        if run {
+            subkernel.state = SubkernelState::Running;
+        }
+        Ok(())
+    } else {
+        Err(Error::SubkernelNotFound)
+    }
+}
+
+pub async fn clear_subkernels() {
+    SUBKERNELS.async_lock().await.clear();
+    MESSAGE_QUEUE.async_lock().await.clear();
+    CURRENT_MESSAGES.async_lock().await.clear();
+}
+
+pub async fn subkernel_finished(id: u32, with_exception: bool, exception_src: u8) {
+    // called upon receiving DRTIO SubkernelRunDone
+    // may be None if session ends and is cleared
+    if let Some(subkernel) = SUBKERNELS.async_lock().await.get_mut(&id) {
+        if subkernel.state == SubkernelState::Running {
+            subkernel.state = SubkernelState::Finished {
+                status: match with_exception {
+                    true => FinishStatus::Exception(exception_src),
+                    false => FinishStatus::Ok,
+                },
+            }
+        }
+    }
+}
+
+pub async fn destination_changed(
+    aux_mutex: &Rc<Mutex<bool>>,
+    routing_table: &RoutingTable,
+    timer: GlobalTimer,
+    destination: u8,
+    up: bool,
+) {
+    let mut locked_subkernels = SUBKERNELS.async_lock().await;
+    for (id, subkernel) in locked_subkernels.iter_mut() {
+        if subkernel.destination == destination {
+            if up {
+                match drtio::subkernel_upload(aux_mutex, routing_table, timer, *id, destination, &subkernel.data).await
+                {
+                    Ok(_) => subkernel.state = SubkernelState::Uploaded,
+                    Err(e) => error!("Error adding subkernel on destination {}: {}", destination, e),
+                }
+            } else {
+                subkernel.state = match subkernel.state {
+                    SubkernelState::Running => SubkernelState::Finished {
+                        status: FinishStatus::CommLost,
+                    },
+                    _ => SubkernelState::NotLoaded,
+                }
+            }
+        }
+    }
+}
+
+pub async fn await_finish(
+    aux_mutex: &Rc<Mutex<bool>>,
+    routing_table: &RoutingTable,
+    timer: GlobalTimer,
+    id: u32,
+    timeout: i64,
+) -> Result<SubkernelFinished, Error> {
+    match SUBKERNELS.async_lock().await.get(&id).unwrap().state {
+        SubkernelState::Running | SubkernelState::Finished { .. } => (),
+        _ => return Err(Error::IncorrectState),
+    }
+    if timeout > 0 {
+        let max_time = timer.get_time() + Milliseconds(timeout as u64);
+        while timer.get_time() < max_time {
+            match SUBKERNELS.async_lock().await.get(&id).unwrap().state {
+                SubkernelState::Finished { .. } => break,
+                _ => (),
+            };
+            task::r#yield().await;
+        }
+        if timer.get_time() >= max_time {
+            error!("Remote subkernel finish await timed out");
+            return Err(Error::Timeout);
+        }
+    } else {
+        // no timeout, wait forever
+        loop {
+            match SUBKERNELS.async_lock().await.get(&id).unwrap().state {
+                SubkernelState::Finished { .. } => break,
+                _ => (),
+            };
+            task::r#yield().await;
+        }
+    }
+    if let Some(subkernel) = SUBKERNELS.async_lock().await.get_mut(&id) {
+        match subkernel.state {
+            SubkernelState::Finished { status } => {
+                subkernel.state = SubkernelState::Uploaded;
+                Ok(SubkernelFinished {
+                    id: id,
+                    status: status,
+                    exception: if let FinishStatus::Exception(dest) = status {
+                        Some(drtio::subkernel_retrieve_exception(aux_mutex, routing_table, timer, dest).await?)
+                    } else {
+                        None
+                    },
+                })
+            }
+            _ => Err(Error::IncorrectState),
+        }
+    } else {
+        Err(Error::SubkernelNotFound)
+    }
+}
+
+pub struct Message {
+    from_id: u32,
+    pub count: u8,
+    pub data: Vec<u8>,
+}
+
+// FIFO queue of messages
+static MESSAGE_QUEUE: Mutex<Vec<Message>> = Mutex::new(Vec::new());
+// currently under construction message(s) (can be from multiple sources)
+static CURRENT_MESSAGES: Mutex<BTreeMap<u32, Message>> = Mutex::new(BTreeMap::new());
+
+pub async fn message_handle_incoming(
+    id: u32,
+    status: PayloadStatus,
+    length: usize,
+    data: &[u8; MASTER_PAYLOAD_MAX_SIZE],
+) {
+    // called when receiving a message from satellite
+    {
+        let subkernel_lock = SUBKERNELS.async_lock().await;
+        let subkernel = subkernel_lock.get(&id);
+        if subkernel.is_some() && subkernel.unwrap().state != SubkernelState::Running {
+            // do not add messages for non-running or deleted subkernels
+            warn!("received a message for a non-running subkernel #{}", id);
+            return;
+        }
+    }
+    let mut current_messages = CURRENT_MESSAGES.async_lock().await;
+
+    if status.is_first() {
+        current_messages.remove(&id);
+    }
+
+    match current_messages.get_mut(&id) {
+        Some(message) => message.data.extend(&data[..length]),
+        None => {
+            current_messages.insert(
+                id,
+                Message {
+                    from_id: id,
+                    count: data[0],
+                    data: data[1..length].to_vec(),
+                },
+            );
+        }
+    };
+    if status.is_last() {
+        // when done, remove from working queue
+        MESSAGE_QUEUE
+            .async_lock()
+            .await
+            .push(current_messages.remove(&id).unwrap());
+    }
+}
+
+pub async fn message_await(id: u32, timeout: i64, timer: GlobalTimer) -> Result<Message, Error> {
+    let is_subkernel = SUBKERNELS.async_lock().await.get(&id).is_some();
+    if is_subkernel {
+        match SUBKERNELS.async_lock().await.get(&id).unwrap().state {
+            SubkernelState::Finished {
+                status: FinishStatus::CommLost,
+            } => return Err(Error::CommLost),
+            SubkernelState::Running | SubkernelState::Finished { .. } => (),
+            _ => return Err(Error::IncorrectState),
+        }
+    }
+    let max_time = timer.get_time() + Milliseconds(timeout as u64);
+    while timeout < 0 || (timeout > 0 && timer.get_time() < max_time) {
+        {
+            let mut message_queue = MESSAGE_QUEUE.async_lock().await;
+            for i in 0..message_queue.len() {
+                let msg = &message_queue[i];
+                if msg.from_id == id {
+                    let message = message_queue.remove(i);
+                    return Ok(message);
+                }
+            }
+        }
+        if is_subkernel {
+            match SUBKERNELS.async_lock().await.get(&id).unwrap().state {
+                SubkernelState::Finished {
+                    status: FinishStatus::CommLost,
+                } => return Err(Error::CommLost),
+                SubkernelState::Finished {
+                    status: FinishStatus::Exception(_),
+                } => return Err(Error::SubkernelException),
+                _ => (),
+            }
+        }
+        task::r#yield().await;
+    }
+    Err(Error::Timeout)
+}
+
+pub async fn message_send<'a>(
+    aux_mutex: &Rc<Mutex<bool>>,
+    routing_table: &RoutingTable,
+    timer: GlobalTimer,
+    id: u32,
+    destination: u8,
+    message: Vec<u8>,
+) -> Result<(), Error> {
+    Ok(drtio::subkernel_send_message(aux_mutex, routing_table, timer, id, destination, &message).await?)
+}

src/satman/Cargo.toml

@@ -1,28 +0,0 @@
-[package]
-authors = ["M-Labs"]
-name = "satman"
-version = "0.0.0"
-build = "build.rs"
-
-[features]
-target_zc706 = ["libboard_zynq/target_zc706", "libsupport_zynq/target_zc706", "libconfig/target_zc706", "libboard_artiq/target_zc706"]
-target_kasli_soc = ["libboard_zynq/target_kasli_soc", "libsupport_zynq/target_kasli_soc", "libconfig/target_kasli_soc", "libboard_artiq/target_kasli_soc"]
-default = ["target_zc706", ]
-
-[build-dependencies]
-build_zynq = { path = "../libbuild_zynq" }
-
-[dependencies]
-log = { version = "0.4", default-features = false }
-embedded-hal = "0.2"
-
-libboard_zynq = { git = "https://git.m-labs.hk/M-Labs/zynq-rs.git", features = ["ipv6"]}
-libsupport_zynq = { default-features = false, features = ["alloc_core"], git = "https://git.m-labs.hk/M-Labs/zynq-rs.git" }
-libcortex_a9 = { git = "https://git.m-labs.hk/M-Labs/zynq-rs.git" }
-libasync = { git = "https://git.m-labs.hk/M-Labs/zynq-rs.git" }
-libregister = { git = "https://git.m-labs.hk/M-Labs/zynq-rs.git" }
-libconfig = { git = "https://git.m-labs.hk/M-Labs/zynq-rs.git", features = ["fat_lfn", "ipv6"] }
-
-libboard_artiq = { path = "../libboard_artiq" }
-unwind = { path = "../libunwind" }
-libc = { path = "../libc" }

src/satman/Cargo.toml.tpl

@@ -0,0 +1,33 @@
+[package]
+authors = ["M-Labs"]
+name = "satman"
+version = "0.0.0"
+build = "build.rs"
+
+[features]
+target_zc706 = ["libboard_zynq/target_zc706", "libsupport_zynq/target_zc706", "libconfig/target_zc706", "libboard_artiq/target_zc706"]
+target_kasli_soc = ["libboard_zynq/target_kasli_soc", "libsupport_zynq/target_kasli_soc", "libconfig/target_kasli_soc", "libboard_artiq/target_kasli_soc"]
+calibrate_wrpll_skew = ["libboard_artiq/calibrate_wrpll_skew"]
+default = ["target_zc706", ]
+
+[build-dependencies]
+build_zynq = { path = "../libbuild_zynq" }
+
+[dependencies]
+log = { version = "0.4", default-features = false }
+core_io = { version = "0.1", features = ["collections"] }
+cslice = "0.3"
+embedded-hal = "0.2"
+
+libboard_zynq = { path = "@@ZYNQ_RS@@/libboard_zynq", features = ["ipv6"]}
+libsupport_zynq = { path = "@@ZYNQ_RS@@/libsupport_zynq", default-features = false, features = ["alloc_core"] }
+libcortex_a9 = { path = "@@ZYNQ_RS@@/libcortex_a9" }
+libasync = { path = "@@ZYNQ_RS@@/libasync" }
+libregister = { path = "@@ZYNQ_RS@@/libregister" }
+libconfig = { path = "@@ZYNQ_RS@@/libconfig", features = ["fat_lfn", "ipv6"] }
+
+libboard_artiq = { path = "../libboard_artiq" }
+unwind = { path = "../libunwind" }
+libc = { path = "../libc" }
+io = { path = "../libio", features = ["alloc"] }
+ksupport = { path = "../libksupport" }

src/satman/src/analyzer.rs

@@ -0,0 +1,126 @@
+use core::cmp::min;
+
+use libboard_artiq::{drtioaux_proto::SAT_PAYLOAD_MAX_SIZE, pl::csr};
+use libcortex_a9::cache;
+
+const BUFFER_SIZE: usize = 512 * 1024;
+
+#[repr(align(64))]
+struct Buffer {
+    data: [u8; BUFFER_SIZE],
+}
+
+static mut BUFFER: Buffer = Buffer { data: [0; BUFFER_SIZE] };
+
+fn arm() {
+    unsafe {
+        let base_addr = &mut BUFFER.data[0] as *mut _ as usize;
+        let last_addr = &mut BUFFER.data[BUFFER_SIZE - 1] as *mut _ as usize;
+        csr::rtio_analyzer::dma_base_address_write(base_addr as u32);
+        csr::rtio_analyzer::message_encoder_overflow_reset_write(1);
+        csr::rtio_analyzer::dma_last_address_write(last_addr as u32);
+        csr::rtio_analyzer::dma_reset_write(1);
+        csr::rtio_analyzer::enable_write(1);
+    }
+}
+
+fn disarm() {
+    unsafe {
+        csr::rtio_analyzer::enable_write(0);
+        while csr::rtio_analyzer::busy_read() != 0 {}
+        cache::dcci_slice(&BUFFER.data);
+    }
+}
+
+pub struct Analyzer {
+    // necessary for keeping track of sent data
+    data_len: usize,
+    sent_bytes: usize,
+    data_pointer: usize,
+}
+
+pub struct Header {
+    pub total_byte_count: u64,
+    pub sent_bytes: u32,
+    pub error: bool,
+}
+
+pub struct AnalyzerSliceMeta {
+    pub len: u16,
+    pub last: bool,
+}
+
+impl Drop for Analyzer {
+    fn drop(&mut self) {
+        disarm();
+    }
+}
+
+impl Analyzer {
+    pub fn new() -> Analyzer {
+        // create and arm new Analyzer
+        arm();
+        Analyzer {
+            data_len: 0,
+            sent_bytes: 0,
+            data_pointer: 0,
+        }
+    }
+
+    pub fn get_header(&mut self) -> Header {
+        disarm();
+
+        let overflow = unsafe { csr::rtio_analyzer::message_encoder_overflow_read() != 0 };
+        let bus_err = unsafe { csr::rtio_analyzer::dma_bus_error_read() != 0 };
+        let total_byte_count = unsafe { csr::rtio_analyzer::dma_byte_count_read() as u64 };
+        let wraparound = total_byte_count >= BUFFER_SIZE as u64;
+        self.data_len = if wraparound {
+            BUFFER_SIZE
+        } else {
+            total_byte_count as usize
+        };
+        self.data_pointer = if wraparound {
+            (total_byte_count % BUFFER_SIZE as u64) as usize
+        } else {
+            0
+        };
+        self.sent_bytes = 0;
+
+        if overflow {
+            warn!("overflow occured");
+        }
+        if bus_err {
+            warn!("bus error occured");
+        }
+
+        Header {
+            total_byte_count: total_byte_count,
+            sent_bytes: self.data_len as u32,
+            error: overflow | bus_err,
+        }
+    }
+
+    pub fn get_data(&mut self, data_slice: &mut [u8; SAT_PAYLOAD_MAX_SIZE]) -> AnalyzerSliceMeta {
+        let data = unsafe { &BUFFER.data[..] };
+        let i = (self.data_pointer + self.sent_bytes) % BUFFER_SIZE;
+        let len = min(SAT_PAYLOAD_MAX_SIZE, self.data_len - self.sent_bytes);
+        let last = self.sent_bytes + len == self.data_len;
+
+        if i + len >= BUFFER_SIZE {
+            data_slice[..(BUFFER_SIZE - i)].clone_from_slice(&data[i..BUFFER_SIZE]);
+            data_slice[(BUFFER_SIZE - i)..len].clone_from_slice(&data[..(i + len) % BUFFER_SIZE]);
+        } else {
+            data_slice[..len].clone_from_slice(&data[i..i + len]);
+        }
+        self.sent_bytes += len;
+
+        if last {
+            arm();
+        }
+
+        AnalyzerSliceMeta {
+            len: len as u16,
+            last: last,
+        }
+    }
+}

src/satman/src/dma.rs

@@ -1,7 +1,13 @@
-use alloc::{collections::btree_map::BTreeMap, vec::Vec};
+use alloc::{collections::btree_map::BTreeMap, string::String, vec::Vec};
+use core::mem;
 
-use libboard_artiq::pl::csr;
+use ksupport::kernel::DmaRecorder;
+use libboard_artiq::{drtio_routing::RoutingTable,
+                     drtioaux_proto::{Packet, PayloadStatus, MASTER_PAYLOAD_MAX_SIZE},
+                     pl::csr};
 use libcortex_a9::cache::dcci_slice;
+use routing::{Router, Sliceable};
+use subkernel::Manager as KernelManager;
 
 const ALIGNMENT: usize = 64;
 
@@ -12,16 +18,20 @@ enum ManagerState {
 }
 
 pub struct RtioStatus {
+    pub source: u8,
     pub id: u32,
     pub error: u8,
     pub channel: u32,
     pub timestamp: u64,
 }
 
+#[derive(Debug)]
 pub enum Error {
     IdNotFound,
     PlaybackInProgress,
     EntryNotComplete,
+    MasterDmaFound,
+    UploadFail,
 }
 
 #[derive(Debug)]
@@ -29,13 +39,228 @@ struct Entry {
     trace: Vec<u8>,
     padding_len: usize,
     complete: bool,
+    duration: i64, // relevant for local DMA
+}
+
+impl Entry {
+    pub fn from_vec(data: Vec<u8>, duration: i64) -> Entry {
+        let mut entry = Entry {
+            trace: data,
+            padding_len: 0,
+            complete: true,
+            duration: duration,
+        };
+        entry.realign();
+        entry
+    }
+
+    pub fn id(&self) -> u32 {
+        self.trace[self.padding_len..].as_ptr() as u32
+    }
+
+    pub fn realign(&mut self) {
+        self.trace.push(0);
+        let data_len = self.trace.len();
+
+        self.trace.reserve(ALIGNMENT - 1);
+        let padding = ALIGNMENT - self.trace.as_ptr() as usize % ALIGNMENT;
+        let padding = if padding == ALIGNMENT { 0 } else { padding };
+        for _ in 0..padding {
+            // Vec guarantees that this will not reallocate
+            self.trace.push(0)
+        }
+        for i in 1..data_len + 1 {
+            self.trace[data_len + padding - i] = self.trace[data_len - i]
+        }
+        self.complete = true;
+        self.padding_len = padding;
+
+        dcci_slice(&self.trace);
+    }
+}
+
+#[derive(Debug)]
+enum RemoteTraceState {
+    Unsent,
+    Sending(usize),
+    Ready,
+    Running(usize),
+}
+
+#[derive(Debug)]
+struct RemoteTraces {
+    remote_traces: BTreeMap<u8, Sliceable>,
+    state: RemoteTraceState,
+}
+
+impl RemoteTraces {
+    pub fn new(traces: BTreeMap<u8, Sliceable>) -> RemoteTraces {
+        RemoteTraces {
+            remote_traces: traces,
+            state: RemoteTraceState::Unsent,
+        }
+    }
+
+    // on subkernel request
+    pub fn upload_traces(
+        &mut self,
+        id: u32,
+        router: &mut Router,
+        rank: u8,
+        self_destination: u8,
+        routing_table: &RoutingTable,
+    ) -> usize {
+        let len = self.remote_traces.len();
+        if len > 0 {
+            self.state = RemoteTraceState::Sending(self.remote_traces.len());
+            for (dest, trace) in self.remote_traces.iter_mut() {
+                // queue up the first packet for all destinations, rest will be sent after first ACK
+                let mut data_slice: [u8; MASTER_PAYLOAD_MAX_SIZE] = [0; MASTER_PAYLOAD_MAX_SIZE];
+                let meta = trace.get_slice_master(&mut data_slice);
+                router.route(
+                    Packet::DmaAddTraceRequest {
+                        source: self_destination,
+                        destination: *dest,
+                        id: id,
+                        status: meta.status,
+                        length: meta.len,
+                        trace: data_slice,
+                    },
+                    routing_table,
+                    rank,
+                    self_destination,
+                );
+            }
+        }
+        len
+    }
+
+    // on incoming Packet::DmaAddTraceReply
+    pub fn ack_upload(
+        &mut self,
+        kernel_manager: &mut KernelManager,
+        source: u8,
+        id: u32,
+        succeeded: bool,
+        router: &mut Router,
+        rank: u8,
+        self_destination: u8,
+        routing_table: &RoutingTable,
+    ) {
+        if let RemoteTraceState::Sending(count) = self.state {
+            if let Some(trace) = self.remote_traces.get_mut(&source) {
+                if trace.at_end() {
+                    if count - 1 == 0 {
+                        self.state = RemoteTraceState::Ready;
+                        if let Some((id, timestamp)) = kernel_manager.ddma_remote_uploaded(succeeded) {
+                            self.playback(id, timestamp, router, rank, self_destination, routing_table);
+                        }
+                    } else {
+                        self.state = RemoteTraceState::Sending(count - 1);
+                    }
+                } else {
+                    // send next slice
+                    let mut data_slice: [u8; MASTER_PAYLOAD_MAX_SIZE] = [0; MASTER_PAYLOAD_MAX_SIZE];
+                    let meta = trace.get_slice_master(&mut data_slice);
+                    router.route(
+                        Packet::DmaAddTraceRequest {
+                            source: self_destination,
+                            destination: meta.destination,
+                            id: id,
+                            status: meta.status,
+                            length: meta.len,
+                            trace: data_slice,
+                        },
+                        routing_table,
+                        rank,
+                        self_destination,
+                    );
+                }
+            }
+        }
+    }
+
+    // on subkernel request
+    pub fn playback(
+        &mut self,
+        id: u32,
+        timestamp: u64,
+        router: &mut Router,
+        rank: u8,
+        self_destination: u8,
+        routing_table: &RoutingTable,
+    ) {
+        // route all the playback requests
+        // remote traces (local trace runs on core1 unlike mainline firmware)
+        self.state = RemoteTraceState::Running(self.remote_traces.len());
+        for (dest, _) in self.remote_traces.iter() {
+            router.route(
+                Packet::DmaPlaybackRequest {
+                    source: self_destination,
+                    destination: *dest,
+                    id: id,
+                    timestamp: timestamp,
+                },
+                routing_table,
+                rank,
+                self_destination,
+            );
+            // response will be ignored (succeeded = false handled by the main thread)
+        }
+    }
+
+    // on incoming Packet::DmaPlaybackDone
+    pub fn remote_finished(&mut self, kernel_manager: &mut KernelManager, error: u8, channel: u32, timestamp: u64) {
+        if let RemoteTraceState::Running(count) = self.state {
+            if error != 0 || count - 1 == 0 {
+                // notify the kernel about a DDMA error or finish
+                kernel_manager.ddma_finished(error, channel, timestamp);
+                self.state = RemoteTraceState::Ready;
+                // further messages will be ignored (if there was an error)
+            } else {
+                // no error and not the last one awaited
+                self.state = RemoteTraceState::Running(count - 1);
+            }
+        }
+    }
+
+    pub fn erase(
+        &mut self,
+        id: u32,
+        router: &mut Router,
+        rank: u8,
+        self_destination: u8,
+        routing_table: &RoutingTable,
+    ) {
+        for (dest, _) in self.remote_traces.iter() {
+            router.route(
+                Packet::DmaRemoveTraceRequest {
+                    source: self_destination,
+                    destination: *dest,
+                    id: id,
+                },
+                routing_table,
+                rank,
+                self_destination,
+            );
+            // response will be ignored as this object will stop existing too
+        }
+    }
+
+    pub fn has_remote_traces(&self) -> bool {
+        self.remote_traces.len() > 0
+    }
 }
 
 #[derive(Debug)]
 pub struct Manager {
-    entries: BTreeMap<u32, Entry>,
+    entries: BTreeMap<(u8, u32), Entry>,
     state: ManagerState,
-    currentid: u32,
+    current_id: u32,
+    current_source: u8,
+
+    remote_entries: BTreeMap<u32, RemoteTraces>,
+    name_map: BTreeMap<String, u32>,
 }
 
 impl Manager {
@@ -45,79 +270,238 @@ impl Manager {
         unsafe { while csr::rtio_dma::enable_read() != 0 {} }
         Manager {
             entries: BTreeMap::new(),
-            currentid: 0,
+            current_id: 0,
+            current_source: 0,
             state: ManagerState::Idle,
+            remote_entries: BTreeMap::new(),
+            name_map: BTreeMap::new(),
         }
     }
 
-    pub fn add(&mut self, id: u32, last: bool, trace: &[u8], trace_len: usize) -> Result<(), Error> {
-        let entry = match self.entries.get_mut(&id) {
+    pub fn add(
+        &mut self,
+        source: u8,
+        id: u32,
+        status: PayloadStatus,
+        trace: &[u8],
+        trace_len: usize,
+    ) -> Result<(), Error> {
+        let entry = match self.entries.get_mut(&(source, id)) {
             Some(entry) => {
-                if entry.complete {
+                if entry.complete || status.is_first() {
                     // replace entry
-                    self.entries.remove(&id);
+                    self.entries.remove(&(source, id));
                     self.entries.insert(
-                        id,
+                        (source, id),
                         Entry {
                             trace: Vec::new(),
                             padding_len: 0,
                             complete: false,
+                            duration: 0,
                         },
                     );
-                    self.entries.get_mut(&id).unwrap()
+                    self.entries.get_mut(&(source, id)).unwrap()
                 } else {
                     entry
                 }
             }
             None => {
                 self.entries.insert(
-                    id,
+                    (source, id),
                     Entry {
                         trace: Vec::new(),
                         padding_len: 0,
                         complete: false,
+                        duration: 0,
                     },
                 );
-                self.entries.get_mut(&id).unwrap()
+                self.entries.get_mut(&(source, id)).unwrap()
             }
         };
         entry.trace.extend(&trace[0..trace_len]);
 
-        if last {
-            entry.trace.push(0);
-            let data_len = entry.trace.len();
-
-            // Realign.
-            entry.trace.reserve(ALIGNMENT - 1);
-            let padding = ALIGNMENT - entry.trace.as_ptr() as usize % ALIGNMENT;
-            let padding = if padding == ALIGNMENT { 0 } else { padding };
-            for _ in 0..padding {
-                // Vec guarantees that this will not reallocate
-                entry.trace.push(0)
-            }
-            for i in 1..data_len + 1 {
-                entry.trace[data_len + padding - i] = entry.trace[data_len - i]
-            }
-            entry.complete = true;
-            entry.padding_len = padding;
-            dcci_slice(&entry.trace);
+        if status.is_last() {
+            entry.realign();
         }
         Ok(())
     }
 
-    pub fn erase(&mut self, id: u32) -> Result<(), Error> {
-        match self.entries.remove(&id) {
+    // api for DRTIO
+    pub fn erase(&mut self, source: u8, id: u32) -> Result<(), Error> {
+        match self.entries.remove(&(source, id)) {
             Some(_) => Ok(()),
             None => Err(Error::IdNotFound),
         }
     }
 
-    pub fn playback(&mut self, id: u32, timestamp: u64) -> Result<(), Error> {
+    // API for subkernel
+    pub fn erase_name(
+        &mut self,
+        name: &str,
+        router: &mut Router,
+        rank: u8,
+        self_destination: u8,
+        routing_table: &RoutingTable,
+    ) {
+        if let Some(id) = self.name_map.get(name) {
+            if let Some(traces) = self.remote_entries.get_mut(&id) {
+                traces.erase(*id, router, rank, self_destination, routing_table);
+                self.remote_entries.remove(&id);
+            }
+            self.entries.remove(&(self_destination, *id));
+            self.name_map.remove(name);
+        }
+    }
+
+    pub fn remote_finished(
+        &mut self,
+        kernel_manager: &mut KernelManager,
+        id: u32,
+        error: u8,
+        channel: u32,
+        timestamp: u64,
+    ) {
+        if let Some(entry) = self.remote_entries.get_mut(&id) {
+            entry.remote_finished(kernel_manager, error, channel, timestamp);
+        }
+    }
+
+    pub fn ack_upload(
+        &mut self,
+        kernel_manager: &mut KernelManager,
+        source: u8,
+        id: u32,
+        succeeded: bool,
+        router: &mut Router,
+        rank: u8,
+        self_destination: u8,
+        routing_table: &RoutingTable,
+    ) {
+        if let Some(entry) = self.remote_entries.get_mut(&id) {
+            entry.ack_upload(
+                kernel_manager,
+                source,
+                id,
+                succeeded,
+                router,
+                rank,
+                self_destination,
+                routing_table,
+            );
+        }
+    }
+
+    // API for subkernel
+    pub fn upload_traces(
+        &mut self,
+        id: u32,
+        router: &mut Router,
+        rank: u8,
+        self_destination: u8,
+        routing_table: &RoutingTable,
+    ) -> Result<usize, Error> {
+        let remote_traces = self.remote_entries.get_mut(&id);
+        let mut len = 0;
+        if let Some(traces) = remote_traces {
+            len = traces.upload_traces(id, router, rank, self_destination, routing_table);
+        }
+        Ok(len)
+    }
+
+    // API for subkernel
+    pub fn playback_remote(
+        &mut self,
+        id: u32,
+        timestamp: u64,
+        router: &mut Router,
+        rank: u8,
+        self_destination: u8,
+        routing_table: &RoutingTable,
+    ) -> Result<(), Error> {
+        if let Some(traces) = self.remote_entries.get_mut(&id) {
+            traces.playback(id, timestamp, router, rank, self_destination, routing_table);
+            Ok(())
+        } else {
+            Err(Error::IdNotFound)
+        }
+    }
+
+    // API for subkernel
+    pub fn cleanup(&mut self, router: &mut Router, rank: u8, self_destination: u8, routing_table: &RoutingTable) {
+        // after subkernel ends, remove all self-generated traces
+        for (_, id) in self.name_map.iter_mut() {
+            if let Some(traces) = self.remote_entries.get_mut(&id) {
+                traces.erase(*id, router, rank, self_destination, routing_table);
+                self.remote_entries.remove(&id);
+            }
+            self.entries.remove(&(self_destination, *id));
+        }
+        self.name_map.clear();
+    }
+
+    // API for subkernel
+    pub fn retrieve(&self, self_destination: u8, name: &String) -> Option<(i32, i64, bool)> {
+        let id = self.name_map.get(name)?;
+        let duration = self.entries.get(&(self_destination, *id))?.duration;
+        let uses_ddma = self.has_remote_traces(*id);
+        Some((*id as i32, duration, uses_ddma))
+    }
+
+    pub fn has_remote_traces(&self, id: u32) -> bool {
+        match self.remote_entries.get(&id) {
+            Some(traces) => traces.has_remote_traces(),
+            _ => false,
+        }
+    }
+
+    pub fn put_record(&mut self, mut recorder: DmaRecorder, self_destination: u8) -> Result<u32, Error> {
+        let mut remote_traces: BTreeMap<u8, Sliceable> = BTreeMap::new();
+
+        let mut local_trace: Vec<u8> = Vec::new();
+        // analyze each entry and put in proper buckets, as the kernel core
+        // sends whole chunks, to limit comms/kernel CPU communication,
+        // and as only comms core has access to varios DMA buffers.
+        let mut ptr = 0;
+        recorder.buffer.push(0);
+        while recorder.buffer[ptr] != 0 {
+            // ptr + 3 = tgt >> 24 (destination)
+            let len = recorder.buffer[ptr] as usize;
+            let destination = recorder.buffer[ptr + 3];
+            if destination == 0 {
+                return Err(Error::MasterDmaFound);
+            } else if destination == self_destination {
+                local_trace.extend(&recorder.buffer[ptr..ptr + len]);
+            } else {
+                if let Some(remote_trace) = remote_traces.get_mut(&destination) {
+                    remote_trace.extend(&recorder.buffer[ptr..ptr + len]);
+                } else {
+                    remote_traces.insert(
+                        destination,
+                        Sliceable::new(destination, recorder.buffer[ptr..ptr + len].to_vec()),
+                    );
+                }
+            }
+            // and jump to the next event
+            ptr += len;
+        }
+        let local_entry = Entry::from_vec(local_trace, recorder.duration);
+
+        let id = local_entry.id();
+        self.entries.insert((self_destination, id), local_entry);
+        self.remote_entries.insert(id, RemoteTraces::new(remote_traces));
+        let mut name = String::new();
+        mem::swap(&mut recorder.name, &mut name);
+        self.name_map.insert(name, id);
+
+        Ok(id)
+    }
+
+    pub fn playback(&mut self, source: u8, id: u32, timestamp: u64) -> Result<(), Error> {
         if self.state != ManagerState::Idle {
             return Err(Error::PlaybackInProgress);
         }
 
-        let entry = match self.entries.get(&id) {
+        let entry = match self.entries.get(&(source, id)) {
             Some(entry) => entry,
             None => {
                 return Err(Error::IdNotFound);
@@ -130,7 +514,8 @@ impl Manager {
         assert!(ptr as u32 % 64 == 0);
 
         self.state = ManagerState::Playback;
-        self.currentid = id;
+        self.current_id = id;
+        self.current_source = source;
 
         unsafe {
             csr::rtio_dma::base_address_write(ptr as u32);
@@ -162,7 +547,8 @@ impl Manager {
                     csr::rtio_dma::error_write(1);
                 }
                 return Some(RtioStatus {
-                    id: self.currentid,
+                    source: self.current_source,
+                    id: self.current_id,
                     error: error,
                     channel: channel,
                     timestamp: timestamp,
@@ -170,4 +556,8 @@ impl Manager {
             }
         }
     }
+
+    pub fn running(&self) -> bool {
+        self.state == ManagerState::Playback
+    }
 }

src/satman/src/main.rs

@@ -1,13 +1,15 @@
 #![no_std]
 #![no_main]
-#![feature(never_type, panic_info_message, asm, naked_functions)]
-#![feature(alloc_error_handler)]
+#![feature(alloc_error_handler, try_trait, never_type, panic_info_message)]
 
 #[macro_use]
 extern crate log;
-
+extern crate core_io;
+extern crate cslice;
 extern crate embedded_hal;
 
+extern crate io;
+extern crate ksupport;
 extern crate libboard_artiq;
 extern crate libboard_zynq;
 extern crate libcortex_a9;
@@ -18,28 +20,40 @@ extern crate unwind;
 
 extern crate alloc;
 
-use core::sync::atomic::{AtomicBool, Ordering};
-
+use analyzer::Analyzer;
 use dma::Manager as DmaManager;
 use embedded_hal::blocking::delay::DelayUs;
+#[cfg(has_grabber)]
+use libboard_artiq::grabber;
 #[cfg(feature = "target_kasli_soc")]
 use libboard_artiq::io_expander;
 #[cfg(has_si5324)]
 use libboard_artiq::si5324;
-use libboard_artiq::{drtio_routing, drtioaux, identifier_read, logger, pl::csr};
+#[cfg(has_si549)]
+use libboard_artiq::si549;
+use libboard_artiq::{drtio_routing, drtioaux,
+                     drtioaux_proto::{MASTER_PAYLOAD_MAX_SIZE, SAT_PAYLOAD_MAX_SIZE},
+                     identifier_read, logger,
+                     pl::csr};
 #[cfg(feature = "target_kasli_soc")]
 use libboard_zynq::error_led::ErrorLED;
-use libboard_zynq::{gic, i2c::I2c, mpcore, print, println, stdio, time::Milliseconds, timer::GlobalTimer};
-use libcortex_a9::{asm, interrupt_handler,
-                   l2c::enable_l2_cache,
-                   notify_spin_lock,
-                   regs::{MPIDR, SP},
-                   spin_lock_yield};
-use libregister::{RegisterR, RegisterW};
-use libsupport_zynq::ram;
+use libboard_zynq::{i2c::I2c, print, println, time::Milliseconds, timer::GlobalTimer};
+use libcortex_a9::{l2c::enable_l2_cache, regs::MPIDR};
+use libregister::RegisterR;
+use libsupport_zynq::{exception_vectors, ram};
+use routing::Router;
+use subkernel::Manager as KernelManager;
 
+mod analyzer;
 mod dma;
 mod repeater;
+mod routing;
+mod subkernel;
+
+// linker symbols
+extern "C" {
+    static __exceptions_start: u32;
+}
 
 fn drtiosat_reset(reset: bool) {
     unsafe {
@@ -74,7 +88,11 @@ macro_rules! forward {
         if hop != 0 {
             let repno = (hop - 1) as usize;
             if repno < $repeaters.len() {
-                return $repeaters[repno].aux_forward($packet, $timer);
+                if $packet.expects_response() {
+                    return $repeaters[repno].aux_forward($packet, $timer);
+                } else {
+                    return $repeaters[repno].aux_send($packet);
+                }
             } else {
                 return Err(drtioaux::Error::RoutingError);
             }
@@ -90,11 +108,15 @@ macro_rules! forward {
 fn process_aux_packet(
     _repeaters: &mut [repeater::Repeater],
     _routing_table: &mut drtio_routing::RoutingTable,
-    _rank: &mut u8,
+    rank: &mut u8,
+    self_destination: &mut u8,
     packet: drtioaux::Packet,
     timer: &mut GlobalTimer,
     i2c: &mut I2c,
     dma_manager: &mut DmaManager,
+    analyzer: &mut Analyzer,
+    kernel_manager: &mut KernelManager,
+    router: &mut Router,
 ) -> Result<(), drtioaux::Error> {
     // In the code below, *_chan_sel_write takes an u8 if there are fewer than 256 channels,
     // and u16 otherwise; hence the `as _` conversion.
@@ -113,15 +135,14 @@ fn process_aux_packet(
             drtioaux::send(0, &drtioaux::Packet::ResetAck)
         }
 
-        drtioaux::Packet::DestinationStatusRequest {
-            destination: _destination,
-        } => {
+        drtioaux::Packet::DestinationStatusRequest { destination } => {
             #[cfg(has_drtio_routing)]
-            let hop = _routing_table.0[_destination as usize][*_rank as usize];
+            let hop = _routing_table.0[destination as usize][*rank as usize];
             #[cfg(not(has_drtio_routing))]
             let hop = 0;
 
             if hop == 0 {
+                *self_destination = destination;
                 let errors;
                 unsafe {
                     errors = csr::drtiosat::rtio_error_read();
@@ -160,7 +181,7 @@ fn process_aux_packet(
                         let repno = hop - 1;
                         match _repeaters[repno].aux_forward(
                             &drtioaux::Packet::DestinationStatusRequest {
-                                destination: _destination,
+                                destination: destination,
                             },
                             timer,
                         ) {
@@ -193,11 +214,11 @@ fn process_aux_packet(
             drtioaux::send(0, &drtioaux::Packet::RoutingAck)
         }
         #[cfg(has_drtio_routing)]
-        drtioaux::Packet::RoutingSetRank { rank } => {
-            *_rank = rank;
-            drtio_routing::interconnect_enable_all(_routing_table, rank);
+        drtioaux::Packet::RoutingSetRank { rank: new_rank } => {
+            *rank = new_rank;
+            drtio_routing::interconnect_enable_all(_routing_table, new_rank);
 
-            let rep_rank = rank + 1;
+            let rep_rank = new_rank + 1;
             for rep in _repeaters.iter() {
                 if let Err(e) = rep.set_rank(rep_rank, timer) {
                     error!("failed to set rank ({:?})", e);
@@ -220,10 +241,10 @@ fn process_aux_packet(
 
         drtioaux::Packet::MonitorRequest {
             destination: _destination,
-            channel: _channel,
-            probe: _probe,
+            channel,
+            probe,
         } => {
-            forward!(_routing_table, _destination, *_rank, _repeaters, &packet, timer);
+            forward!(_routing_table, _destination, *rank, _repeaters, &packet, timer);
             let value;
             #[cfg(has_rtio_moninj)]
             unsafe {
@@ -241,11 +262,11 @@ fn process_aux_packet(
         }
         drtioaux::Packet::InjectionRequest {
             destination: _destination,
-            channel: _channel,
-            overrd: _overrd,
-            value: _value,
+            channel,
+            overrd,
+            value,
         } => {
-            forward!(_routing_table, _destination, *_rank, _repeaters, &packet, timer);
+            forward!(_routing_table, _destination, *rank, _repeaters, &packet, timer);
             #[cfg(has_rtio_moninj)]
             unsafe {
                 csr::rtio_moninj::inj_chan_sel_write(channel as _);
@@ -256,10 +277,10 @@ fn process_aux_packet(
         }
         drtioaux::Packet::InjectionStatusRequest {
             destination: _destination,
-            channel: _channel,
-            overrd: _overrd,
+            channel,
+            overrd,
         } => {
-            forward!(_routing_table, _destination, *_rank, _repeaters, &packet, timer);
+            forward!(_routing_table, _destination, *rank, _repeaters, &packet, timer);
             let value;
             #[cfg(has_rtio_moninj)]
             unsafe {
@@ -278,7 +299,7 @@ fn process_aux_packet(
             destination: _destination,
             busno: _busno,
         } => {
-            forward!(_routing_table, _destination, *_rank, _repeaters, &packet, timer);
+            forward!(_routing_table, _destination, *rank, _repeaters, &packet, timer);
             let succeeded = i2c.start().is_ok();
             drtioaux::send(0, &drtioaux::Packet::I2cBasicReply { succeeded: succeeded })
         }
@@ -286,7 +307,7 @@ fn process_aux_packet(
             destination: _destination,
             busno: _busno,
         } => {
-            forward!(_routing_table, _destination, *_rank, _repeaters, &packet, timer);
+            forward!(_routing_table, _destination, *rank, _repeaters, &packet, timer);
             let succeeded = i2c.restart().is_ok();
             drtioaux::send(0, &drtioaux::Packet::I2cBasicReply { succeeded: succeeded })
         }
@@ -294,7 +315,7 @@ fn process_aux_packet(
             destination: _destination,
             busno: _busno,
         } => {
-            forward!(_routing_table, _destination, *_rank, _repeaters, &packet, timer);
+            forward!(_routing_table, _destination, *rank, _repeaters, &packet, timer);
             let succeeded = i2c.stop().is_ok();
             drtioaux::send(0, &drtioaux::Packet::I2cBasicReply { succeeded: succeeded })
         }
@@ -303,7 +324,7 @@ fn process_aux_packet(
             busno: _busno,
             data,
         } => {
-            forward!(_routing_table, _destination, *_rank, _repeaters, &packet, timer);
+            forward!(_routing_table, _destination, *rank, _repeaters, &packet, timer);
             match i2c.write(data) {
                 Ok(ack) => drtioaux::send(
                     0,
@@ -326,7 +347,7 @@ fn process_aux_packet(
             busno: _busno,
             ack,
         } => {
-            forward!(_routing_table, _destination, *_rank, _repeaters, &packet, timer);
+            forward!(_routing_table, _destination, *rank, _repeaters, &packet, timer);
             match i2c.read(ack) {
                 Ok(data) => drtioaux::send(
                     0,
@@ -350,7 +371,7 @@ fn process_aux_packet(
             address,
             mask,
         } => {
-            forward!(_routing_table, _destination, *_rank, _repeaters, &packet, timer);
+            forward!(_routing_table, _destination, *rank, _repeaters, &packet, timer);
             let ch = match mask {
                 //decode from mainline, PCA9548-centric API
                 0x00 => None,
@@ -376,7 +397,7 @@ fn process_aux_packet(
             div: _div,
             cs: _cs,
         } => {
-            forward!(_routing_table, _destination, *_rank, _repeaters, &packet, timer);
+            forward!(_routing_table, _destination, *rank, _repeaters, &packet, timer);
             // todo: reimplement when/if SPI is available
             //let succeeded = spi::set_config(busno, flags, length, div, cs).is_ok();
             drtioaux::send(0, &drtioaux::Packet::SpiBasicReply { succeeded: false })
@@ -386,7 +407,7 @@ fn process_aux_packet(
             busno: _busno,
             data: _data,
         } => {
-            forward!(_routing_table, _destination, *_rank, _repeaters, &packet, timer);
+            forward!(_routing_table, _destination, *rank, _repeaters, &packet, timer);
             // todo: reimplement when/if SPI is available
             //let succeeded = spi::write(busno, data).is_ok();
             drtioaux::send(0, &drtioaux::Packet::SpiBasicReply { succeeded: false })
@@ -395,7 +416,7 @@ fn process_aux_packet(
             destination: _destination,
             busno: _busno,
         } => {
-            forward!(_routing_table, _destination, *_rank, _repeaters, &packet, timer);
+            forward!(_routing_table, _destination, *rank, _repeaters, &packet, timer);
             // todo: reimplement when/if SPI is available
             // match spi::read(busno) {
             //     Ok(data) => drtioaux::send(0,
@@ -412,37 +433,267 @@ fn process_aux_packet(
             )
         }
 
-        drtioaux::Packet::DmaAddTraceRequest {
+        drtioaux::Packet::AnalyzerHeaderRequest {
             destination: _destination,
+        } => {
+            forward!(_routing_table, _destination, *rank, _repeaters, &packet, timer);
+            let header = analyzer.get_header();
+            drtioaux::send(
+                0,
+                &drtioaux::Packet::AnalyzerHeader {
+                    total_byte_count: header.total_byte_count,
+                    sent_bytes: header.sent_bytes,
+                    overflow_occurred: header.error,
+                },
+            )
+        }
+        drtioaux::Packet::AnalyzerDataRequest {
+            destination: _destination,
+        } => {
+            forward!(_routing_table, _destination, *rank, _repeaters, &packet, timer);
+            let mut data_slice: [u8; SAT_PAYLOAD_MAX_SIZE] = [0; SAT_PAYLOAD_MAX_SIZE];
+            let meta = analyzer.get_data(&mut data_slice);
+            drtioaux::send(
+                0,
+                &drtioaux::Packet::AnalyzerData {
+                    last: meta.last,
+                    length: meta.len,
+                    data: data_slice,
+                },
+            )
+        }
+
+        drtioaux::Packet::DmaAddTraceRequest {
+            source,
+            destination,
             id,
-            last,
+            status,
             length,
             trace,
         } => {
-            forward!(_routing_table, _destination, *_rank, _repeaters, &packet, timer);
-            let succeeded = dma_manager.add(id, last, &trace, length as usize).is_ok();
-            drtioaux::send(0, &drtioaux::Packet::DmaAddTraceReply { succeeded: succeeded })
+            forward!(_routing_table, destination, *rank, _repeaters, &packet, timer);
+            *self_destination = destination;
+            let succeeded = dma_manager.add(source, id, status, &trace, length as usize).is_ok();
+            router.send(
+                drtioaux::Packet::DmaAddTraceReply {
+                    source: *self_destination,
+                    destination: source,
+                    id: id,
+                    succeeded: succeeded,
+                },
+                _routing_table,
+                *rank,
+                *self_destination,
+            )
+        }
+        drtioaux::Packet::DmaAddTraceReply {
+            source,
+            destination: _destination,
+            id,
+            succeeded,
+        } => {
+            forward!(_routing_table, _destination, *rank, _repeaters, &packet, timer);
+            dma_manager.ack_upload(
+                kernel_manager,
+                source,
+                id,
+                succeeded,
+                router,
+                *rank,
+                *self_destination,
+                _routing_table,
+            );
+            Ok(())
         }
         drtioaux::Packet::DmaRemoveTraceRequest {
+            source,
             destination: _destination,
             id,
         } => {
-            forward!(_routing_table, _destination, *_rank, _repeaters, &packet, timer);
-            let succeeded = dma_manager.erase(id).is_ok();
-            drtioaux::send(0, &drtioaux::Packet::DmaRemoveTraceReply { succeeded: succeeded })
+            forward!(_routing_table, _destination, *rank, _repeaters, &packet, timer);
+            let succeeded = dma_manager.erase(source, id).is_ok();
+            router.send(
+                drtioaux::Packet::DmaRemoveTraceReply {
+                    destination: source,
+                    succeeded: succeeded,
+                },
+                _routing_table,
+                *rank,
+                *self_destination,
+            )
+        }
+        drtioaux::Packet::DmaRemoveTraceReply {
+            destination: _destination,
+            succeeded: _,
+        } => {
+            forward!(_routing_table, _destination, *rank, _repeaters, &packet, timer);
+            Ok(())
         }
         drtioaux::Packet::DmaPlaybackRequest {
+            source,
             destination: _destination,
             id,
             timestamp,
         } => {
-            forward!(_routing_table, _destination, *_rank, _repeaters, &packet, timer);
-            let succeeded = dma_manager.playback(id, timestamp).is_ok();
-            drtioaux::send(0, &drtioaux::Packet::DmaPlaybackReply { succeeded: succeeded })
+            forward!(_routing_table, _destination, *rank, _repeaters, &packet, timer);
+            let succeeded = if !kernel_manager.running() {
+                dma_manager.playback(source, id, timestamp).is_ok()
+            } else {
+                false
+            };
+            router.send(
+                drtioaux::Packet::DmaPlaybackReply {
+                    destination: source,
+                    succeeded: succeeded,
+                },
+                _routing_table,
+                *rank,
+                *self_destination,
+            )
+        }
+        drtioaux::Packet::DmaPlaybackReply {
+            destination: _destination,
+            succeeded,
+        } => {
+            forward!(_routing_table, _destination, *rank, _repeaters, &packet, timer);
+            if !succeeded {
+                kernel_manager.ddma_nack();
+            }
+            Ok(())
+        }
+        drtioaux::Packet::DmaPlaybackStatus {
+            source: _,
+            destination: _destination,
+            id,
+            error,
+            channel,
+            timestamp,
+        } => {
+            forward!(_routing_table, _destination, *rank, _repeaters, &packet, timer);
+            dma_manager.remote_finished(kernel_manager, id, error, channel, timestamp);
+            Ok(())
         }
 
-        _ => {
-            warn!("received unexpected aux packet");
+        drtioaux::Packet::SubkernelAddDataRequest {
+            destination,
+            id,
+            status,
+            length,
+            data,
+        } => {
+            forward!(_routing_table, destination, *rank, _repeaters, &packet, timer);
+            *self_destination = destination;
+            let succeeded = kernel_manager.add(id, status, &data, length as usize).is_ok();
+            drtioaux::send(0, &drtioaux::Packet::SubkernelAddDataReply { succeeded: succeeded })
+        }
+        drtioaux::Packet::SubkernelLoadRunRequest {
+            source,
+            destination: _destination,
+            id,
+            run,
+        } => {
+            forward!(_routing_table, _destination, *rank, _repeaters, &packet, timer);
+            let mut succeeded = kernel_manager.load(id).is_ok();
+            // allow preloading a kernel with delayed run
+            if run {
+                if dma_manager.running() {
+                    // cannot run kernel while DDMA is running
+                    succeeded = false;
+                } else {
+                    succeeded |= kernel_manager.run(source, id).is_ok();
+                }
+            }
+            router.send(
+                drtioaux::Packet::SubkernelLoadRunReply {
+                    destination: source,
+                    succeeded: succeeded,
+                },
+                _routing_table,
+                *rank,
+                *self_destination,
+            )
+        }
+        drtioaux::Packet::SubkernelLoadRunReply {
+            destination: _destination,
+            succeeded,
+        } => {
+            forward!(_routing_table, _destination, *rank, _repeaters, &packet, timer);
+            // received if local subkernel started another, remote subkernel
+            kernel_manager.subkernel_load_run_reply(succeeded);
+            Ok(())
+        }
+        drtioaux::Packet::SubkernelFinished {
+            destination: _destination,
+            id,
+            with_exception,
+            exception_src,
+        } => {
+            forward!(_routing_table, _destination, *rank, _repeaters, &packet, timer);
+            kernel_manager.remote_subkernel_finished(id, with_exception, exception_src);
+            Ok(())
+        }
+        drtioaux::Packet::SubkernelExceptionRequest {
+            destination: _destination,
+        } => {
+            forward!(_routing_table, _destination, *rank, _repeaters, &packet, timer);
+            let mut data_slice: [u8; SAT_PAYLOAD_MAX_SIZE] = [0; SAT_PAYLOAD_MAX_SIZE];
+            let meta = kernel_manager.exception_get_slice(&mut data_slice);
+            drtioaux::send(
+                0,
+                &drtioaux::Packet::SubkernelException {
+                    last: meta.status.is_last(),
+                    length: meta.len,
+                    data: data_slice,
+                },
+            )
+        }
+        drtioaux::Packet::SubkernelMessage {
+            source,
+            destination: _destination,
+            id,
+            status,
+            length,
+            data,
+        } => {
+            forward!(_routing_table, _destination, *rank, _repeaters, &packet, timer);
+            kernel_manager.message_handle_incoming(status, id, length as usize, &data);
+            router.send(
+                drtioaux::Packet::SubkernelMessageAck { destination: source },
+                _routing_table,
+                *rank,
+                *self_destination,
+            )
+        }
+        drtioaux::Packet::SubkernelMessageAck {
+            destination: _destination,
+        } => {
+            forward!(_routing_table, _destination, *rank, _repeaters, &packet, timer);
+            if kernel_manager.message_ack_slice() {
+                let mut data_slice: [u8; MASTER_PAYLOAD_MAX_SIZE] = [0; MASTER_PAYLOAD_MAX_SIZE];
+                if let Some(meta) = kernel_manager.message_get_slice(&mut data_slice) {
+                    // route and not send immediately as ACKs are not a beginning of a transaction
+                    router.route(
+                        drtioaux::Packet::SubkernelMessage {
+                            source: *self_destination,
+                            destination: meta.destination,
+                            id: kernel_manager.get_current_id().unwrap(),
+                            status: meta.status,
+                            length: meta.len as u16,
+                            data: data_slice,
+                        },
+                        _routing_table,
+                        *rank,
+                        *self_destination,
+                    );
+                } else {
+                    error!("Error receiving message slice");
+                }
+            }
+            Ok(())
+        }
+
+        p => {
+            warn!("received unexpected aux packet: {:?}", p);
             Ok(())
         }
     }
@@ -452,20 +703,35 @@ fn process_aux_packets(
     repeaters: &mut [repeater::Repeater],
     routing_table: &mut drtio_routing::RoutingTable,
     rank: &mut u8,
+    self_destination: &mut u8,
     timer: &mut GlobalTimer,
     i2c: &mut I2c,
     dma_manager: &mut DmaManager,
+    analyzer: &mut Analyzer,
+    kernel_manager: &mut KernelManager,
+    router: &mut Router,
 ) {
     let result = drtioaux::recv(0).and_then(|packet| {
-        if let Some(packet) = packet {
-            process_aux_packet(repeaters, routing_table, rank, packet, timer, i2c, dma_manager)
+        if let Some(packet) = packet.or_else(|| router.get_local_packet()) {
+            process_aux_packet(
+                repeaters,
+                routing_table,
+                rank,
+                self_destination,
+                packet,
+                timer,
+                i2c,
+                dma_manager,
+                analyzer,
+                kernel_manager,
+                router,
+            )
         } else {
             Ok(())
         }
     });
-    match result {
-        Ok(()) => (),
-        Err(e) => warn!("aux packet error ({:?})", e),
+    if let Err(e) = result {
+        warn!("aux packet error ({:?})", e);
     }
 }
 
@@ -521,6 +787,8 @@ fn hardware_tick(ts: &mut u64, timer: &mut GlobalTimer) {
     if now > ts_ms {
         ts_ms = now + Milliseconds(200);
         *ts = ts_ms.0;
+        #[cfg(has_grabber)]
+        grabber::tick();
     }
 }
 
@@ -548,10 +816,43 @@ const SI5324_SETTINGS: si5324::FrequencySettings = si5324::FrequencySettings {
     crystal_as_ckin2: true,
 };
 
+#[cfg(all(has_si549, rtio_frequency = "125.0"))]
+const SI549_SETTINGS: si549::FrequencySetting = si549::FrequencySetting {
+    main: si549::DividerConfig {
+        hsdiv: 0x058,
+        lsdiv: 0,
+        fbdiv: 0x04815791F25,
+    },
+    helper: si549::DividerConfig {
+        // 125MHz*32767/32768
+        hsdiv: 0x058,
+        lsdiv: 0,
+        fbdiv: 0x04814E8F442,
+    },
+};
+
+#[cfg(all(has_si549, rtio_frequency = "100.0"))]
+const SI549_SETTINGS: si549::FrequencySetting = si549::FrequencySetting {
+    main: si549::DividerConfig {
+        hsdiv: 0x06C,
+        lsdiv: 0,
+        fbdiv: 0x046C5F49797,
+    },
+    helper: si549::DividerConfig {
+        // 100MHz*32767/32768
+        hsdiv: 0x06C,
+        lsdiv: 0,
+        fbdiv: 0x046C5670BBD,
+    },
+};
+
 static mut LOG_BUFFER: [u8; 1 << 17] = [0; 1 << 17];
 
 #[no_mangle]
 pub extern "C" fn main_core0() -> i32 {
+    unsafe {
+        exception_vectors::set_vector_table(&__exceptions_start as *const u32 as u32);
+    }
     enable_l2_cache(0x8);
 
     let mut timer = GlobalTimer::start();
@@ -566,31 +867,46 @@ pub extern "C" fn main_core0() -> i32 {
 
     ram::init_alloc_core0();
 
-    let mut i2c = I2c::i2c0();
-    i2c.init().expect("I2C initialization failed");
+    ksupport::i2c::init();
+    let mut i2c = unsafe { (ksupport::i2c::I2C_BUS).as_mut().unwrap() };
+
+    #[cfg(feature = "target_kasli_soc")]
+    let (mut io_expander0, mut io_expander1);
     #[cfg(feature = "target_kasli_soc")]
     {
-        for expander_i in 0..=1 {
-            let mut io_expander = io_expander::IoExpander::new(&mut i2c, expander_i).unwrap();
-            io_expander.init().expect("I2C I/O expander #0 initialization failed");
-            // Actively drive TX_DISABLE to false on SFP0..3
-            io_expander.set_oe(0, 1 << 1).unwrap();
-            io_expander.set_oe(1, 1 << 1).unwrap();
-            io_expander.set(0, 1, false);
-            io_expander.set(1, 1, false);
-            io_expander.service().unwrap();
-        }
+        io_expander0 = io_expander::IoExpander::new(&mut i2c, 0).unwrap();
+        io_expander1 = io_expander::IoExpander::new(&mut i2c, 1).unwrap();
+        io_expander0
+            .init(&mut i2c)
+            .expect("I2C I/O expander #0 initialization failed");
+        io_expander1
+            .init(&mut i2c)
+            .expect("I2C I/O expander #1 initialization failed");
+
+        // Drive CLK_SEL to true
+        #[cfg(has_si549)]
+        io_expander0.set(1, 7, true);
+
+        // Drive TX_DISABLE to false on SFP0..3
+        io_expander0.set(0, 1, false);
+        io_expander1.set(0, 1, false);
+        io_expander0.set(1, 1, false);
+        io_expander1.set(1, 1, false);
+        io_expander0.service(&mut i2c).unwrap();
+        io_expander1.service(&mut i2c).unwrap();
     }
 
     #[cfg(has_si5324)]
     si5324::setup(&mut i2c, &SI5324_SETTINGS, si5324::Input::Ckin1, &mut timer).expect("cannot initialize Si5324");
+    #[cfg(has_si549)]
+    si549::main_setup(&mut timer, &SI549_SETTINGS).expect("cannot initialize main Si549");
 
     timer.delay_us(100_000);
     info!("Switching SYS clocks...");
     unsafe {
-        csr::drtio_transceiver::stable_clkin_write(1);
+        csr::gt_drtio::stable_clkin_write(1);
     }
-    timer.delay_us(20_000); // wait for CPLL/QPLL/MMCM lock
+    timer.delay_us(50_000); // wait for CPLL/QPLL/MMCM lock
     let clk = unsafe { csr::sys_crg::current_clock_read() };
     if clk == 1 {
         info!("SYS CLK switched successfully");
@@ -599,8 +915,10 @@ pub extern "C" fn main_core0() -> i32 {
     }
 
     unsafe {
-        csr::drtio_transceiver::txenable_write(0xffffffffu32 as _);
+        csr::gt_drtio::txenable_write(0xffffffffu32 as _);
     }
+    #[cfg(has_si549)]
+    si549::helper_setup(&mut timer, &SI549_SETTINGS).expect("cannot initialize helper Si549");
 
     #[cfg(has_drtio_routing)]
     let mut repeaters = [repeater::Repeater::default(); csr::DRTIOREP.len()];
@@ -611,16 +929,31 @@ pub extern "C" fn main_core0() -> i32 {
     }
     let mut routing_table = drtio_routing::RoutingTable::default_empty();
     let mut rank = 1;
+    let mut destination = 1;
 
     let mut hardware_tick_ts = 0;
 
+    let mut control = ksupport::kernel::Control::start();
+
     loop {
+        let mut router = Router::new();
+
         while !drtiosat_link_rx_up() {
             drtiosat_process_errors();
             #[allow(unused_mut)]
             for mut rep in repeaters.iter_mut() {
-                rep.service(&routing_table, rank, &mut timer);
+                rep.service(&routing_table, rank, destination, &mut router, &mut timer);
             }
+            #[cfg(feature = "target_kasli_soc")]
+            {
+                io_expander0
+                    .service(&mut i2c)
+                    .expect("I2C I/O expander #0 service failed");
+                io_expander1
+                    .service(&mut i2c)
+                    .expect("I2C I/O expander #1 service failed");
+            }
+
             hardware_tick(&mut hardware_tick_ts, &mut timer);
         }
 
@@ -631,10 +964,15 @@ pub extern "C" fn main_core0() -> i32 {
             si5324::siphaser::calibrate_skew(&mut timer).expect("failed to calibrate skew");
         }
 
-        // DMA manager created here, so when link is dropped, all DMA traces
+        #[cfg(has_wrpll)]
+        si549::wrpll::select_recovered_clock(true, &mut timer);
+
+        // Various managers created here, so when link is dropped, all DMA traces
         // are cleared out for a clean slate on subsequent connections,
         // without a manual intervention.
         let mut dma_manager = DmaManager::new();
+        let mut analyzer = Analyzer::new();
+        let mut kernel_manager = KernelManager::new(&mut control);
 
         drtioaux::reset(0);
         drtiosat_reset(false);
@@ -646,13 +984,26 @@ pub extern "C" fn main_core0() -> i32 {
                 &mut repeaters,
                 &mut routing_table,
                 &mut rank,
+                &mut destination,
                 &mut timer,
                 &mut i2c,
                 &mut dma_manager,
+                &mut analyzer,
+                &mut kernel_manager,
+                &mut router,
             );
             #[allow(unused_mut)]
             for mut rep in repeaters.iter_mut() {
-                rep.service(&routing_table, rank, &mut timer);
+                rep.service(&routing_table, rank, destination, &mut router, &mut timer);
+            }
+            #[cfg(feature = "target_kasli_soc")]
+            {
+                io_expander0
+                    .service(&mut i2c)
+                    .expect("I2C I/O expander #0 service failed");
+                io_expander1
+                    .service(&mut i2c)
+                    .expect("I2C I/O expander #1 service failed");
             }
             hardware_tick(&mut hardware_tick_ts, &mut timer);
             if drtiosat_tsc_loaded() {
@@ -671,19 +1022,40 @@ pub extern "C" fn main_core0() -> i32 {
                     "playback done, error: {}, channel: {}, timestamp: {}",
                     status.error, status.channel, status.timestamp
                 );
-                if let Err(e) = drtioaux::send(
-                    0,
-                    &drtioaux::Packet::DmaPlaybackStatus {
-                        destination: rank,
+                router.route(
+                    drtioaux::Packet::DmaPlaybackStatus {
+                        source: destination,
+                        destination: status.source,
                         id: status.id,
                         error: status.error,
                         channel: status.channel,
                         timestamp: status.timestamp,
                     },
-                ) {
-                    error!("error sending DMA playback status: {:?}", e);
+                    &routing_table,
+                    rank,
+                    destination,
+                );
+            }
+
+            kernel_manager.process_kern_requests(
+                &mut router,
+                &routing_table,
+                rank,
+                destination,
+                &mut dma_manager,
+                &timer,
+            );
+
+            #[cfg(has_drtio_routing)]
+            if let Some((repno, packet)) = router.get_downstream_packet() {
+                if let Err(e) = repeaters[repno].aux_send(&packet) {
+                    warn!("[REP#{}] Error when sending packet to satellite ({:?})", repno, e)
                 }
             }
+
+            if let Some(packet) = router.get_upstream_packet() {
+                drtioaux::send(0, &packet).unwrap();
+            }
         }
 
         drtiosat_reset_phy(true);
@@ -692,6 +1064,8 @@ pub extern "C" fn main_core0() -> i32 {
         info!("uplink is down, switching to local oscillator clock");
         #[cfg(has_siphaser)]
         si5324::siphaser::select_recovered_clock(&mut i2c, false, &mut timer).expect("failed to switch clocks");
+        #[cfg(has_wrpll)]
+        si549::wrpll::select_recovered_clock(false, &mut timer);
     }
 }
 
@@ -699,46 +1073,8 @@ extern "C" {
     static mut __stack1_start: u32;
 }
 
-interrupt_handler!(IRQ, irq, __irq_stack0_start, __irq_stack1_start, {
-    if MPIDR.read().cpu_id() == 1 {
-        let mpcore = mpcore::RegisterBlock::mpcore();
-        let mut gic = gic::InterruptController::gic(mpcore);
-        let id = gic.get_interrupt_id();
-        if id.0 == 0 {
-            gic.end_interrupt(id);
-            asm::exit_irq();
-            SP.write(&mut __stack1_start as *mut _ as u32);
-            asm::enable_irq();
-            CORE1_RESTART.store(false, Ordering::Relaxed);
-            notify_spin_lock();
-            main_core1();
-        }
-        stdio::drop_uart();
-    }
-    loop {}
-});
-
 static mut PANICKED: [bool; 2] = [false; 2];
 
-static CORE1_RESTART: AtomicBool = AtomicBool::new(false);
-
-pub fn restart_core1() {
-    let mut interrupt_controller = gic::InterruptController::gic(mpcore::RegisterBlock::mpcore());
-    CORE1_RESTART.store(true, Ordering::Relaxed);
-    interrupt_controller.send_sgi(gic::InterruptId(0), gic::CPUCore::Core1.into());
-    while CORE1_RESTART.load(Ordering::Relaxed) {
-        spin_lock_yield();
-    }
-}
-
-#[no_mangle]
-pub fn main_core1() {
-    let mut interrupt_controller = gic::InterruptController::gic(mpcore::RegisterBlock::mpcore());
-    interrupt_controller.enable_interrupts();
-
-    loop {}
-}
-
 #[no_mangle]
 pub extern "C" fn exception(_vect: u32, _regs: *const u32, pc: u32, ea: u32) {
     fn hexdump(addr: u32) {
@@ -794,23 +1130,3 @@ pub fn panic_fmt(info: &core::panic::PanicInfo) -> ! {
 
     loop {}
 }
-
-// linker symbols
-extern "C" {
-    static __text_start: u32;
-    static __text_end: u32;
-    static __exidx_start: u32;
-    static __exidx_end: u32;
-}
-
-#[no_mangle]
-extern "C" fn dl_unwind_find_exidx(_pc: *const u32, len_ptr: *mut u32) -> *const u32 {
-    let length;
-    let start: *const u32;
-    unsafe {
-        length = (&__exidx_end as *const u32).offset_from(&__exidx_start) as u32;
-        start = &__exidx_start;
-        *len_ptr = length;
-    }
-    start
-}

src/satman/src/repeater.rs

@@ -6,6 +6,7 @@ use libboard_artiq::{drtio_routing, drtioaux};
 #[cfg(has_drtio_routing)]
 use libboard_zynq::time::Milliseconds;
 use libboard_zynq::timer::GlobalTimer;
+use routing::Router;
 
 #[cfg(has_drtio_routing)]
 fn rep_link_rx_up(repno: u8) -> bool {
@@ -53,7 +54,14 @@ impl Repeater {
         self.state == RepeaterState::Up
     }
 
-    pub fn service(&mut self, routing_table: &drtio_routing::RoutingTable, rank: u8, timer: &mut GlobalTimer) {
+    pub fn service(
+        &mut self,
+        routing_table: &drtio_routing::RoutingTable,
+        rank: u8,
+        destination: u8,
+        router: &mut Router,
+        timer: &mut GlobalTimer,
+    ) {
         self.process_local_errors();
 
         match self.state {
@@ -111,7 +119,7 @@ impl Repeater {
                 }
             }
             RepeaterState::Up => {
-                self.process_unsolicited_aux();
+                self.process_unsolicited_aux(routing_table, rank, destination, router);
                 if !rep_link_rx_up(self.repno) {
                     info!("[REP#{}] link is down", self.repno);
                     self.state = RepeaterState::Down;
@@ -126,9 +134,15 @@ impl Repeater {
         }
     }
 
-    fn process_unsolicited_aux(&self) {
+    fn process_unsolicited_aux(
+        &self,
+        routing_table: &drtio_routing::RoutingTable,
+        rank: u8,
+        destination: u8,
+        router: &mut Router,
+    ) {
         match drtioaux::recv(self.auxno) {
-            Ok(Some(packet)) => warn!("[REP#{}] unsolicited aux packet: {:?}", self.repno, packet),
+            Ok(Some(packet)) => router.route(packet, routing_table, rank, destination),
             Ok(None) => (),
             Err(_) => warn!("[REP#{}] aux packet error", self.repno),
         }
@@ -191,15 +205,19 @@ impl Repeater {
     }
 
     pub fn aux_forward(&self, request: &drtioaux::Packet, timer: &mut GlobalTimer) -> Result<(), drtioaux::Error> {
-        if self.state != RepeaterState::Up {
-            return Err(drtioaux::Error::LinkDown);
-        }
-        drtioaux::send(self.auxno, request).unwrap();
+        self.aux_send(request)?;
         let reply = self.recv_aux_timeout(200, timer)?;
         drtioaux::send(0, &reply).unwrap();
         Ok(())
     }
 
+    pub fn aux_send(&self, request: &drtioaux::Packet) -> Result<(), drtioaux::Error> {
+        if self.state != RepeaterState::Up {
+            return Err(drtioaux::Error::LinkDown);
+        }
+        drtioaux::send(self.auxno, request)
+    }
+
     pub fn sync_tsc(&self, timer: &mut GlobalTimer) -> Result<(), drtioaux::Error> {
         if self.state != RepeaterState::Up {
             return Ok(());
@@ -302,7 +320,15 @@ impl Repeater {
         Repeater::default()
     }
 
-    pub fn service(&self, _routing_table: &drtio_routing::RoutingTable, _rank: u8, _timer: &mut GlobalTimer) {}
+    pub fn service(
+        &self,
+        _routing_table: &drtio_routing::RoutingTable,
+        _rank: u8,
+        _destination: u8,
+        _router: &mut Router,
+        _timer: &mut GlobalTimer,
+    ) {
+    }
 
     pub fn sync_tsc(&self, _timer: &mut GlobalTimer) -> Result<(), drtioaux::Error> {
         Ok(())

src/satman/src/routing.rs

@@ -0,0 +1,174 @@
+use alloc::{collections::vec_deque::VecDeque, vec::Vec};
+use core::cmp::min;
+
+#[cfg(has_drtio_routing)]
+use libboard_artiq::pl::csr;
+use libboard_artiq::{drtio_routing, drtioaux,
+                     drtioaux_proto::{PayloadStatus, MASTER_PAYLOAD_MAX_SIZE, SAT_PAYLOAD_MAX_SIZE}};
+
+pub struct SliceMeta {
+    pub destination: u8,
+    pub len: u16,
+    pub status: PayloadStatus,
+}
+
+/* represents data that has to be sent to Master */
+#[derive(Debug)]
+pub struct Sliceable {
+    it: usize,
+    data: Vec<u8>,
+    destination: u8,
+}
+
+macro_rules! get_slice_fn {
+    ($name:tt, $size:expr) => {
+        pub fn $name(&mut self, data_slice: &mut [u8; $size]) -> SliceMeta {
+            let first = self.it == 0;
+            let len = min($size, self.data.len() - self.it);
+            let last = self.it + len == self.data.len();
+            let status = PayloadStatus::from_status(first, last);
+
+            data_slice[..len].clone_from_slice(&self.data[self.it..self.it + len]);
+            self.it += len;
+
+            SliceMeta {
+                destination: self.destination,
+                len: len as u16,
+                status: status,
+            }
+        }
+    };
+}
+
+impl Sliceable {
+    pub fn new(destination: u8, data: Vec<u8>) -> Sliceable {
+        Sliceable {
+            it: 0,
+            data: data,
+            destination: destination,
+        }
+    }
+
+    pub fn at_end(&self) -> bool {
+        self.it == self.data.len()
+    }
+
+    pub fn extend(&mut self, data: &[u8]) {
+        self.data.extend(data);
+    }
+
+    get_slice_fn!(get_slice_sat, SAT_PAYLOAD_MAX_SIZE);
+    get_slice_fn!(get_slice_master, MASTER_PAYLOAD_MAX_SIZE);
+}
+
+// Packets from downstream (further satellites) are received and routed appropriately.
+// they're passed as soon as possible downstream (within the subtree), or sent upstream,
+// which is notified about pending packets.
+// for rank 1 (connected to master) satellites, these packets are passed as an answer to DestinationStatusRequest;
+// for higher ranks, after getting a notification, it will transact with downstream to get the pending packets.
+
+// forward! macro is not deprecated, as routable packets are only these that can originate
+// from both master and satellite, e.g. DDMA and Subkernel.
+
+pub struct Router {
+    upstream_queue: VecDeque<drtioaux::Packet>,
+    local_queue: VecDeque<drtioaux::Packet>,
+    #[cfg(has_drtio_routing)]
+    downstream_queue: VecDeque<(usize, drtioaux::Packet)>,
+}
+
+impl Router {
+    pub fn new() -> Router {
+        Router {
+            upstream_queue: VecDeque::new(),
+            local_queue: VecDeque::new(),
+            #[cfg(has_drtio_routing)]
+            downstream_queue: VecDeque::new(),
+        }
+    }
+
+    // Called by local sources (DDMA, kernel) and by repeaters on receiving async data;
+    // messages are always buffered for both upstream and downstream
+    pub fn route(
+        &mut self,
+        packet: drtioaux::Packet,
+        _routing_table: &drtio_routing::RoutingTable,
+        _rank: u8,
+        self_destination: u8,
+    ) {
+        let destination = packet.routable_destination();
+        #[cfg(has_drtio_routing)]
+        {
+            if let Some(destination) = destination {
+                let hop = _routing_table.0[destination as usize][_rank as usize] as usize;
+                if destination == self_destination {
+                    self.local_queue.push_back(packet);
+                } else if hop > 0 && hop < csr::DRTIOREP.len() {
+                    let repno = (hop - 1) as usize;
+                    self.downstream_queue.push_back((repno, packet));
+                } else {
+                    self.upstream_queue.push_back(packet);
+                }
+            } else {
+                error!("Received an unroutable packet: {:?}", packet);
+            }
+        }
+        #[cfg(not(has_drtio_routing))]
+        {
+            if destination == Some(self_destination) {
+                self.local_queue.push_back(packet);
+            } else {
+                self.upstream_queue.push_back(packet);
+            }
+        }
+    }
+
+    // Sends a packet to a required destination, routing if necessary
+    pub fn send(
+        &mut self,
+        packet: drtioaux::Packet,
+        _routing_table: &drtio_routing::RoutingTable,
+        _rank: u8,
+        _destination: u8,
+    ) -> Result<(), drtioaux::Error> {
+        #[cfg(has_drtio_routing)]
+        {
+            let destination = packet.routable_destination();
+            if let Some(destination) = destination {
+                let hop = _routing_table.0[destination as usize][_rank as usize] as usize;
+                if destination == 0 {
+                    // response is needed immediately if master required it
+                    drtioaux::send(0, &packet)?;
+                } else if !(hop > 0 && hop < csr::DRTIOREP.len()) {
+                    // higher rank can wait
+                    self.upstream_queue.push_back(packet);
+                } else {
+                    let repno = (hop - 1) as usize;
+                    // transaction will occur at closest possible opportunity
+                    self.downstream_queue.push_back((repno, packet));
+                }
+                Ok(())
+            } else {
+                // packet not supported in routing, fallback - sent directly
+                drtioaux::send(0, &packet)
+            }
+        }
+        #[cfg(not(has_drtio_routing))]
+        {
+            drtioaux::send(0, &packet)
+        }
+    }
+
+    pub fn get_upstream_packet(&mut self) -> Option<drtioaux::Packet> {
+        self.upstream_queue.pop_front()
+    }
+
+    #[cfg(has_drtio_routing)]
+    pub fn get_downstream_packet(&mut self) -> Option<(usize, drtioaux::Packet)> {
+        self.downstream_queue.pop_front()
+    }
+
+    pub fn get_local_packet(&mut self) -> Option<drtioaux::Packet> {
+        self.local_queue.pop_front()
+    }
+}

src/satman/src/subkernel.rs

@@ -0,0 +1,985 @@
+use alloc::{collections::BTreeMap,
+            format,
+            string::{String, ToString},
+            vec::Vec};
+use core::{option::NoneError, slice, str};
+
+use core_io::{Error as IoError, Write};
+use cslice::AsCSlice;
+use dma::{Error as DmaError, Manager as DmaManager};
+use io::{Cursor, ProtoWrite};
+use ksupport::{eh_artiq, kernel, rpc};
+use libboard_artiq::{drtio_routing::RoutingTable,
+                     drtioaux,
+                     drtioaux_proto::{PayloadStatus, MASTER_PAYLOAD_MAX_SIZE, SAT_PAYLOAD_MAX_SIZE},
+                     pl::csr};
+use libboard_zynq::{time::Milliseconds, timer::GlobalTimer};
+use libcortex_a9::sync_channel::Receiver;
+use log::warn;
+use routing::{Router, SliceMeta, Sliceable};
+
+#[derive(Debug, Clone, PartialEq)]
+enum KernelState {
+    Absent,
+    Loaded,
+    Running,
+    MsgAwait {
+        max_time: Option<Milliseconds>,
+        id: u32,
+        tags: Vec<u8>,
+    },
+    MsgSending,
+    SubkernelAwaitLoad,
+    SubkernelAwaitFinish {
+        max_time: Option<Milliseconds>,
+        id: u32,
+    },
+    DmaUploading,
+    DmaPendingPlayback {
+        id: u32,
+        timestamp: u64,
+    },
+    DmaPendingAwait {
+        id: u32,
+        timestamp: u64,
+        max_time: Milliseconds,
+    },
+    DmaAwait {
+        max_time: Milliseconds,
+    },
+}
+
+#[derive(Debug)]
+pub enum Error {
+    Load(String),
+    KernelNotFound,
+    Unexpected(String),
+    NoMessage,
+    AwaitingMessage,
+    SubkernelIoError,
+    DrtioError,
+    KernelException(Sliceable),
+    DmaError(DmaError),
+}
+
+impl From<NoneError> for Error {
+    fn from(_: NoneError) -> Error {
+        Error::KernelNotFound
+    }
+}
+
+impl From<IoError> for Error {
+    fn from(_value: IoError) -> Error {
+        Error::SubkernelIoError
+    }
+}
+
+impl From<DmaError> for Error {
+    fn from(value: DmaError) -> Error {
+        Error::DmaError(value)
+    }
+}
+
+impl From<()> for Error {
+    fn from(_: ()) -> Error {
+        Error::NoMessage
+    }
+}
+
+impl From<drtioaux::Error> for Error {
+    fn from(_value: drtioaux::Error) -> Error {
+        Error::DrtioError
+    }
+}
+
+macro_rules! unexpected {
+    ($($arg:tt)*) => (return Err(Error::Unexpected(format!($($arg)*))));
+}
+
+/* represents interkernel messages */
+struct Message {
+    count: u8,
+    id: u32,
+    data: Vec<u8>,
+}
+
+#[derive(PartialEq)]
+enum OutMessageState {
+    NoMessage,
+    MessageBeingSent,
+    MessageSent,
+    MessageAcknowledged,
+}
+
+/* for dealing with incoming and outgoing interkernel messages */
+struct MessageManager {
+    out_message: Option<Sliceable>,
+    out_state: OutMessageState,
+    in_queue: Vec<Message>,
+    in_buffer: Option<Message>,
+}
+
+// Per-run state
+struct Session {
+    id: u32,
+    kernel_state: KernelState,
+    last_exception: Option<Sliceable>,
+    messages: MessageManager,
+    source: u8, // which destination requested running the kernel
+    subkernels_finished: Vec<u32>,
+}
+
+impl Session {
+    pub fn new(id: u32) -> Session {
+        Session {
+            id: id,
+            kernel_state: KernelState::Absent,
+            last_exception: None,
+            messages: MessageManager::new(),
+            source: 0,
+            subkernels_finished: Vec::new(),
+        }
+    }
+
+    fn running(&self) -> bool {
+        match self.kernel_state {
+            KernelState::Absent | KernelState::Loaded => false,
+            _ => true,
+        }
+    }
+}
+
+#[derive(Debug)]
+struct KernelLibrary {
+    library: Vec<u8>,
+    complete: bool,
+}
+
+pub struct Manager<'a> {
+    kernels: BTreeMap<u32, KernelLibrary>,
+    session: Session,
+    control: &'a mut kernel::Control,
+    cache: BTreeMap<String, Vec<i32>>,
+    last_finished: Option<SubkernelFinished>,
+}
+
+pub struct SubkernelFinished {
+    pub id: u32,
+    pub with_exception: bool,
+    pub exception_source: u8,
+    pub source: u8,
+}
+
+impl MessageManager {
+    pub fn new() -> MessageManager {
+        MessageManager {
+            out_message: None,
+            out_state: OutMessageState::NoMessage,
+            in_queue: Vec::new(),
+            in_buffer: None,
+        }
+    }
+
+    pub fn handle_incoming(
+        &mut self,
+        status: PayloadStatus,
+        id: u32,
+        length: usize,
+        data: &[u8; MASTER_PAYLOAD_MAX_SIZE],
+    ) {
+        // called when receiving a message from master
+        if status.is_first() {
+            self.in_buffer = None;
+        }
+        match self.in_buffer.as_mut() {
+            Some(message) => message.data.extend(&data[..length]),
+            None => {
+                self.in_buffer = Some(Message {
+                    count: data[0],
+                    id: id,
+                    data: data[1..length].to_vec(),
+                });
+            }
+        };
+        if status.is_last() {
+            // when done, remove from working queue
+            self.in_queue.push(self.in_buffer.take().unwrap());
+        }
+    }
+
+    pub fn was_message_acknowledged(&mut self) -> bool {
+        match self.out_state {
+            OutMessageState::MessageAcknowledged => {
+                self.out_state = OutMessageState::NoMessage;
+                true
+            }
+            _ => false,
+        }
+    }
+
+    pub fn get_outgoing_slice(&mut self, data_slice: &mut [u8; MASTER_PAYLOAD_MAX_SIZE]) -> Option<SliceMeta> {
+        if self.out_state != OutMessageState::MessageBeingSent {
+            return None;
+        }
+        let meta = self.out_message.as_mut()?.get_slice_master(data_slice);
+        if meta.status.is_last() {
+            // clear the message slot
+            self.out_message = None;
+            // notify kernel with a flag that message is sent
+            self.out_state = OutMessageState::MessageSent;
+        }
+        Some(meta)
+    }
+
+    pub fn ack_slice(&mut self) -> bool {
+        // returns whether or not there's more to be sent
+        match self.out_state {
+            OutMessageState::MessageBeingSent => true,
+            OutMessageState::MessageSent => {
+                self.out_state = OutMessageState::MessageAcknowledged;
+                false
+            }
+            _ => {
+                warn!("received unsolicited SubkernelMessageAck");
+                false
+            }
+        }
+    }
+
+    pub fn accept_outgoing(
+        &mut self,
+        id: u32,
+        self_destination: u8,
+        destination: u8,
+        message: Vec<u8>,
+        routing_table: &RoutingTable,
+        rank: u8,
+        router: &mut Router,
+    ) -> Result<(), Error> {
+        self.out_message = Some(Sliceable::new(destination, message));
+
+        let mut data_slice: [u8; MASTER_PAYLOAD_MAX_SIZE] = [0; MASTER_PAYLOAD_MAX_SIZE];
+        self.out_state = OutMessageState::MessageBeingSent;
+        let meta = self.get_outgoing_slice(&mut data_slice).unwrap();
+        router.route(
+            drtioaux::Packet::SubkernelMessage {
+                source: self_destination,
+                destination: destination,
+                id: id,
+                status: meta.status,
+                length: meta.len as u16,
+                data: data_slice,
+            },
+            routing_table,
+            rank,
+            self_destination,
+        );
+        Ok(())
+    }
+
+    pub fn get_incoming(&mut self, id: u32) -> Option<Message> {
+        for i in 0..self.in_queue.len() {
+            if self.in_queue[i].id == id {
+                return Some(self.in_queue.remove(i));
+            }
+        }
+        None
+    }
+}
+
+impl<'a> Manager<'_> {
+    pub fn new(control: &mut kernel::Control) -> Manager {
+        Manager {
+            kernels: BTreeMap::new(),
+            session: Session::new(0),
+            control: control,
+            cache: BTreeMap::new(),
+            last_finished: None,
+        }
+    }
+
+    pub fn add(&mut self, id: u32, status: PayloadStatus, data: &[u8], data_len: usize) -> Result<(), Error> {
+        let kernel = match self.kernels.get_mut(&id) {
+            Some(kernel) => {
+                if kernel.complete || status.is_first() {
+                    // replace entry
+                    self.kernels.remove(&id);
+                    self.kernels.insert(
+                        id,
+                        KernelLibrary {
+                            library: Vec::new(),
+                            complete: false,
+                        },
+                    );
+                    self.kernels.get_mut(&id)?
+                } else {
+                    kernel
+                }
+            }
+            None => {
+                self.kernels.insert(
+                    id,
+                    KernelLibrary {
+                        library: Vec::new(),
+                        complete: false,
+                    },
+                );
+                self.kernels.get_mut(&id)?
+            }
+        };
+        kernel.library.extend(&data[0..data_len]);
+
+        kernel.complete = status.is_last();
+        Ok(())
+    }
+
+    pub fn running(&self) -> bool {
+        self.session.running()
+    }
+
+    pub fn get_current_id(&self) -> Option<u32> {
+        match self.running() {
+            true => Some(self.session.id),
+            false => None,
+        }
+    }
+
+    pub fn run(&mut self, source: u8, id: u32) -> Result<(), Error> {
+        if self.session.kernel_state != KernelState::Loaded || self.session.id != id {
+            self.load(id)?;
+        }
+        self.session.kernel_state = KernelState::Running;
+        self.session.source = source;
+        unsafe {
+            csr::cri_con::selected_write(2);
+        }
+
+        self.control.tx.send(kernel::Message::StartRequest);
+        Ok(())
+    }
+
+    pub fn message_handle_incoming(
+        &mut self,
+        status: PayloadStatus,
+        id: u32,
+        length: usize,
+        slice: &[u8; MASTER_PAYLOAD_MAX_SIZE],
+    ) {
+        if !self.running() {
+            return;
+        }
+        self.session.messages.handle_incoming(status, id, length, slice);
+    }
+
+    pub fn message_get_slice(&mut self, slice: &mut [u8; MASTER_PAYLOAD_MAX_SIZE]) -> Option<SliceMeta> {
+        if !self.running() {
+            return None;
+        }
+        self.session.messages.get_outgoing_slice(slice)
+    }
+
+    pub fn message_ack_slice(&mut self) -> bool {
+        if !self.running() {
+            warn!("received unsolicited SubkernelMessageAck");
+            return false;
+        }
+        self.session.messages.ack_slice()
+    }
+
+    pub fn load(&mut self, id: u32) -> Result<(), Error> {
+        if self.session.id == id && self.session.kernel_state == KernelState::Loaded {
+            return Ok(());
+        }
+        if !self.kernels.get(&id)?.complete {
+            return Err(Error::KernelNotFound);
+        }
+        self.session = Session::new(id);
+        self.control.restart();
+
+        self.control
+            .tx
+            .send(kernel::Message::LoadRequest(self.kernels.get(&id)?.library.clone()));
+        let reply = self.control.rx.recv();
+        match reply {
+            kernel::Message::LoadCompleted => Ok(()),
+            kernel::Message::LoadFailed => Err(Error::Load("kernel load failed".to_string())),
+            _ => Err(Error::Load(format!(
+                "unexpected kernel CPU reply to load request: {:?}",
+                reply
+            ))),
+        }
+    }
+
+    pub fn exception_get_slice(&mut self, data_slice: &mut [u8; SAT_PAYLOAD_MAX_SIZE]) -> SliceMeta {
+        match self.session.last_exception.as_mut() {
+            Some(exception) => exception.get_slice_sat(data_slice),
+            None => SliceMeta {
+                destination: 0,
+                len: 0,
+                status: PayloadStatus::FirstAndLast,
+            },
+        }
+    }
+
+    fn kernel_stop(&mut self) {
+        self.session.kernel_state = KernelState::Absent;
+        unsafe {
+            csr::cri_con::selected_write(0);
+        }
+    }
+
+    fn runtime_exception(&mut self, cause: Error) {
+        let raw_exception: Vec<u8> = Vec::new();
+        let mut writer = Cursor::new(raw_exception);
+        match write_exception(
+            &mut writer,
+            &[Some(eh_artiq::Exception {
+                id: 11, // SubkernelError, defined in ksupport
+                message: format!("in subkernel id {}: {:?}", self.session.id, cause).as_c_slice(),
+                param: [0, 0, 0],
+                file: file!().as_c_slice(),
+                line: line!(),
+                column: column!(),
+                function: format!("subkernel id {}", self.session.id).as_c_slice(),
+            })],
+            &[eh_artiq::StackPointerBacktrace {
+                stack_pointer: 0,
+                initial_backtrace_size: 0,
+                current_backtrace_size: 0,
+            }],
+            &[],
+            0,
+        ) {
+            Ok(_) => self.session.last_exception = Some(Sliceable::new(0, writer.into_inner())),
+            Err(_) => error!("Error writing exception data"),
+        }
+        self.kernel_stop();
+    }
+
+    pub fn ddma_finished(&mut self, error: u8, channel: u32, timestamp: u64) {
+        if let KernelState::DmaAwait { .. } = self.session.kernel_state {
+            self.control.tx.send(kernel::Message::DmaAwaitRemoteReply {
+                timeout: false,
+                error: error,
+                channel: channel,
+                timestamp: timestamp,
+            });
+            self.session.kernel_state = KernelState::Running;
+        }
+    }
+
+    pub fn ddma_nack(&mut self) {
+        // for simplicity treat it as a timeout...
+        if let KernelState::DmaAwait { .. } = self.session.kernel_state {
+            self.control.tx.send(kernel::Message::DmaAwaitRemoteReply {
+                timeout: true,
+                error: 0,
+                channel: 0,
+                timestamp: 0,
+            });
+            self.session.kernel_state = KernelState::Running;
+        }
+    }
+
+    pub fn ddma_remote_uploaded(&mut self, succeeded: bool) -> Option<(u32, u64)> {
+        // returns a tuple of id, timestamp in case a playback needs to be started immediately
+        if !succeeded {
+            self.kernel_stop();
+            self.runtime_exception(Error::DmaError(DmaError::UploadFail));
+        }
+        let res = match self.session.kernel_state {
+            KernelState::DmaPendingPlayback { id, timestamp } => {
+                self.session.kernel_state = KernelState::Running;
+                Some((id, timestamp))
+            }
+            KernelState::DmaPendingAwait {
+                id,
+                timestamp,
+                max_time,
+            } => {
+                self.session.kernel_state = KernelState::DmaAwait { max_time: max_time };
+                Some((id, timestamp))
+            }
+            KernelState::DmaUploading => {
+                self.session.kernel_state = KernelState::Running;
+                None
+            }
+            _ => None,
+        };
+        res
+    }
+
+    pub fn process_kern_requests(
+        &mut self,
+        router: &mut Router,
+        routing_table: &RoutingTable,
+        rank: u8,
+        destination: u8,
+        dma_manager: &mut DmaManager,
+        timer: &GlobalTimer,
+    ) {
+        if let Some(subkernel_finished) = self.last_finished.take() {
+            info!(
+                "subkernel {} finished, with exception: {}",
+                subkernel_finished.id, subkernel_finished.with_exception
+            );
+            router.route(
+                drtioaux::Packet::SubkernelFinished {
+                    destination: subkernel_finished.source,
+                    id: subkernel_finished.id,
+                    with_exception: subkernel_finished.with_exception,
+                    exception_src: subkernel_finished.exception_source,
+                },
+                &routing_table,
+                rank,
+                destination,
+            );
+        }
+
+        if !self.running() {
+            return;
+        }
+
+        match self.process_external_messages(timer) {
+            Ok(()) => (),
+            Err(Error::AwaitingMessage) => return, // kernel still waiting, do not process kernel messages
+            Err(Error::KernelException(exception)) => {
+                self.session.last_exception = Some(exception);
+                self.last_finished = Some(SubkernelFinished {
+                    id: self.session.id,
+                    with_exception: true,
+                    exception_source: destination,
+                    source: self.session.source,
+                });
+            }
+            Err(e) => {
+                error!("Error while running processing external messages: {:?}", e);
+                self.runtime_exception(e);
+                self.last_finished = Some(SubkernelFinished {
+                    id: self.session.id,
+                    with_exception: true,
+                    exception_source: destination,
+                    source: self.session.source,
+                });
+            }
+        }
+
+        match self.process_kern_message(router, routing_table, rank, destination, dma_manager, timer) {
+            Ok(true) => {
+                self.last_finished = Some(SubkernelFinished {
+                    id: self.session.id,
+                    with_exception: false,
+                    exception_source: 0,
+                    source: self.session.source,
+                });
+            }
+            Ok(false) | Err(Error::NoMessage) => (),
+            Err(Error::KernelException(exception)) => {
+                self.session.last_exception = Some(exception);
+                self.last_finished = Some(SubkernelFinished {
+                    id: self.session.id,
+                    with_exception: true,
+                    exception_source: destination,
+                    source: self.session.source,
+                });
+            }
+            Err(e) => {
+                error!("Error while running kernel: {:?}", e);
+                self.runtime_exception(e);
+                self.last_finished = Some(SubkernelFinished {
+                    id: self.session.id,
+                    with_exception: true,
+                    exception_source: destination,
+                    source: self.session.source,
+                });
+            }
+        }
+    }
+
+    pub fn subkernel_load_run_reply(&mut self, succeeded: bool) {
+        if self.session.kernel_state == KernelState::SubkernelAwaitLoad {
+            self.control
+                .tx
+                .send(kernel::Message::SubkernelLoadRunReply { succeeded: succeeded });
+            self.session.kernel_state = KernelState::Running;
+        } else {
+            warn!("received unsolicited SubkernelLoadRunReply");
+        }
+    }
+
+    pub fn remote_subkernel_finished(&mut self, id: u32, with_exception: bool, exception_source: u8) {
+        if with_exception {
+            self.kernel_stop();
+            self.last_finished = Some(SubkernelFinished {
+                source: self.session.source,
+                id: self.session.id,
+                with_exception: true,
+                exception_source: exception_source,
+            })
+        } else {
+            self.session.subkernels_finished.push(id);
+        }
+    }
+
+    fn process_kern_message(
+        &mut self,
+        router: &mut Router,
+        routing_table: &RoutingTable,
+        rank: u8,
+        self_destination: u8,
+        dma_manager: &mut DmaManager,
+        timer: &GlobalTimer,
+    ) -> Result<bool, Error> {
+        let reply = self.control.rx.try_recv()?;
+        match reply {
+            kernel::Message::KernelFinished(_async_errors) => {
+                self.kernel_stop();
+                dma_manager.cleanup(router, rank, self_destination, routing_table);
+                return Ok(true);
+            }
+            kernel::Message::KernelException(exceptions, stack_pointers, backtrace, async_errors) => {
+                error!("exception in kernel");
+                for exception in exceptions {
+                    error!("{:?}", exception.unwrap());
+                }
+                error!("stack pointers: {:?}", stack_pointers);
+                error!("backtrace: {:?}", backtrace);
+                let buf: Vec<u8> = Vec::new();
+                let mut writer = Cursor::new(buf);
+                match write_exception(&mut writer, exceptions, stack_pointers, backtrace, async_errors) {
+                    Ok(()) => (),
+                    Err(_) => error!("Error writing exception data"),
+                }
+                self.kernel_stop();
+                return Err(Error::KernelException(Sliceable::new(0, writer.into_inner())));
+            }
+            kernel::Message::CachePutRequest(key, value) => {
+                self.cache.insert(key, value);
+            }
+            kernel::Message::CacheGetRequest(key) => {
+                const DEFAULT: Vec<i32> = Vec::new();
+                let value = self.cache.get(&key).unwrap_or(&DEFAULT).clone();
+                self.control.tx.send(kernel::Message::CacheGetReply(value));
+            }
+
+            kernel::Message::DmaPutRequest(recorder) => {
+                // ddma is always used on satellites
+                if let Ok(id) = dma_manager.put_record(recorder, self_destination) {
+                    dma_manager.upload_traces(id, router, rank, self_destination, routing_table)?;
+                    self.session.kernel_state = KernelState::DmaUploading;
+                } else {
+                    unexpected!("DMAError: found an unsupported call to RTIO devices on master")
+                }
+            }
+            kernel::Message::DmaEraseRequest(name) => {
+                dma_manager.erase_name(&name, router, rank, self_destination, routing_table);
+            }
+            kernel::Message::DmaGetRequest(name) => {
+                let dma_meta = dma_manager.retrieve(self_destination, &name);
+                self.control.tx.send(kernel::Message::DmaGetReply(dma_meta));
+            }
+            kernel::Message::DmaStartRemoteRequest { id, timestamp } => {
+                if self.session.kernel_state != KernelState::DmaUploading {
+                    dma_manager.playback_remote(
+                        id as u32,
+                        timestamp as u64,
+                        router,
+                        rank,
+                        self_destination,
+                        routing_table,
+                    )?;
+                } else {
+                    self.session.kernel_state = KernelState::DmaPendingPlayback {
+                        id: id as u32,
+                        timestamp: timestamp as u64,
+                    };
+                }
+            }
+            kernel::Message::DmaAwaitRemoteRequest(_id) => {
+                let max_time = timer.get_time() + Milliseconds(10000);
+                self.session.kernel_state = match self.session.kernel_state {
+                    // if we are still waiting for the traces to be uploaded, extend the state by timeout
+                    KernelState::DmaPendingPlayback { id, timestamp } => KernelState::DmaPendingAwait {
+                        id: id,
+                        timestamp: timestamp,
+                        max_time: max_time,
+                    },
+                    _ => KernelState::DmaAwait { max_time: max_time },
+                };
+            }
+
+            kernel::Message::SubkernelMsgSend {
+                id: _id,
+                destination: msg_dest,
+                data,
+            } => {
+                let msg_dest = msg_dest.or(Some(self.session.source)).unwrap();
+                self.session.messages.accept_outgoing(
+                    self.session.id,
+                    self_destination,
+                    msg_dest,
+                    data,
+                    routing_table,
+                    rank,
+                    router,
+                )?;
+                self.session.kernel_state = KernelState::MsgSending;
+            }
+            kernel::Message::SubkernelMsgRecvRequest { id, timeout, tags } => {
+                let id = if id == -1 { self.session.id } else { id as u32 };
+                let max_time = if timeout > 0 {
+                    Some(timer.get_time() + Milliseconds(timeout as u64))
+                } else {
+                    None
+                };
+                self.session.kernel_state = KernelState::MsgAwait {
+                    max_time: max_time,
+                    id: id,
+                    tags: tags,
+                };
+            }
+            kernel::Message::SubkernelLoadRunRequest {
+                id,
+                destination: sk_destination,
+                run,
+            } => {
+                self.session.kernel_state = KernelState::SubkernelAwaitLoad;
+                router.route(
+                    drtioaux::Packet::SubkernelLoadRunRequest {
+                        source: self_destination,
+                        destination: sk_destination,
+                        id: id,
+                        run: run,
+                    },
+                    routing_table,
+                    rank,
+                    self_destination,
+                );
+            }
+
+            kernel::Message::SubkernelAwaitFinishRequest { id, timeout } => {
+                let max_time = if timeout > 0 {
+                    Some(timer.get_time() + Milliseconds(timeout as u64))
+                } else {
+                    None
+                };
+                self.session.kernel_state = KernelState::SubkernelAwaitFinish {
+                    max_time: max_time,
+                    id: id,
+                };
+            }
+            kernel::Message::UpDestinationsRequest(destination) => {
+                self.control.tx.send(kernel::Message::UpDestinationsReply(
+                    destination == (self_destination as i32),
+                ));
+            }
+            _ => {
+                unexpected!("unexpected message from core1 while kernel was running: {:?}", reply);
+            }
+        }
+        Ok(false)
+    }
+
+    fn process_external_messages(&mut self, timer: &GlobalTimer) -> Result<(), Error> {
+        match &self.session.kernel_state {
+            KernelState::MsgAwait { max_time, id, tags } => {
+                if let Some(max_time) = *max_time {
+                    if timer.get_time() > max_time {
+                        self.control.tx.send(kernel::Message::SubkernelMsgRecvReply {
+                            status: kernel::SubkernelStatus::Timeout,
+                            count: 0,
+                        });
+                        self.session.kernel_state = KernelState::Running;
+                        return Ok(());
+                    }
+                }
+                if let Some(message) = self.session.messages.get_incoming(*id) {
+                    self.control.tx.send(kernel::Message::SubkernelMsgRecvReply {
+                        status: kernel::SubkernelStatus::NoError,
+                        count: message.count,
+                    });
+                    let tags = tags.clone();
+                    self.session.kernel_state = KernelState::Running;
+                    self.pass_message_to_kernel(&message, tags, timer)
+                } else {
+                    Err(Error::AwaitingMessage)
+                }
+            }
+            KernelState::MsgSending => {
+                if self.session.messages.was_message_acknowledged() {
+                    self.session.kernel_state = KernelState::Running;
+                    self.control.tx.send(kernel::Message::SubkernelMsgSent);
+                    Ok(())
+                } else {
+                    Err(Error::AwaitingMessage)
+                }
+            }
+            KernelState::SubkernelAwaitFinish { max_time, id } => {
+                if let Some(max_time) = *max_time {
+                    if timer.get_time() > max_time {
+                        self.control.tx.send(kernel::Message::SubkernelAwaitFinishReply {
+                            status: kernel::SubkernelStatus::Timeout,
+                        });
+                        self.session.kernel_state = KernelState::Running;
+                        return Ok(());
+                    }
+                }
+                let mut i = 0;
+                for status in &self.session.subkernels_finished {
+                    if *status == *id {
+                        self.control.tx.send(kernel::Message::SubkernelAwaitFinishReply {
+                            status: kernel::SubkernelStatus::NoError,
+                        });
+                        self.session.kernel_state = KernelState::Running;
+                        self.session.subkernels_finished.swap_remove(i);
+                        break;
+                    }
+                    i += 1;
+                }
+                Ok(())
+            }
+            KernelState::DmaAwait { max_time } | KernelState::DmaPendingAwait { max_time, .. } => {
+                if timer.get_time() > *max_time {
+                    self.control.tx.send(kernel::Message::DmaAwaitRemoteReply {
+                        timeout: true,
+                        error: 0,
+                        channel: 0,
+                        timestamp: 0,
+                    });
+                    self.session.kernel_state = KernelState::Running;
+                }
+                Ok(())
+            }
+            _ => Ok(()),
+        }
+    }
+
+    fn pass_message_to_kernel(&mut self, message: &Message, tags: Vec<u8>, timer: &GlobalTimer) -> Result<(), Error> {
+        let mut reader = Cursor::new(&message.data);
+        let mut current_tags: &[u8] = &tags;
+        let mut i = message.count;
+        loop {
+            let slot = match recv_w_timeout(&mut self.control.rx, timer, 100)? {
+                kernel::Message::RpcRecvRequest(slot) => slot,
+                other => unexpected!("expected root value slot from core1, not {:?}", other),
+            };
+            let mut exception: Option<Sliceable> = None;
+            let mut unexpected: Option<String> = None;
+            let remaining_tags = rpc::recv_return(&mut reader, current_tags, slot, &mut |size| {
+                if size == 0 {
+                    0 as *mut ()
+                } else {
+                    self.control.tx.send(kernel::Message::RpcRecvReply(Ok(size)));
+                    match recv_w_timeout(&mut self.control.rx, timer, 100) {
+                        Ok(kernel::Message::RpcRecvRequest(slot)) => slot,
+                        Ok(kernel::Message::KernelException(exceptions, stack_pointers, backtrace, async_errors)) => {
+                            let buf: Vec<u8> = Vec::new();
+                            let mut writer = Cursor::new(buf);
+                            match write_exception(&mut writer, exceptions, stack_pointers, backtrace, async_errors) {
+                                Ok(()) => {
+                                    exception = Some(Sliceable::new(0, writer.into_inner()));
+                                }
+                                Err(_) => {
+                                    unexpected = Some("Error writing exception data".to_string());
+                                }
+                            };
+                            0 as *mut ()
+                        }
+                        other => {
+                            unexpected = Some(format!("expected nested value slot from kernel CPU, not {:?}", other));
+                            0 as *mut ()
+                        }
+                    }
+                }
+            })?;
+            if let Some(exception) = exception {
+                self.kernel_stop();
+                return Err(Error::KernelException(exception));
+            } else if let Some(unexpected) = unexpected {
+                self.kernel_stop();
+                unexpected!("{}", unexpected);
+            }
+            self.control.tx.send(kernel::Message::RpcRecvReply(Ok(0)));
+            i -= 1;
+            if i == 0 {
+                break;
+            } else {
+                current_tags = remaining_tags;
+            }
+        }
+        Ok(())
+    }
+}
+
+fn write_exception<W>(
+    writer: &mut W,
+    exceptions: &[Option<eh_artiq::Exception>],
+    stack_pointers: &[eh_artiq::StackPointerBacktrace],
+    backtrace: &[(usize, usize)],
+    async_errors: u8,
+) -> Result<(), Error>
+where
+    W: Write + ?Sized,
+{
+    /* header */
+    writer.write_bytes(&[0x5a, 0x5a, 0x5a, 0x5a, /*Reply::KernelException*/ 9])?;
+    writer.write_u32(exceptions.len() as u32)?;
+    for exception in exceptions.iter() {
+        let exception = exception.as_ref().unwrap();
+        writer.write_u32(exception.id)?;
+
+        if exception.message.len() == usize::MAX {
+            // exception with host string
+            writer.write_u32(u32::MAX)?;
+            writer.write_u32(exception.message.as_ptr() as u32)?;
+        } else {
+            let msg =
+                str::from_utf8(unsafe { slice::from_raw_parts(exception.message.as_ptr(), exception.message.len()) })
+                    .unwrap()
+                    .replace(
+                        "{rtio_channel_info:0}",
+                        &format!(
+                            "0x{:04x}:{}",
+                            exception.param[0],
+                            ksupport::resolve_channel_name(exception.param[0] as u32)
+                        ),
+                    );
+            writer.write_string(&msg)?;
+        }
+        writer.write_u64(exception.param[0] as u64)?;
+        writer.write_u64(exception.param[1] as u64)?;
+        writer.write_u64(exception.param[2] as u64)?;
+        writer.write_bytes(exception.file.as_ref())?;
+        writer.write_u32(exception.line)?;
+        writer.write_u32(exception.column)?;
+        writer.write_bytes(exception.function.as_ref())?;
+    }
+
+    for sp in stack_pointers.iter() {
+        writer.write_u32(sp.stack_pointer as u32)?;
+        writer.write_u32(sp.initial_backtrace_size as u32)?;
+        writer.write_u32(sp.current_backtrace_size as u32)?;
+    }
+    writer.write_u32(backtrace.len() as u32)?;
+    for &(addr, sp) in backtrace {
+        writer.write_u32(addr as u32)?;
+        writer.write_u32(sp as u32)?;
+    }
+    writer.write_u8(async_errors as u8)?;
+    Ok(())
+}
+
+fn recv_w_timeout(
+    rx: &mut Receiver<'_, kernel::Message>,
+    timer: &GlobalTimer,
+    timeout: u64,
+) -> Result<kernel::Message, Error> {
+    let max_time = timer.get_time() + Milliseconds(timeout);
+    while timer.get_time() < max_time {
+        match rx.try_recv() {
+            Err(_) => (),
+            Ok(message) => return Ok(message),
+        }
+    }
+    Err(Error::NoMessage)
+}