Gateware: remove redundant si549.py & wrpll.py

rtio_clocking: update mmcm setting to use HIGH bandwidth

.gitignore

@@ -3,3 +3,11 @@ examples/*.elf
 __pycache__
 
 build
+
+src/libboard_artiq/Cargo.toml
+src/libc/Cargo.toml
+src/libdyld/Cargo.toml
+src/libio/Cargo.toml
+src/libksupport/Cargo.toml
+src/runtime/Cargo.toml
+src/satman/Cargo.toml

README.md

@@ -58,12 +58,11 @@ Notes:
 - When calling make, you need to specify both the variant and firmware type.
 - Firmware type must be either ``runtime`` for DRTIO-less or DRTIO master variants, or ``satman`` for DRTIO satellite.
 - If the board is connected to the local machine, use the ``local_run.sh`` script.
-- To update ``zynq-rs``, update the cargo files as per usual for Rust projects, but also keep ``flake.lock`` in sync.
 
 License
 -------
 
-Copyright (C) 2019-2023 M-Labs Limited.
+Copyright (C) 2019-2024 M-Labs Limited.
 
 ARTIQ is free software: you can redistribute it and/or modify
 it under the terms of the GNU Lesser General Public License as published by

flake.lock

@@ -11,16 +11,15 @@
         "src-pythonparser": "src-pythonparser"
       },
       "locked": {
-        "lastModified": 1695781625,
+        "lastModified": 1716972728,
-        "narHash": "sha256-zv8A0Mz2hIYLfD78GxLgXpOkTgnt/kT6rGWzP6NkXkM=",
+        "narHash": "sha256-88J+eckZamtwhcCQkPpKLu6R1hmgj5+C9n2U5i+sHUE=",
-        "ref": "release-7",
+        "ref": "refs/heads/master",
-        "rev": "cc81464f53a02cb41d7b4f9efae75f815c63ea4d",
+        "rev": "49e402780bebba437c6098047ab1dc68eaf5a17c",
-        "revCount": 8185,
+        "revCount": 8808,
         "type": "git",
         "url": "https://github.com/m-labs/artiq.git"
       },
       "original": {
-        "ref": "release-7",
         "type": "git",
         "url": "https://github.com/m-labs/artiq.git"
       }
@@ -38,11 +37,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1664405593,
+        "lastModified": 1707216368,
-        "narHash": "sha256-yP441NerlLGig7n+9xHsx8yCtZ+Ggd0VqfBSzc20E04=",
+        "narHash": "sha256-ZXoqzG2QsVsybALLYXs473avXcyKSZNh2kIgcPo60XQ=",
         "owner": "m-labs",
         "repo": "artiq-comtools",
-        "rev": "15ddac62813ef623a076ccf982b3bc63d314e651",
+        "rev": "e5d0204490bccc07ef9141b0d7c405ab01cb8273",
         "type": "github"
       },
       "original": {
@@ -52,12 +51,15 @@
       }
     },
     "flake-utils": {
+      "inputs": {
+        "systems": "systems"
+      },
       "locked": {
-        "lastModified": 1659877975,
+        "lastModified": 1694529238,
-        "narHash": "sha256-zllb8aq3YO3h8B/U0/J1WBgAL8EX5yWf5pMj3G0NAmc=",
+        "narHash": "sha256-zsNZZGTGnMOf9YpHKJqMSsa0dXbfmxeoJ7xHlrt+xmY=",
         "owner": "numtide",
         "repo": "flake-utils",
-        "rev": "c0e246b9b83f637f4681389ecabcb2681b4f3af0",
+        "rev": "ff7b65b44d01cf9ba6a71320833626af21126384",
         "type": "github"
       },
       "original": {
@@ -69,11 +71,11 @@
     "mozilla-overlay": {
       "flake": false,
       "locked": {
-        "lastModified": 1687771476,
+        "lastModified": 1704373101,
-        "narHash": "sha256-TSpqz6qYVRoqkEdOCawEQ4/cWt/4pracmvw17HK1tgE=",
+        "narHash": "sha256-+gi59LRWRQmwROrmE1E2b3mtocwueCQqZ60CwLG+gbg=",
         "owner": "mozilla",
         "repo": "nixpkgs-mozilla",
-        "rev": "3a44b8783514e7d6db4b63df96071b6c2b014b07",
+        "rev": "9b11a87c0cc54e308fa83aac5b4ee1816d5418a2",
         "type": "github"
       },
       "original": {
@@ -85,11 +87,11 @@
     "mozilla-overlay_2": {
       "flake": false,
       "locked": {
-        "lastModified": 1695805681,
+        "lastModified": 1704373101,
-        "narHash": "sha256-1ElPLD8eFfnuIk0G52HGGpRtQZ4QPCjChRlEOfkZ5ro=",
+        "narHash": "sha256-+gi59LRWRQmwROrmE1E2b3mtocwueCQqZ60CwLG+gbg=",
         "owner": "mozilla",
         "repo": "nixpkgs-mozilla",
-        "rev": "6eabade97bc28d707a8b9d82ad13ef143836736e",
+        "rev": "9b11a87c0cc54e308fa83aac5b4ee1816d5418a2",
         "type": "github"
       },
       "original": {
@@ -101,11 +103,11 @@
     "mozilla-overlay_3": {
       "flake": false,
       "locked": {
-        "lastModified": 1695805681,
+        "lastModified": 1704373101,
-        "narHash": "sha256-1ElPLD8eFfnuIk0G52HGGpRtQZ4QPCjChRlEOfkZ5ro=",
+        "narHash": "sha256-+gi59LRWRQmwROrmE1E2b3mtocwueCQqZ60CwLG+gbg=",
         "owner": "mozilla",
         "repo": "nixpkgs-mozilla",
-        "rev": "6eabade97bc28d707a8b9d82ad13ef143836736e",
+        "rev": "9b11a87c0cc54e308fa83aac5b4ee1816d5418a2",
         "type": "github"
       },
       "original": {
@@ -116,16 +118,16 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1685573264,
+        "lastModified": 1716542732,
-        "narHash": "sha256-Zffu01pONhs/pqH07cjlF10NnMDLok8ix5Uk4rhOnZQ=",
+        "narHash": "sha256-0Y9fRr0CUqWT4KgBITmaGwlnNIGMYuydu2L8iLTfHU4=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "380be19fbd2d9079f677978361792cb25e8a3635",
+        "rev": "d12251ef6e8e6a46e05689eeccd595bdbd3c9e60",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "nixos-22.05",
+        "ref": "nixos-24.05",
         "repo": "nixpkgs",
         "type": "github"
       }
@@ -145,11 +147,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1685540542,
+        "lastModified": 1708937641,
-        "narHash": "sha256-wQJwL3xc6QVQbiJrt71/Z9tp4Eq1yqdGddcEiv7sPCw=",
+        "narHash": "sha256-Hkb9VYFzFgkYxfbh4kYcDSn7DbMUYehoQDeTALrxo2Q=",
         "owner": "m-labs",
         "repo": "sipyco",
-        "rev": "f5bf2ba875340a31a135aea14ad184575ca800ac",
+        "rev": "4a28b311ce0069454b4e8fe1e6049db11b9f1296",
         "type": "github"
       },
       "original": {
@@ -161,11 +163,11 @@
     "src-migen": {
       "flake": false,
       "locked": {
-        "lastModified": 1674045327,
+        "lastModified": 1715484909,
-        "narHash": "sha256-oYdeY0MbTReKbAwmSznnqw0wNawdInJoFJVWW3tesFA=",
+        "narHash": "sha256-4DCHBUBfc/VA+7NW2Hr0+JP4NnKPru2uVJyZjCCk0Ws=",
         "owner": "m-labs",
         "repo": "migen",
-        "rev": "ccaee68e14d3636e1d8fb2e0864dd89b1b1f7384",
+        "rev": "4790bb577681a8c3a8d226bc196a4e5deb39e4df",
         "type": "github"
       },
       "original": {
@@ -177,11 +179,11 @@
     "src-misoc": {
       "flake": false,
       "locked": {
-        "lastModified": 1685415268,
+        "lastModified": 1715647536,
-        "narHash": "sha256-g4+yeSV+HtWjcllM5wk4vNBUVCXtDOzUSKhxXPT7Fyc=",
+        "narHash": "sha256-q+USDcaKHABwW56Jzq8u94iGPWlyLXMyVt0j/Gyg+IE=",
         "ref": "refs/heads/master",
-        "rev": "6d48ce77b6746d3226a682790fbc95b90340986e",
+        "rev": "fea9de558c730bc394a5936094ae95bb9d6fa726",
-        "revCount": 2440,
+        "revCount": 2455,
         "submodules": true,
         "type": "git",
         "url": "https://github.com/m-labs/misoc.git"
@@ -208,6 +210,21 @@
         "type": "github"
       }
     },
+    "systems": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    },
     "zynq-rs": {
       "inputs": {
         "mozilla-overlay": "mozilla-overlay_3",
@@ -217,11 +234,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1701572971,
+        "lastModified": 1716519432,
-        "narHash": "sha256-f8IPIYenff7B8S3p/pjVxT/0HVZuJXtdg2jtMpaZEbM=",
+        "narHash": "sha256-vgKBJCQRPCutJ4n+FtJNczMZULWW7J3B8icf/PUothw=",
         "ref": "refs/heads/master",
-        "rev": "91bae572f913abc2f95acb899ca5afa33eeaa036",
+        "rev": "46dc25b89e46b9043129d77e3c9348916748e325",
-        "revCount": 634,
+        "revCount": 645,
         "type": "git",
         "url": "https://git.m-labs.hk/m-labs/zynq-rs"
       },

flake.nix

@@ -1,7 +1,7 @@
 {
   description = "ARTIQ port to the Zynq-7000 platform";
 
-  inputs.artiq.url = git+https://github.com/m-labs/artiq.git?ref=release-7;
+  inputs.artiq.url = git+https://github.com/m-labs/artiq.git;
   inputs.mozilla-overlay = { url = github:mozilla/nixpkgs-mozilla; flake = false; };
   inputs.zynq-rs.url = git+https://git.m-labs.hk/m-labs/zynq-rs;
   inputs.zynq-rs.inputs.nixpkgs.follows = "artiq/nixpkgs";
@@ -11,7 +11,9 @@
     pkgs = import artiq.inputs.nixpkgs { system = "x86_64-linux"; overlays = [ (import mozilla-overlay) ]; };
     zynqpkgs = zynq-rs.packages.x86_64-linux;
     artiqpkgs = artiq.packages.x86_64-linux;
+    llvmPackages_11 = zynq-rs.llvmPackages_11;
 
+    rust = zynq-rs.rust;
     rustPlatform = zynq-rs.rustPlatform;
 
     fastnumbers = pkgs.python3Packages.buildPythonPackage rec {
@@ -37,63 +39,53 @@
 
     ramda = pkgs.python3Packages.buildPythonPackage {
       pname = "ramda";
-      version = "unstable-2019-02-01";
+      version = "unstable-2020-04-11";
 
       src = pkgs.fetchFromGitHub {
         owner = "peteut";
         repo = "ramda.py";
-        rev = "bd58f8e69d0e9a713d9c1f286a1ac5e5603956b1";
+        rev = "d315a9717ebd639366bf3fe26bad9e3d08ec3c49";
-        sha256 = "0qzd5yp9lbaham8p1wiymdjapzbqsli7lvngv24c3z4ybd9jlq9g";
+        sha256 = "sha256-bmSt/IHDnULsZjsC6edELnNH7LoJSVF4L4XhwBAXRkY=";
       };
 
       nativeBuildInputs = with pkgs.python3Packages; [ pbr ];
       propagatedBuildInputs = with pkgs.python3Packages; [ future fastnumbers ];
 
-      checkInputs = with pkgs.python3Packages; [ pytest pytest-flake8 ];
+      checkInputs = with pkgs.python3Packages; [ pytest ];
       checkPhase = "pytest";
       doCheck = false;
 
       preBuild = ''
-        export PBR_VERSION=0.0.1
+        export PBR_VERSION=0.5.5
       '';
     };
 
     migen-axi = pkgs.python3Packages.buildPythonPackage {
       pname = "migen-axi";
-      version = "unstable-2021-09-15";
+      version = "unstable-2023-01-06";
 
       src = pkgs.fetchFromGitHub {
         owner = "peteut";
         repo = "migen-axi";
-        rev = "9763505ee96acd7572280a2d1233721342dc7c3f";
+        rev = "27eaa84a70a3abfe1930c86c36c4de2cd652da35";
-        sha256 = "15c7g05n183rka66fl1glzp6h7xjlpy1p6k8biry24dangsmxmvg";
+        sha256 = "sha256-3Y9W5ns+1wbVd14iePzgSBzE+LxnGMUDtUw3BccFt80=";
       };
 
-      nativeBuildInputs = with pkgs.python3Packages; [ pbr ];
+      format = "pyproject";
+
       propagatedBuildInputs = with pkgs.python3Packages; [ setuptools click numpy toolz jinja2 ramda artiqpkgs.migen artiqpkgs.misoc ];
 
+      checkInputs = with pkgs.python3Packages; [ pytest-runner pytestCheckHook pytest-timeout ];
+
+      # migen/misoc version checks are broken with pyproject for some reason
       postPatch = ''
-        substituteInPlace requirements.txt \
+        sed -i "1,4d" pyproject.toml
-          --replace "jinja2==2.11.3" "jinja2"
+        substituteInPlace pyproject.toml \
-        substituteInPlace requirements.txt \
+          --replace '"migen@git+https://github.com/m-labs/migen",' ""
-          --replace "future==0.18.2" "future"
+        substituteInPlace pyproject.toml \
-        substituteInPlace requirements.txt \
+          --replace '"misoc@git+https://github.com/m-labs/misoc.git",' ""
-          --replace "ramda==0.5.5" "ramda"
+        # pytest-flake8 is broken with recent flake8. Re-enable after fix.
-        substituteInPlace requirements.txt \
+        substituteInPlace setup.cfg --replace '--flake8' ""
-          --replace "colorama==0.4.3" "colorama"
-        substituteInPlace requirements.txt \
-          --replace "toolz==0.10.0" "toolz"
-        substituteInPlace requirements.txt \
-          --replace "pyserial==3.4" "pyserial"
-        substituteInPlace requirements.txt \
-          --replace "markupsafe==1.1.1" "markupsafe"
-      '';
-
-      checkInputs = with pkgs.python3Packages; [ pytest pytest-timeout pytest-flake8 ];
-      checkPhase = "pytest";
-
-      preBuild = ''
-        export PBR_VERSION=0.0.1
       '';
     };
     binutils = { platform, target, zlib }: pkgs.stdenv.mkDerivation rec {
@@ -122,34 +114,33 @@
       "nist_clock_satellite" "nist_qc2_satellite" "acpki_nist_clock_satellite" "acpki_nist_qc2_satellite" 
       "nist_clock_satellite_100mhz" "nist_qc2_satellite_100mhz" "acpki_nist_clock_satellite_100mhz" "acpki_nist_qc2_satellite_100mhz"
     ];
-    build = { target, variant, json ? null }: let
+    board-package-set = { target, variant, json ? null }: let
       szl = zynqpkgs."${target}-szl";
       fsbl = zynqpkgs."${target}-fsbl";
       fwtype = if builtins.elem variant sat_variants then "satman" else "runtime";
 
       firmware = rustPlatform.buildRustPackage rec {
-
         name = "firmware";
         src = ./src;
         cargoLock = { 
           lockFile = src/Cargo.lock;
           outputHashes = {
-            "libasync-0.0.0" = "sha256-f8IPIYenff7B8S3p/pjVxT/0HVZuJXtdg2jtMpaZEbM=";
+              "tar-no-std-0.1.8" = "sha256-xm17108v4smXOqxdLvHl9CxTCJslmeogjm4Y87IXFuM=";
-          };
+            };
         };
 
         nativeBuildInputs = [
           pkgs.gnumake
-          (pkgs.python3.withPackages(ps: (with artiqpkgs; [ ps.jsonschema migen migen-axi misoc artiq ])))
+          (pkgs.python3.withPackages(ps: [ ps.jsonschema artiqpkgs.migen migen-axi artiqpkgs.misoc artiqpkgs.artiq ]))
-          artiqpkgs.artiq
           zynqpkgs.cargo-xbuild
-          pkgs.llvmPackages_9.llvm
+          llvmPackages_11.llvm
-          pkgs.llvmPackages_9.clang-unwrapped
+          llvmPackages_11.clang-unwrapped
         ];
         buildPhase = ''
-          export XARGO_RUST_SRC="${rustPlatform.rust.rustc}/lib/rustlib/src/rust/library"
+          export XARGO_RUST_SRC="${rust}/lib/rustlib/src/rust/library"
-          export CLANG_EXTRA_INCLUDE_DIR="${pkgs.llvmPackages_9.clang-unwrapped.lib}/lib/clang/9.0.1/include"
+          export CLANG_EXTRA_INCLUDE_DIR="${llvmPackages_11.clang-unwrapped.lib}/lib/clang/11.1.0/include"
           export CARGO_HOME=$(mktemp -d cargo-home.XXX)
+          export ZYNQ_RS=${zynq-rs}
           make TARGET=${target} GWARGS="${if json == null then "-V ${variant}" else json}" ${fwtype}
         '';
 
@@ -163,12 +154,12 @@
 
         doCheck = false;
         dontFixup = true;
+        auditable = false;
       };
       gateware = pkgs.runCommand "${target}-${variant}-gateware"
         {
           nativeBuildInputs = [ 
-            (pkgs.python3.withPackages(ps: (with artiqpkgs; [ ps.jsonschema migen migen-axi misoc artiq ])))
+            (pkgs.python3.withPackages(ps: [ ps.jsonschema artiqpkgs.migen migen-axi artiqpkgs.misoc artiqpkgs.artiq ]))
-            artiqpkgs.artiq
             artiqpkgs.vivado
           ];
         }
@@ -252,8 +243,7 @@
       name = "gateware-sim";
       
       nativeBuildInputs = [ 
-        (pkgs.python3.withPackages(ps: (with artiqpkgs; [ migen migen-axi artiq ]))) 
+        (pkgs.python3.withPackages(ps: [ artiqpkgs.migen migen-axi artiqpkgs.artiq ]))
-        artiqpkgs.artiq
       ];
 
       phases = [ "buildPhase" ];
@@ -265,8 +255,26 @@
         '';
     };
 
+    fmt-check = pkgs.stdenvNoCC.mkDerivation {
+      name = "fmt-check";
+
+      src = ./src;
+
+      nativeBuildInputs = [ rust pkgs.gnumake ];
+
+      phases = [ "unpackPhase" "buildPhase" ];
+
+      buildPhase =
+        ''
+        export ZYNQ_RS=${zynq-rs}
+        make manifests
+        cargo fmt -- --check
+        touch $out
+        '';
+    };
+
     # for hitl-tests
-    zc706-nist_qc2 = (build { target = "zc706"; variant = "nist_qc2"; });
+    zc706-nist_qc2 = (board-package-set { target = "zc706"; variant = "nist_qc2"; });
     zc706-hitl-tests = pkgs.stdenv.mkDerivation {
       name = "zc706-hitl-tests";
 
@@ -329,35 +337,42 @@
         '';
     };
   in rec {
-    packages.x86_64-linux = (build { target = "zc706"; variant = "nist_clock"; }) //
+    packages.x86_64-linux =
-      (build { target = "zc706"; variant = "nist_clock_master"; }) //
+      {
-      (build { target = "zc706"; variant = "nist_clock_satellite"; }) //
+        inherit fastnumbers artiq-netboot ramda migen-axi binutils-arm;
-      (build { target = "zc706"; variant = "nist_clock_satellite_100mhz"; }) //
+      } //
-      (build { target = "zc706"; variant = "nist_qc2"; }) //
+      (board-package-set { target = "zc706"; variant = "nist_clock"; }) //
-      (build { target = "zc706"; variant = "nist_qc2_master"; }) //
+      (board-package-set { target = "zc706"; variant = "nist_clock_master"; }) //
-      (build { target = "zc706"; variant = "nist_qc2_satellite"; }) //
+      (board-package-set { target = "zc706"; variant = "nist_clock_master_100mhz"; }) //
-      (build { target = "zc706"; variant = "nist_qc2_satellite_100mhz"; }) //
+      (board-package-set { target = "zc706"; variant = "nist_clock_satellite"; }) //
-      (build { target = "zc706"; variant = "acpki_nist_clock"; }) //
+      (board-package-set { target = "zc706"; variant = "nist_clock_satellite_100mhz"; }) //
-      (build { target = "zc706"; variant = "acpki_nist_clock_master"; }) //
+      (board-package-set { target = "zc706"; variant = "nist_qc2"; }) //
-      (build { target = "zc706"; variant = "acpki_nist_clock_satellite"; }) //
+      (board-package-set { target = "zc706"; variant = "nist_qc2_master"; }) //
-      (build { target = "zc706"; variant = "acpki_nist_clock_satellite_100mhz"; }) //
+      (board-package-set { target = "zc706"; variant = "nist_qc2_master_100mhz"; }) //
-      (build { target = "zc706"; variant = "acpki_nist_qc2"; }) //
+      (board-package-set { target = "zc706"; variant = "nist_qc2_satellite"; }) //
-      (build { target = "zc706"; variant = "acpki_nist_qc2_master"; }) //
+      (board-package-set { target = "zc706"; variant = "nist_qc2_satellite_100mhz"; }) //
-      (build { target = "zc706"; variant = "acpki_nist_qc2_satellite"; }) //
+      (board-package-set { target = "zc706"; variant = "acpki_nist_clock"; }) //
-      (build { target = "zc706"; variant = "acpki_nist_qc2_satellite_100mhz"; }) //
+      (board-package-set { target = "zc706"; variant = "acpki_nist_clock_master"; }) //
-      (build { target = "kasli_soc"; variant = "demo"; json = ./demo.json; }) //
+      (board-package-set { target = "zc706"; variant = "acpki_nist_clock_master_100mhz"; }) //
-      (build { target = "kasli_soc"; variant = "master"; json = ./kasli-soc-master.json; }) //
+      (board-package-set { target = "zc706"; variant = "acpki_nist_clock_satellite"; }) //
-      (build { target = "kasli_soc"; variant = "satellite"; json = ./kasli-soc-satellite.json; });
+      (board-package-set { target = "zc706"; variant = "acpki_nist_clock_satellite_100mhz"; }) //
+      (board-package-set { target = "zc706"; variant = "acpki_nist_qc2"; }) //
+      (board-package-set { target = "zc706"; variant = "acpki_nist_qc2_master"; }) //
+      (board-package-set { target = "zc706"; variant = "acpki_nist_qc2_master_100mhz"; }) //
+      (board-package-set { target = "zc706"; variant = "acpki_nist_qc2_satellite"; }) //
+      (board-package-set { target = "zc706"; variant = "acpki_nist_qc2_satellite_100mhz"; }) //
+      (board-package-set { target = "kasli_soc"; variant = "demo"; json = ./demo.json; }) //
+      (board-package-set { target = "kasli_soc"; variant = "master"; json = ./kasli-soc-master.json; }) //
+      (board-package-set { target = "kasli_soc"; variant = "satellite"; json = ./kasli-soc-satellite.json; });
 
-    hydraJobs = packages.x86_64-linux // { inherit zc706-hitl-tests; inherit gateware-sim; };
+    hydraJobs = packages.x86_64-linux // { inherit zc706-hitl-tests; inherit gateware-sim; inherit fmt-check; };
 
     devShell.x86_64-linux = pkgs.mkShell {
       name = "artiq-zynq-dev-shell";
       buildInputs = with pkgs; [
-        rustPlatform.rust.rustc
+        rust
-        rustPlatform.rust.cargo
+        llvmPackages_11.llvm
-        llvmPackages_9.llvm
+        llvmPackages_11.clang-unwrapped
-        llvmPackages_9.clang-unwrapped
         gnumake
         cacert
         zynqpkgs.cargo-xbuild
@@ -369,13 +384,14 @@
         artiqpkgs.vivado
         binutils-arm
       ];
-      XARGO_RUST_SRC = "${rustPlatform.rust.rustc}/lib/rustlib/src/rust/library";
+      XARGO_RUST_SRC = "${rust}/lib/rustlib/src/rust/library";
-      CLANG_EXTRA_INCLUDE_DIR = "${pkgs.llvmPackages_9.clang-unwrapped.lib}/lib/clang/9.0.1/include";
+      CLANG_EXTRA_INCLUDE_DIR = "${llvmPackages_11.clang-unwrapped.lib}/lib/clang/11.1.0/include";
+      ZYNQ_RS = "${zynq-rs}";
       OPENOCD_ZYNQ = "${zynq-rs}/openocd";
       SZL = "${zynqpkgs.szl}";
     };
 
-    makeArtiqZynqPackage = build;
+    makeArtiqZynqPackage = board-package-set;
 
   };
 }

src/Cargo.lock

@@ -1,5 +1,13 @@
 # This file is automatically @generated by Cargo.
 # It is not intended for manual editing.
+version = 3
+
+[[package]]
+name = "arrayvec"
+version = "0.7.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711"
+
 [[package]]
 name = "async-recursion"
 version = "0.3.2"
@@ -47,9 +55,9 @@ checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610"
 
 [[package]]
 name = "cc"
-version = "1.0.73"
+version = "1.0.77"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11"
+checksum = "e9f73505338f7d905b19d18738976aae232eb46b8efc15554ffc56deb5d9ebe4"
 
 [[package]]
 name = "cfg-if"
@@ -136,9 +144,9 @@ dependencies = [
 
 [[package]]
 name = "futures"
-version = "0.3.21"
+version = "0.3.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f73fe65f54d1e12b726f517d3e2135ca3125a437b6d998caf1962961f7172d9e"
+checksum = "38390104763dc37a5145a53c29c63c1290b5d316d6086ec32c293f6736051bb0"
 dependencies = [
  "futures-channel",
  "futures-core",
@@ -150,9 +158,9 @@ dependencies = [
 
 [[package]]
 name = "futures-channel"
-version = "0.3.21"
+version = "0.3.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c3083ce4b914124575708913bca19bfe887522d6e2e6d0952943f5eac4a74010"
+checksum = "52ba265a92256105f45b719605a571ffe2d1f0fea3807304b522c1d778f79eed"
 dependencies = [
  "futures-core",
  "futures-sink",
@@ -160,21 +168,21 @@ dependencies = [
 
 [[package]]
 name = "futures-core"
-version = "0.3.21"
+version = "0.3.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0c09fd04b7e4073ac7156a9539b57a484a8ea920f79c7c675d05d289ab6110d3"
+checksum = "04909a7a7e4633ae6c4a9ab280aeb86da1236243a77b694a49eacd659a4bd3ac"
 
 [[package]]
 name = "futures-io"
-version = "0.3.21"
+version = "0.3.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fc4045962a5a5e935ee2fdedaa4e08284547402885ab326734432bed5d12966b"
+checksum = "00f5fb52a06bdcadeb54e8d3671f8888a39697dcb0b81b23b55174030427f4eb"
 
 [[package]]
 name = "futures-macro"
-version = "0.3.21"
+version = "0.3.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "33c1e13800337f4d4d7a316bf45a567dbcb6ffe087f16424852d97e97a91f512"
+checksum = "bdfb8ce053d86b91919aad980c220b1fb8401a9394410e1c289ed7e66b61835d"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -183,21 +191,21 @@ dependencies = [
 
 [[package]]
 name = "futures-sink"
-version = "0.3.21"
+version = "0.3.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "21163e139fa306126e6eedaf49ecdb4588f939600f0b1e770f4205ee4b7fa868"
+checksum = "39c15cf1a4aa79df40f1bb462fb39676d0ad9e366c2a33b590d7c66f4f81fcf9"
 
 [[package]]
 name = "futures-task"
-version = "0.3.21"
+version = "0.3.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "57c66a976bf5909d801bbef33416c41372779507e7a6b3a5e25e4749c58f776a"
+checksum = "2ffb393ac5d9a6eaa9d3fdf37ae2776656b706e200c8e16b1bdb227f5198e6ea"
 
 [[package]]
 name = "futures-util"
-version = "0.3.21"
+version = "0.3.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d8b7abd5d659d9b90c8cba917f6ec750a74e2dc23902ef9cd4cc8c8b22e6036a"
+checksum = "197676987abd2f9cadff84926f410af1c183608d36641465df73ae8211dc65d6"
 dependencies = [
  "futures-core",
  "futures-macro",
@@ -216,10 +224,37 @@ dependencies = [
  "libsupport_zynq",
 ]
 
+[[package]]
+name = "ksupport"
+version = "0.1.0"
+dependencies = [
+ "build_zynq",
+ "byteorder",
+ "core_io",
+ "cslice",
+ "dwarf",
+ "dyld",
+ "io",
+ "libasync",
+ "libboard_artiq",
+ "libboard_zynq",
+ "libc",
+ "libconfig",
+ "libcortex_a9",
+ "libm",
+ "libregister",
+ "libsupport_zynq",
+ "log",
+ "log_buffer",
+ "nb 0.1.3",
+ "unwind",
+ "vcell",
+ "void",
+]
+
 [[package]]
 name = "libasync"
 version = "0.0.0"
-source = "git+https://git.m-labs.hk/M-Labs/zynq-rs.git#91bae572f913abc2f95acb899ca5afa33eeaa036"
 dependencies = [
  "embedded-hal",
  "libcortex_a9",
@@ -242,6 +277,7 @@ dependencies = [
  "libconfig",
  "libcortex_a9",
  "libregister",
+ "libsupport_zynq",
  "log",
  "log_buffer",
  "nb 1.0.0",
@@ -251,7 +287,6 @@ dependencies = [
 [[package]]
 name = "libboard_zynq"
 version = "0.0.0"
-source = "git+https://git.m-labs.hk/M-Labs/zynq-rs.git#91bae572f913abc2f95acb899ca5afa33eeaa036"
 dependencies = [
  "bit_field",
  "embedded-hal",
@@ -276,7 +311,6 @@ dependencies = [
 [[package]]
 name = "libconfig"
 version = "0.1.0"
-source = "git+https://git.m-labs.hk/M-Labs/zynq-rs.git#91bae572f913abc2f95acb899ca5afa33eeaa036"
 dependencies = [
  "core_io",
  "fatfs",
@@ -287,7 +321,6 @@ dependencies = [
 [[package]]
 name = "libcortex_a9"
 version = "0.0.0"
-source = "git+https://git.m-labs.hk/M-Labs/zynq-rs.git#91bae572f913abc2f95acb899ca5afa33eeaa036"
 dependencies = [
  "bit_field",
  "libregister",
@@ -296,14 +329,13 @@ dependencies = [
 
 [[package]]
 name = "libm"
-version = "0.2.2"
+version = "0.2.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "33a33a362ce288760ec6a508b94caaec573ae7d3bbbd91b87aa0bad4456839db"
+checksum = "348108ab3fba42ec82ff6e9564fc4ca0247bdccdc68dd8af9764bbc79c3c8ffb"
 
 [[package]]
 name = "libregister"
 version = "0.0.0"
-source = "git+https://git.m-labs.hk/M-Labs/zynq-rs.git#91bae572f913abc2f95acb899ca5afa33eeaa036"
 dependencies = [
  "bit_field",
  "vcell",
@@ -313,7 +345,6 @@ dependencies = [
 [[package]]
 name = "libsupport_zynq"
 version = "0.0.0"
-source = "git+https://git.m-labs.hk/M-Labs/zynq-rs.git#91bae572f913abc2f95acb899ca5afa33eeaa036"
 dependencies = [
  "cc",
  "compiler_builtins",
@@ -332,9 +363,9 @@ checksum = "822add9edb1860698b79522510da17bef885171f75aa395cff099d770c609c24"
 
 [[package]]
 name = "log"
-version = "0.4.14"
+version = "0.4.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710"
+checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e"
 dependencies = [
  "cfg-if 1.0.0",
 ]
@@ -379,18 +410,18 @@ dependencies = [
 
 [[package]]
 name = "num-traits"
-version = "0.2.14"
+version = "0.2.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290"
+checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd"
 dependencies = [
  "autocfg",
 ]
 
 [[package]]
 name = "pin-project-lite"
-version = "0.2.8"
+version = "0.2.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e280fbe77cc62c91527259e9442153f4688736748d24660126286329742b4c6c"
+checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116"
 
 [[package]]
 name = "pin-utils"
@@ -400,18 +431,18 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
 
 [[package]]
 name = "proc-macro2"
-version = "1.0.36"
+version = "1.0.43"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c7342d5883fbccae1cc37a2353b09c87c9b0f3afd73f5fb9bba687a1f733b029"
+checksum = "0a2ca2c61bc9f3d74d2886294ab7b9853abd9c1ad903a3ac7815c58989bb7bab"
 dependencies = [
- "unicode-xid",
+ "unicode-ident",
 ]
 
 [[package]]
 name = "quote"
-version = "1.0.15"
+version = "1.0.21"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "864d3e96a899863136fc6e99f3d7cae289dafe43bf2c5ac19b70df7210c0a145"
+checksum = "bbe448f377a7d6961e30f5955f9b8d106c3f5e449d493ee1b125c1d43c2b5179"
 dependencies = [
  "proc-macro2",
 ]
@@ -436,20 +467,20 @@ dependencies = [
  "embedded-hal",
  "futures",
  "io",
+ "ksupport",
  "libasync",
  "libboard_artiq",
  "libboard_zynq",
  "libc",
  "libconfig",
  "libcortex_a9",
- "libm",
  "libregister",
  "libsupport_zynq",
  "log",
  "log_buffer",
- "nb 0.1.3",
  "num-derive",
  "num-traits",
+ "tar-no-std",
  "unwind",
  "vcell",
  "void",
@@ -469,7 +500,11 @@ name = "satman"
 version = "0.0.0"
 dependencies = [
  "build_zynq",
+ "core_io",
+ "cslice",
  "embedded-hal",
+ "io",
+ "ksupport",
  "libasync",
  "libboard_artiq",
  "libboard_zynq",
@@ -501,20 +536,30 @@ dependencies = [
 
 [[package]]
 name = "syn"
-version = "1.0.86"
+version = "1.0.101"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8a65b3f4ffa0092e9887669db0eae07941f023991ab58ea44da8fe8e2d511c6b"
+checksum = "e90cde112c4b9690b8cbe810cba9ddd8bc1d7472e2cae317b69e9438c1cba7d2"
 dependencies = [
  "proc-macro2",
  "quote",
- "unicode-xid",
+ "unicode-ident",
 ]
 
 [[package]]
-name = "unicode-xid"
+name = "tar-no-std"
-version = "0.2.2"
+version = "0.1.8"
+source = "git+https://git.m-labs.hk/M-Labs/tar-no-std?rev=2ab6dc5#2ab6dc58e5249c59c4eb03eaf3a119bcdd678d32"
+dependencies = [
+ "arrayvec",
+ "bitflags",
+ "log",
+]
+
+[[package]]
+name = "unicode-ident"
+version = "1.0.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3"
+checksum = "6ceab39d59e4c9499d4e5a8ee0e2735b891bb7308ac83dfb4e80cad195c9f6f3"
 
 [[package]]
 name = "unwind"

src/Cargo.toml

@@ -5,6 +5,7 @@ members = [
   "libdwarf",
   "libio",
   "libunwind",
+  "libksupport",
   "runtime",
   "satman"
 ]

src/Makefile

@@ -7,13 +7,20 @@ runtime: ../build/runtime.bin
 
 satman: ../build/satman.bin
 
-.PHONY: all runtime_target satman_target
+.PHONY: all manifests
+
+manifests = libboard_artiq/Cargo.toml libc/Cargo.toml libdyld/Cargo.toml libio/Cargo.toml libksupport/Cargo.toml runtime/Cargo.toml satman/Cargo.toml
+
+$(manifests): %.toml: %.toml.tpl
+	sed s+@@ZYNQ_RS@@+$(ZYNQ_RS)+g $< > $@
+
+manifests: $(manifests)
 
 ../build/pl.rs ../build/rustc-cfg ../build/mem.rs: gateware/* 
 	mkdir -p ../build
 	python gateware/$(TARGET).py -r ../build/pl.rs -c ../build/rustc-cfg -m ../build/mem.rs $(GWARGS)
 
-../build/firmware/armv7-none-eabihf/release/runtime: ../build/pl.rs ../build/rustc-cfg ../build/mem.rs $(shell find . -type f -print)
+../build/firmware/armv7-none-eabihf/release/runtime: ../build/pl.rs ../build/rustc-cfg ../build/mem.rs $(manifests) $(shell find . -type f -not -name Cargo.toml -print)
 	cd runtime && \
 	XBUILD_SYSROOT_PATH=`pwd`/../../build/sysroot \
 	cargo xbuild --release \
@@ -23,7 +30,7 @@ satman: ../build/satman.bin
 ../build/runtime.bin: ../build/firmware/armv7-none-eabihf/release/runtime
 	llvm-objcopy -O binary ../build/firmware/armv7-none-eabihf/release/runtime ../build/runtime.bin
 
-../build/firmware/armv7-none-eabihf/release/satman: ../build/pl.rs ../build/rustc-cfg ../build/mem.rs $(shell find . -type f -print)
+../build/firmware/armv7-none-eabihf/release/satman: ../build/pl.rs ../build/rustc-cfg ../build/mem.rs $(manifests) $(shell find . -type f -not -name Cargo.toml -print)
 	cd satman && \
 	XBUILD_SYSROOT_PATH=`pwd`/../../build/sysroot \
 	cargo xbuild --release \

src/gateware/config.py

@@ -0,0 +1,16 @@
+from misoc.integration import cpu_interface
+
+def write_csr_file(soc, filename):
+    with open(filename, "w") as f:
+        f.write(cpu_interface.get_csr_rust(
+            soc.get_csr_regions(), soc.get_csr_groups(), soc.get_constants()))
+
+def write_mem_file(soc, filename):
+    with open(filename, "w") as f:
+        f.write(cpu_interface.get_mem_rust(
+            soc.get_memory_regions(), soc.get_memory_groups(), None))
+
+def write_rustc_cfg_file(soc, filename):
+    with open(filename, "w") as f:
+        f.write(cpu_interface.get_rust_cfg(
+            soc.get_csr_regions(), soc.get_constants()))

src/gateware/ddmtd.py

@@ -0,0 +1,119 @@
+from migen import *
+from migen.genlib.cdc import PulseSynchronizer, MultiReg
+from misoc.interconnect.csr import *
+
+
+class DDMTDSampler(Module):
+    def __init__(self, cd_ref, main_clk_se):
+        self.ref_beating = Signal()
+        self.main_beating = Signal()
+
+        # # #
+
+        ref_clk = Signal()
+        self.specials +=[
+            # ISERDESE2 can only be driven from fabric via IDELAYE2 (see UG471)
+            Instance("IDELAYE2",
+                    p_DELAY_SRC="DATAIN",
+                    p_HIGH_PERFORMANCE_MODE="TRUE",
+                    p_REFCLK_FREQUENCY=208.3,   # REFCLK frequency from IDELAYCTRL
+                    p_IDELAY_VALUE=0,
+
+                    i_DATAIN=cd_ref.clk,
+
+                    o_DATAOUT=ref_clk
+            ),
+            Instance("ISERDESE2",
+                    p_IOBDELAY="IFD",   # use DDLY as input
+                    p_DATA_RATE="SDR",
+                    p_DATA_WIDTH=2,     # min is 2
+                    p_NUM_CE=1,
+
+                    i_DDLY=ref_clk,
+                    i_CE1=1,
+                    i_CLK=ClockSignal("helper"),
+                    i_CLKDIV=ClockSignal("helper"),
+
+                    o_Q1=self.ref_beating
+            ),
+            Instance("ISERDESE2",
+                    p_DATA_RATE="SDR",
+                    p_DATA_WIDTH=2,     # min is 2
+                    p_NUM_CE=1,
+
+                    i_D=main_clk_se,
+                    i_CE1=1,
+                    i_CLK=ClockSignal("helper"),
+                    i_CLKDIV=ClockSignal("helper"),
+
+                    o_Q1=self.main_beating,
+            ),
+        ]
+
+
+class DDMTDDeglitcherMedianEdge(Module):
+    def __init__(self, counter, input_signal, stable_0_period=100, stable_1_period=100):
+        self.tag = Signal(len(counter))
+        self.detect = Signal()
+
+        stable_0_counter = Signal(reset=stable_0_period - 1, max=stable_0_period)
+        stable_1_counter = Signal(reset=stable_1_period - 1, max=stable_1_period)
+
+        # # #
+
+        # Based on CERN's median edge deglitcher FSM
+        # https://white-rabbit.web.cern.ch/documents/Precise_time_and_frequency_transfer_in_a_White_Rabbit_network.pdf (p.72)
+        fsm = ClockDomainsRenamer("helper")(FSM(reset_state="WAIT_STABLE_0"))
+        self.submodules += fsm
+
+        fsm.act("WAIT_STABLE_0",
+            If(stable_0_counter != 0,
+                NextValue(stable_0_counter, stable_0_counter - 1)
+            ).Else(
+                NextValue(stable_0_counter, stable_0_period - 1),
+                NextState("WAIT_EDGE")
+            ),
+            If(input_signal,
+                NextValue(stable_0_counter, stable_0_period - 1)
+            ),
+        )
+        fsm.act("WAIT_EDGE",
+            If(input_signal,
+                NextValue(self.tag, counter),
+                NextState("GOT_EDGE")
+            )
+        )
+        fsm.act("GOT_EDGE",
+            If(stable_1_counter != 0,
+                NextValue(stable_1_counter, stable_1_counter - 1)
+            ).Else(
+                NextValue(stable_1_counter, stable_1_period - 1),
+                self.detect.eq(1),
+                NextState("WAIT_STABLE_0")
+            ),
+            If(~input_signal,
+                NextValue(self.tag, self.tag + 1),
+                NextValue(stable_1_counter, stable_1_period - 1)
+            ),
+        )
+        
+
+class DDMTD(Module):
+    def __init__(self, counter, input_signal):
+
+        # in helper clock domain
+        self.h_tag = Signal(len(counter))
+        self.h_tag_update = Signal()
+
+        # # #
+
+        deglitcher = DDMTDDeglitcherMedianEdge(counter, input_signal)
+        self.submodules += deglitcher
+
+        self.sync.helper += [
+            self.h_tag_update.eq(0),
+            If(deglitcher.detect,
+                self.h_tag_update.eq(1),
+                self.h_tag.eq(deglitcher.tag)
+               )
+        ]

src/gateware/drtio_aux_controller.py

@@ -1,12 +1,12 @@
 """Auxiliary controller, common to satellite and master"""
 
-from artiq.gateware.drtio.aux_controller import Transmitter, Receiver
+from artiq.gateware.drtio.aux_controller import (max_packet, aux_buffer_count,
+    Transmitter, Receiver)
 from migen.fhdl.simplify import FullMemoryWE
 from misoc.interconnect.csr import *
 from migen_axi.interconnect.sram import SRAM
 from migen_axi.interconnect import axi
 
-max_packet = 1024
 
 class _DRTIOAuxControllerBase(Module):
     def __init__(self, link_layer):
@@ -27,12 +27,12 @@ class DRTIOAuxControllerAxi(_DRTIOAuxControllerBase):
         tx_sdram_if = SRAM(self.transmitter.mem, read_only=False)
         rx_sdram_if = SRAM(self.receiver.mem, read_only=True)
         aw_decoder = axi.AddressDecoder(self.bus.aw,
-            [(lambda a: a[log2_int(max_packet)] == 0, tx_sdram_if.bus.aw),
+            [(lambda a: a[log2_int(max_packet*aux_buffer_count)] == 0, tx_sdram_if.bus.aw),
-             (lambda a: a[log2_int(max_packet)] == 1, rx_sdram_if.bus.aw)],
+             (lambda a: a[log2_int(max_packet*aux_buffer_count)] == 1, rx_sdram_if.bus.aw)],
             register=True)
         ar_decoder = axi.AddressDecoder(self.bus.ar,
-            [(lambda a: a[log2_int(max_packet)] == 0, tx_sdram_if.bus.ar),
+            [(lambda a: a[log2_int(max_packet*aux_buffer_count)] == 0, tx_sdram_if.bus.ar),
-             (lambda a: a[log2_int(max_packet)] == 1, rx_sdram_if.bus.ar)],
+             (lambda a: a[log2_int(max_packet*aux_buffer_count)] == 1, rx_sdram_if.bus.ar)],
             register=True)
         # unlike wb, axi address decoder only connects ar/aw lanes,
         # the rest must also be connected!
@@ -82,4 +82,4 @@ class DRTIOAuxControllerBare(_DRTIOAuxControllerBase):
         return self.receiver.mem.get_port(write_capable=False)
 
     def get_mem_size(self):
-        return max_packet
+        return max_packet*aux_buffer_count

src/gateware/kasli_soc.py

@@ -11,72 +11,23 @@ from migen_axi.integration.soc_core import SoCCore
 from migen_axi.platforms import kasli_soc
 from misoc.interconnect.csr import *
 from misoc.cores import virtual_leds
-from misoc.integration import cpu_interface
 
 from artiq.coredevice import jsondesc
 from artiq.gateware import rtio, eem_7series
-from artiq.gateware.rtio.phy import ttl_simple
 from artiq.gateware.rtio.xilinx_clocking import fix_serdes_timing_path
-from artiq.gateware.rtio.xilinx_clocking import RTIOClockMultiplier
+from artiq.gateware.rtio.phy import ttl_simple
-from artiq.gateware.drtio.transceiver import gtx_7series
+from artiq.gateware.drtio.transceiver import gtx_7series, eem_serdes
 from artiq.gateware.drtio.siphaser import SiPhaser7Series
 from artiq.gateware.drtio.rx_synchronizer import XilinxRXSynchronizer
 from artiq.gateware.drtio import *
+from artiq.gateware.wrpll import wrpll
 
 import dma
 import analyzer
 import acpki
 import drtio_aux_controller
-
+import zynq_clocking
-class RTIOCRG(Module, AutoCSR):
+from config import write_csr_file, write_mem_file, write_rustc_cfg_file
-    def __init__(self, platform):
-        self.pll_reset = CSRStorage(reset=1)
-        self.pll_locked = CSRStatus()
-        self.clock_domains.cd_rtio = ClockDomain()
-        self.clock_domains.cd_rtiox4 = ClockDomain(reset_less=True)
-
-        clk_synth = platform.request("cdr_clk_clean_fabric")
-        clk_synth_se = Signal()
-        platform.add_period_constraint(clk_synth.p, 8.0)
-        self.specials += [
-            Instance("IBUFGDS",
-                p_DIFF_TERM="TRUE", p_IBUF_LOW_PWR="FALSE",
-                i_I=clk_synth.p, i_IB=clk_synth.n, o_O=clk_synth_se),
-        ]
-
-        pll_locked = Signal()
-        rtio_clk = Signal()
-        rtiox4_clk = Signal()
-        fb_clk = Signal()
-        self.specials += [
-            Instance("PLLE2_ADV",
-                     p_STARTUP_WAIT="FALSE", o_LOCKED=pll_locked,
-                     p_BANDWIDTH="HIGH",
-                     p_REF_JITTER1=0.001,
-                     p_CLKIN1_PERIOD=8.0, p_CLKIN2_PERIOD=8.0,
-                     i_CLKIN2=clk_synth_se,
-                     # Warning: CLKINSEL=0 means CLKIN2 is selected
-                     i_CLKINSEL=0,
-
-                     # VCO @ 1.5GHz when using 125MHz input
-                     p_CLKFBOUT_MULT=12, p_DIVCLK_DIVIDE=1,
-                     i_CLKFBIN=fb_clk,
-                     i_RST=self.pll_reset.storage,
-
-                     o_CLKFBOUT=fb_clk,
-
-                     p_CLKOUT0_DIVIDE=3, p_CLKOUT0_PHASE=0.0,
-                     o_CLKOUT0=rtiox4_clk,
-
-                     p_CLKOUT1_DIVIDE=12, p_CLKOUT1_PHASE=0.0,
-                     o_CLKOUT1=rtio_clk),
-            Instance("BUFG", i_I=rtio_clk, o_O=self.cd_rtio.clk),
-            Instance("BUFG", i_I=rtiox4_clk, o_O=self.cd_rtiox4.clk),
-
-            AsyncResetSynchronizer(self.cd_rtio, ~pll_locked),
-            MultiReg(pll_locked, self.pll_locked.status)
-        ]
-
 
 eem_iostandard_dict = {
      0: "LVDS_25",
@@ -111,10 +62,54 @@ class SMAClkinForward(Module):
         ]
 
 
+class GTPBootstrapClock(Module):
+    def __init__(self, platform, freq=125e6):
+        self.clock_domains.cd_bootstrap = ClockDomain(reset_less=True)
+        self.cd_bootstrap.clk.attr.add("keep")
+
+        bootstrap_125 = platform.request("clk125_gtp")
+        bootstrap_se = Signal()
+        clk_out = Signal()
+        platform.add_period_constraint(bootstrap_125.p, 8.0)
+        self.specials += [
+            Instance("IBUFDS_GTE2",
+                i_CEB=0,
+                i_I=bootstrap_125.p, i_IB=bootstrap_125.n, 
+                o_O=bootstrap_se,
+                p_CLKCM_CFG="TRUE",
+                p_CLKRCV_TRST="TRUE",
+                p_CLKSWING_CFG=3),
+            Instance("BUFG", i_I=bootstrap_se, o_O=clk_out)
+        ]
+        if freq == 125e6:
+            self.comb += self.cd_bootstrap.clk.eq(clk_out)
+        elif freq == 100e6:
+            pll_fb = Signal()
+            pll_out = Signal()
+            self.specials += [
+                Instance("PLLE2_BASE",
+                    p_CLKIN1_PERIOD=8.0,
+                    i_CLKIN1=clk_out,
+                    i_CLKFBIN=pll_fb,
+                    o_CLKFBOUT=pll_fb,
+
+                    # VCO @ 1GHz
+                    p_CLKFBOUT_MULT=8, p_DIVCLK_DIVIDE=1,
+
+                    # 100MHz for bootstrap
+                    p_CLKOUT1_DIVIDE=10, p_CLKOUT1_PHASE=0.0, o_CLKOUT1=pll_out,
+                ),
+                Instance("BUFG", i_I=pll_out, o_O=self.cd_bootstrap.clk)
+            ]
+        else:
+            raise ValueError("Bootstrap frequency must be 100 or 125MHz")
+
+
 class GenericStandalone(SoCCore):
     def __init__(self, description, acpki=False):
         self.acpki = acpki
-        sys_clk_freq = 125e6
+        clk_freq = description["rtio_frequency"]
+        with_wrpll = description["enable_wrpll"]
 
         platform = kasli_soc.Platform()
         platform.toolchain.bitstream_commands.extend([
@@ -123,27 +118,49 @@ class GenericStandalone(SoCCore):
         ident = description["variant"]
         if self.acpki:
             ident = "acpki_" + ident
-        SoCCore.__init__(self, platform=platform, csr_data_width=32, ident=ident)
+        SoCCore.__init__(self, platform=platform, csr_data_width=32, ident=ident, ps_cd_sys=False)
 
-        self.config["hw_rev"] = description["hw_rev"]
+        self.config["HW_REV"] = description["hw_rev"]
+        clk_synth = platform.request("cdr_clk_clean_fabric")
+        clk_synth_se = Signal()
+        clk_synth_se_buf = Signal()
+        platform.add_period_constraint(clk_synth.p, 8.0)
 
-        platform.add_platform_command("create_clock -name clk_fpga_0 -period 8 [get_pins \"PS7/FCLKCLK[0]\"]")
+        self.specials += [ 
-        platform.add_platform_command("set_input_jitter clk_fpga_0 0.24")
+            Instance("IBUFGDS",
-
+                p_DIFF_TERM="TRUE", p_IBUF_LOW_PWR="FALSE",
-        self.submodules += SMAClkinForward(self.platform)
+                i_I=clk_synth.p, i_IB=clk_synth.n, o_O=clk_synth_se
-
+            ),
-        self.config["HAS_SI5324"] = None
+            Instance("BUFG", i_I=clk_synth_se, o_O=clk_synth_se_buf),
-        self.config["SI5324_SOFT_RESET"] = None
+        ]
-
-        self.crg = self.ps7 # HACK for eem_7series to find the clock
-        self.submodules.rtio_crg = RTIOCRG(self.platform)
-        self.csr_devices.append("rtio_crg")
-        self.platform.add_period_constraint(self.rtio_crg.cd_rtio.clk, 8.)
-        self.platform.add_false_path_constraints(
-            self.ps7.cd_sys.clk,
-            self.rtio_crg.cd_rtio.clk)
         fix_serdes_timing_path(platform)
-        self.config["CLOCK_FREQUENCY"] = int(sys_clk_freq)
+        self.submodules.bootstrap = GTPBootstrapClock(self.platform, clk_freq)
+        self.config["RTIO_FREQUENCY"] = str(clk_freq/1e6)
+        self.config["CLOCK_FREQUENCY"] = int(clk_freq)
+
+        self.submodules.sys_crg = zynq_clocking.SYSCRG(self.platform, self.ps7, clk_synth_se_buf)
+        platform.add_false_path_constraints(
+            self.bootstrap.cd_bootstrap.clk, self.sys_crg.cd_sys.clk)
+        self.csr_devices.append("sys_crg")
+        self.crg = self.ps7 # HACK for eem_7series to find the clock
+        self.crg.cd_sys = self.sys_crg.cd_sys
+
+        if with_wrpll:
+            self.submodules.wrpll_refclk = wrpll.FrequencyMultiplier(platform.request("sma_clkin"))
+            self.submodules.wrpll = wrpll.WRPLL(
+                platform=self.platform,
+                cd_ref=self.wrpll_refclk.cd_ref,
+                main_clk_se=clk_synth_se)
+            self.csr_devices.append("wrpll_refclk")
+            self.csr_devices.append("wrpll")
+            self.comb += self.ps7.core.core0.nfiq.eq(self.wrpll.ev.irq)
+            self.config["HAS_SI549"] = None
+            self.config["WRPLL_REF_CLK"] = "SMA_CLKIN"
+        else:
+            self.submodules += SMAClkinForward(self.platform)
+            self.config["HAS_SI5324"] = None
+            self.config["SI5324_SOFT_RESET"] = None
+
 
         self.rtio_channels = []
         has_grabber = any(peripheral["type"] == "grabber" for peripheral in description["peripherals"])
@@ -159,7 +176,7 @@ class GenericStandalone(SoCCore):
         self.config["RTIO_LOG_CHANNEL"] = len(self.rtio_channels)
         self.rtio_channels.append(rtio.LogChannel())
 
-        self.submodules.rtio_tsc = rtio.TSC("async", glbl_fine_ts_width=3)
+        self.submodules.rtio_tsc = rtio.TSC(glbl_fine_ts_width=3)
         self.submodules.rtio_core = rtio.Core(
             self.rtio_tsc, self.rtio_channels, lane_count=description["sed_lanes"]
         )
@@ -197,14 +214,15 @@ class GenericStandalone(SoCCore):
             self.add_csr_group("grabber", self.grabber_csr_group)
             for grabber in self.grabber_csr_group:
                 self.platform.add_false_path_constraints(
-                    self.rtio_crg.cd_rtio.clk, getattr(self, grabber).deserializer.cd_cl.clk)
+                    self.sys_crg.cd_sys.clk, getattr(self, grabber).deserializer.cd_cl.clk)
 
 
 class GenericMaster(SoCCore):
     def __init__(self, description, acpki=False):
-        sys_clk_freq = 125e6
+        clk_freq = description["rtio_frequency"]
-        rtio_clk_freq = description["rtio_frequency"]
+        with_wrpll = description["enable_wrpll"]
 
+        has_drtio_over_eem = any(peripheral["type"] == "shuttler" for peripheral in description["peripherals"])
         self.acpki = acpki
 
         platform = kasli_soc.Platform()
@@ -214,34 +232,68 @@ class GenericMaster(SoCCore):
         ident = description["variant"]
         if self.acpki:
             ident = "acpki_" + ident
-        SoCCore.__init__(self, platform=platform, csr_data_width=32, ident=ident)
+        SoCCore.__init__(self, platform=platform, csr_data_width=32, ident=ident, ps_cd_sys=False)
 
-        self.config["hw_rev"] = description["hw_rev"]
+        self.config["HW_REV"] = description["hw_rev"]
-
-        platform.add_platform_command("create_clock -name clk_fpga_0 -period 8 [get_pins \"PS7/FCLKCLK[0]\"]")
-        platform.add_platform_command("set_input_jitter clk_fpga_0 0.24")
-
-        self.submodules += SMAClkinForward(self.platform)
 
         data_pads = [platform.request("sfp", i) for i in range(4)]
 
-        self.submodules.drtio_transceiver = gtx_7series.GTX(
+        self.submodules.gt_drtio = gtx_7series.GTX(
             clock_pads=platform.request("clk_gtp"),
             pads=data_pads,
-            sys_clk_freq=sys_clk_freq)
+            clk_freq=clk_freq)
-        self.csr_devices.append("drtio_transceiver")
+        self.csr_devices.append("gt_drtio")
-        self.config["CLOCK_FREQUENCY"] = int(sys_clk_freq)
+        self.config["RTIO_FREQUENCY"] = str(clk_freq/1e6)
+        self.config["CLOCK_FREQUENCY"] = int(clk_freq)
 
+        txout_buf = Signal()
+        gtx0 = self.gt_drtio.gtxs[0]
+        self.specials += Instance("BUFG", i_I=gtx0.txoutclk, o_O=txout_buf)
+
+        ext_async_rst = Signal()
+
+        self.submodules.bootstrap = GTPBootstrapClock(self.platform, clk_freq)
+        self.submodules.sys_crg = zynq_clocking.SYSCRG(
+            self.platform,
+            self.ps7,
+            txout_buf,
+            clk_sw=self.gt_drtio.stable_clkin.storage,
+            clk_sw_status=gtx0.tx_init.done,
+            ext_async_rst=ext_async_rst)
+        self.csr_devices.append("sys_crg")
         self.crg = self.ps7 # HACK for eem_7series to find the clock
-        self.submodules.rtio_crg = RTIOClockMultiplier(rtio_clk_freq)
+        self.crg.cd_sys = self.sys_crg.cd_sys
-        self.csr_devices.append("rtio_crg")
+        platform.add_false_path_constraints(
+            self.bootstrap.cd_bootstrap.clk, self.sys_crg.cd_sys.clk)
         fix_serdes_timing_path(platform)
 
-        self.config["HAS_SI5324"] = None
+        self.comb += ext_async_rst.eq(self.sys_crg.clk_sw_fsm.o_clk_sw & ~gtx0.tx_init.done)
-        self.config["SI5324_SOFT_RESET"] = None
+        self.specials += MultiReg(self.sys_crg.clk_sw_fsm.o_clk_sw & self.sys_crg.mmcm_locked, self.gt_drtio.clk_path_ready, odomain="bootstrap")
+
+        if with_wrpll:
+            clk_synth = platform.request("cdr_clk_clean_fabric")
+            clk_synth_se = Signal()
+            platform.add_period_constraint(clk_synth.p, 8.0)
+            self.specials += Instance("IBUFGDS", p_DIFF_TERM="TRUE", p_IBUF_LOW_PWR="FALSE", i_I=clk_synth.p, i_IB=clk_synth.n, o_O=clk_synth_se)
+            self.submodules.wrpll_refclk = wrpll.FrequencyMultiplier(platform.request("sma_clkin"))
+            self.submodules.wrpll = wrpll.WRPLL(
+                platform=self.platform,
+                cd_ref=self.wrpll_refclk.cd_ref,
+                main_clk_se=clk_synth_se)
+            self.csr_devices.append("wrpll_refclk")
+            self.csr_devices.append("wrpll")
+            self.comb += self.ps7.core.core0.nfiq.eq(self.wrpll.ev.irq)
+            self.config["HAS_SI549"] = None
+            self.config["WRPLL_REF_CLK"] = "SMA_CLKIN"
+        else:
+            self.submodules += SMAClkinForward(self.platform)
+            self.config["HAS_SI5324"] = None
+            self.config["SI5324_SOFT_RESET"] = None
 
         self.rtio_channels = []
         has_grabber = any(peripheral["type"] == "grabber" for peripheral in description["peripherals"])
+        if has_drtio_over_eem:
+            self.eem_drtio_channels = []
         if has_grabber:
             self.grabber_csr_group = []
         eem_7series.add_peripherals(self, description["peripherals"], iostandard=eem_iostandard)
@@ -254,23 +306,23 @@ class GenericMaster(SoCCore):
         self.config["RTIO_LOG_CHANNEL"] = len(self.rtio_channels)
         self.rtio_channels.append(rtio.LogChannel())
 
-        self.submodules.rtio_tsc = rtio.TSC("async", glbl_fine_ts_width=3)
+        self.submodules.rtio_tsc = rtio.TSC(glbl_fine_ts_width=3)
 
-        drtio_csr_group = []
+        self.drtio_csr_group = []
-        drtioaux_csr_group = []
+        self.drtioaux_csr_group = []
-        drtioaux_memory_group = []
+        self.drtioaux_memory_group = []
         self.drtio_cri = []
-        for i in range(len(self.drtio_transceiver.channels)):
+        for i in range(len(self.gt_drtio.channels)):
             core_name = "drtio" + str(i)
             coreaux_name = "drtioaux" + str(i)
             memory_name = "drtioaux" + str(i) + "_mem"
-            drtio_csr_group.append(core_name)
+            self.drtio_csr_group.append(core_name)
-            drtioaux_csr_group.append(coreaux_name)
+            self.drtioaux_csr_group.append(coreaux_name)
-            drtioaux_memory_group.append(memory_name)
+            self.drtioaux_memory_group.append(memory_name)
 
             cdr = ClockDomainsRenamer({"rtio_rx": "rtio_rx" + str(i)})
 
-            core = cdr(DRTIOMaster(self.rtio_tsc, self.drtio_transceiver.channels[i]))
+            core = cdr(DRTIOMaster(self.rtio_tsc, self.gt_drtio.channels[i]))
             setattr(self.submodules, core_name, core)
             self.drtio_cri.append(core.cri)
             self.csr_devices.append(core_name)
@@ -285,9 +337,10 @@ class GenericMaster(SoCCore):
             self.add_memory_region(memory_name, self.mem_map["csr"] + memory_address, size * 2)
         self.config["HAS_DRTIO"] = None
         self.config["HAS_DRTIO_ROUTING"] = None
-        self.add_csr_group("drtio", drtio_csr_group)
+
-        self.add_csr_group("drtioaux", drtioaux_csr_group)
+        if has_drtio_over_eem:
-        self.add_memory_group("drtioaux_mem", drtioaux_memory_group)
+            self.add_eem_drtio(self.eem_drtio_channels)
+        self.add_drtio_cpuif_groups()
 
         self.submodules.rtio_core = rtio.Core(
             self.rtio_tsc, self.rtio_channels, lane_count=description["sed_lanes"]
@@ -321,7 +374,7 @@ class GenericMaster(SoCCore):
         self.submodules.routing_table = rtio.RoutingTableAccess(self.cri_con)
         self.csr_devices.append("routing_table")
 
-        self.submodules.rtio_analyzer = analyzer.Analyzer(self.rtio_tsc, self.cri_con.switch.slave,
+        self.submodules.rtio_analyzer = analyzer.Analyzer(self.rtio_tsc, self.rtio_core.cri,
                                                           self.ps7.s_axi_hp1)
         self.csr_devices.append("rtio_analyzer")
 
@@ -329,16 +382,55 @@ class GenericMaster(SoCCore):
             self.config["HAS_GRABBER"] = None
             self.add_csr_group("grabber", self.grabber_csr_group)
         
+
         self.submodules.virtual_leds = virtual_leds.VirtualLeds()
         self.csr_devices.append("virtual_leds")
 
         self.comb += [self.virtual_leds.get(i).eq(channel.rx_ready)
-                for i, channel in enumerate(self.drtio_transceiver.channels)]
+                for i, channel in enumerate(self.gt_drtio.channels)]
+
+    def add_eem_drtio(self, eem_drtio_channels):
+        # Must be called before invoking add_rtio() to construct the CRI
+        # interconnect properly
+        self.submodules.eem_transceiver = eem_serdes.EEMSerdes(self.platform, eem_drtio_channels)
+        self.csr_devices.append("eem_transceiver")
+        self.config["HAS_DRTIO_EEM"] = None
+        self.config["EEM_DRTIO_COUNT"] = len(eem_drtio_channels)
+
+        cdr = ClockDomainsRenamer({"rtio_rx": "sys"})
+        for i in range(len(self.eem_transceiver.channels)):
+            channel = i + len(self.gt_drtio.channels)
+            core_name = "drtio" + str(channel)
+            coreaux_name = "drtioaux" + str(channel)
+            memory_name = "drtioaux" + str(channel) + "_mem"
+            self.drtio_csr_group.append(core_name)
+            self.drtioaux_csr_group.append(coreaux_name)
+            self.drtioaux_memory_group.append(memory_name)
+
+            core = cdr(DRTIOMaster(self.rtio_tsc, self.eem_transceiver.channels[i]))
+            setattr(self.submodules, core_name, core)
+            self.drtio_cri.append(core.cri)
+            self.csr_devices.append(core_name)
+
+            coreaux = cdr(drtio_aux_controller.DRTIOAuxControllerBare(core.link_layer))
+            setattr(self.submodules, coreaux_name, coreaux)
+            self.csr_devices.append(coreaux_name)
+
+            size = coreaux.get_mem_size()
+            memory_address = self.axi2csr.register_port(coreaux.get_tx_port(), size)
+            self.axi2csr.register_port(coreaux.get_rx_port(), size)
+            self.add_memory_region(memory_name, self.mem_map["csr"] + memory_address, size * 2)
+
+    def add_drtio_cpuif_groups(self):
+        self.add_csr_group("drtio", self.drtio_csr_group)
+        self.add_csr_group("drtioaux", self.drtioaux_csr_group)
+        self.add_memory_group("drtioaux_mem", self.drtioaux_memory_group)
+
 
 class GenericSatellite(SoCCore):
     def __init__(self, description, acpki=False):
-        sys_clk_freq = 125e6
+        clk_freq = description["rtio_frequency"]
-        rtio_clk_freq = description["rtio_frequency"]
+        with_wrpll = description["enable_wrpll"]
 
         self.acpki = acpki
 
@@ -349,25 +441,42 @@ class GenericSatellite(SoCCore):
         ident = description["variant"]
         if self.acpki:
             ident = "acpki_" + ident
-        SoCCore.__init__(self, platform=platform, csr_data_width=32, ident=ident)
+        SoCCore.__init__(self, platform=platform, csr_data_width=32, ident=ident, ps_cd_sys=False)
 
-        self.config["hw_rev"] = description["hw_rev"]
+        self.config["HW_REV"] = description["hw_rev"]
-
-        platform.add_platform_command("create_clock -name clk_fpga_0 -period 8 [get_pins \"PS7/FCLKCLK[0]\"]")
-        platform.add_platform_command("set_input_jitter clk_fpga_0 0.24")
-
-        self.crg = self.ps7 # HACK for eem_7series to find the clock
-        self.submodules.rtio_crg = RTIOClockMultiplier(rtio_clk_freq)
-        self.csr_devices.append("rtio_crg")
-        fix_serdes_timing_path(platform)
 
         data_pads = [platform.request("sfp", i) for i in range(4)]
         
-        self.submodules.drtio_transceiver = gtx_7series.GTX(
+        self.submodules.gt_drtio = gtx_7series.GTX(
             clock_pads=platform.request("clk_gtp"),  
             pads=data_pads,
-            sys_clk_freq=sys_clk_freq)
+            clk_freq=clk_freq)
-        self.csr_devices.append("drtio_transceiver")
+        self.csr_devices.append("gt_drtio")
+
+        txout_buf = Signal()
+        gtx0 = self.gt_drtio.gtxs[0]
+        self.specials += Instance("BUFG", i_I=gtx0.txoutclk, o_O=txout_buf)
+
+        ext_async_rst = Signal()
+
+        self.submodules.bootstrap = GTPBootstrapClock(self.platform, clk_freq)
+        self.submodules.sys_crg = zynq_clocking.SYSCRG(
+            self.platform, 
+            self.ps7,
+            txout_buf,
+            clk_sw=self.gt_drtio.stable_clkin.storage,
+            clk_sw_status=gtx0.tx_init.done,
+            ext_async_rst=ext_async_rst)
+        platform.add_false_path_constraints(
+            self.bootstrap.cd_bootstrap.clk, self.sys_crg.cd_sys.clk)
+        self.csr_devices.append("sys_crg")
+        self.crg = self.ps7 # HACK for eem_7series to find the clock
+        self.crg.cd_sys = self.sys_crg.cd_sys
+
+        fix_serdes_timing_path(platform)
+
+        self.comb += ext_async_rst.eq(self.sys_crg.clk_sw_fsm.o_clk_sw & ~gtx0.tx_init.done)
+        self.specials += MultiReg(self.sys_crg.clk_sw_fsm.o_clk_sw & self.sys_crg.mmcm_locked, self.gt_drtio.clk_path_ready, odomain="bootstrap")
 
         self.rtio_channels = []
         has_grabber = any(peripheral["type"] == "grabber" for peripheral in description["peripherals"])
@@ -383,13 +492,13 @@ class GenericSatellite(SoCCore):
         self.config["RTIO_LOG_CHANNEL"] = len(self.rtio_channels)
         self.rtio_channels.append(rtio.LogChannel())
 
-        self.submodules.rtio_tsc = rtio.TSC("sync", glbl_fine_ts_width=3)
+        self.submodules.rtio_tsc = rtio.TSC(glbl_fine_ts_width=3)
 
         drtioaux_csr_group = []
         drtioaux_memory_group = []
         drtiorep_csr_group = []
         self.drtio_cri = []
-        for i in range(len(self.drtio_transceiver.channels)):
+        for i in range(len(self.gt_drtio.channels)):
             coreaux_name = "drtioaux" + str(i)
             memory_name = "drtioaux" + str(i) + "_mem"
             drtioaux_csr_group.append(coreaux_name)
@@ -400,7 +509,7 @@ class GenericSatellite(SoCCore):
             if i == 0:
                 self.submodules.rx_synchronizer = cdr(XilinxRXSynchronizer())
                 core = cdr(DRTIOSatellite(
-                    self.rtio_tsc, self.drtio_transceiver.channels[i],
+                    self.rtio_tsc, self.gt_drtio.channels[i],
                     self.rx_synchronizer))
                 self.submodules.drtiosat = core
                 self.csr_devices.append("drtiosat")
@@ -409,7 +518,7 @@ class GenericSatellite(SoCCore):
                 drtiorep_csr_group.append(corerep_name)
 
                 core = cdr(DRTIORepeater(
-                    self.rtio_tsc, self.drtio_transceiver.channels[i]))
+                    self.rtio_tsc, self.gt_drtio.channels[i]))
                 setattr(self.submodules, corerep_name, core)
                 self.drtio_cri.append(core.cri)
                 self.csr_devices.append(corerep_name)
@@ -454,9 +563,9 @@ class GenericSatellite(SoCCore):
         self.comb += self.drtiosat.async_errors.eq(self.local_io.async_errors)
 
         self.submodules.cri_con = rtio.CRIInterconnectShared(
-            [self.drtiosat.cri],
+            [self.drtiosat.cri, self.rtio_dma.cri, self.rtio.cri],
             [self.local_io.cri] + self.drtio_cri,
-            mode="sync", enable_routing=True)
+            enable_routing=True)
         self.csr_devices.append("cri_con")
 
         self.submodules.routing_table = rtio.RoutingTableAccess(self.cri_con)
@@ -465,31 +574,42 @@ class GenericSatellite(SoCCore):
         self.submodules.rtio_moninj = rtio.MonInj(self.rtio_channels)
         self.csr_devices.append("rtio_moninj")
 
-        rtio_clk_period = 1e9/rtio_clk_freq
+        self.submodules.rtio_analyzer = analyzer.Analyzer(self.rtio_tsc, self.local_io.cri,
-        self.config["RTIO_FREQUENCY"] = str(rtio_clk_freq/1e6)
+                                                          self.ps7.s_axi_hp1)
-        self.config["CLOCK_FREQUENCY"] = int(sys_clk_freq)
+        self.csr_devices.append("rtio_analyzer")
 
-        self.submodules.siphaser = SiPhaser7Series(
+        rtio_clk_period = 1e9/clk_freq
-            si5324_clkin=platform.request("cdr_clk"),
+        self.config["RTIO_FREQUENCY"] = str(clk_freq/1e6)
-            rx_synchronizer=self.rx_synchronizer,
+        self.config["CLOCK_FREQUENCY"] = int(clk_freq)
-            ultrascale=False,
-            rtio_clk_freq=self.drtio_transceiver.rtio_clk_freq)
-        platform.add_false_path_constraints(
-            self.crg.cd_sys.clk, self.siphaser.mmcm_freerun_output)
-        self.csr_devices.append("siphaser")
-        self.config["HAS_SI5324"] = None
-        self.config["SI5324_SOFT_RESET"] = None
 
-        gtx0 = self.drtio_transceiver.gtxs[0]
+        if with_wrpll:
-        platform.add_period_constraint(gtx0.txoutclk, rtio_clk_period)
+            clk_synth = platform.request("cdr_clk_clean_fabric")
-        platform.add_period_constraint(gtx0.rxoutclk, rtio_clk_period)
+            clk_synth_se = Signal()
+            platform.add_period_constraint(clk_synth.p, 8.0)
+            self.specials += Instance("IBUFGDS", p_DIFF_TERM="TRUE", p_IBUF_LOW_PWR="FALSE", i_I=clk_synth.p, i_IB=clk_synth.n, o_O=clk_synth_se)
+            self.submodules.wrpll = wrpll.WRPLL(
+                platform=self.platform,
+                cd_ref=self.gt_drtio.cd_rtio_rx0,
+                main_clk_se=clk_synth_se)
+            self.submodules.wrpll_skewtester = wrpll.SkewTester(self.rx_synchronizer)
+            self.csr_devices.append("wrpll_skewtester")
+            self.csr_devices.append("wrpll")
+            self.comb += self.ps7.core.core0.nfiq.eq(self.wrpll.ev.irq)
+            self.config["HAS_SI549"] = None
+            self.config["WRPLL_REF_CLK"] = "GT_CDR"
+        else:
+            self.submodules.siphaser = SiPhaser7Series(
+                si5324_clkin=platform.request("cdr_clk"),
+                rx_synchronizer=self.rx_synchronizer,
+                ultrascale=False,
+                rtio_clk_freq=self.gt_drtio.rtio_clk_freq)
+            self.csr_devices.append("siphaser")
+            self.config["HAS_SI5324"] = None
+            self.config["SI5324_SOFT_RESET"] = None
+
+        gtx0 = self.gt_drtio.gtxs[0]
         platform.add_false_path_constraints(
-            self.crg.cd_sys.clk,
             gtx0.txoutclk, gtx0.rxoutclk)
-        for gtx in self.drtio_transceiver.gtxs[1:]:
-            platform.add_period_constraint(gtx.rxoutclk, rtio_clk_period)
-            platform.add_false_path_constraints(
-                self.crg.cd_sys.clk, gtx.rxoutclk)
 
         if has_grabber:
             self.config["HAS_GRABBER"] = None
@@ -500,31 +620,7 @@ class GenericSatellite(SoCCore):
         self.csr_devices.append("virtual_leds")
 
         self.comb += [self.virtual_leds.get(i).eq(channel.rx_ready)
-                for i, channel in enumerate(self.drtio_transceiver.channels)]
+                for i, channel in enumerate(self.gt_drtio.channels)]
-
-def write_mem_file(soc, filename):
-    with open(filename, "w") as f:
-        f.write(cpu_interface.get_mem_rust(
-            soc.get_memory_regions(), soc.get_memory_groups(), None))
-
-
-def write_csr_file(soc, filename):
-    with open(filename, "w") as f:
-        f.write(cpu_interface.get_csr_rust(
-            soc.get_csr_regions(), soc.get_csr_groups(), soc.get_constants()))
-
-
-def write_rustc_cfg_file(soc, filename):
-    with open(filename, "w") as f:
-        for name, origin, busword, obj in soc.get_csr_regions():
-            f.write("has_{}\n".format(name.lower()))
-        for name, value in soc.get_constants():
-            if name.upper().startswith("CONFIG_"):
-                if value is None:
-                    f.write("{}\n".format(name.lower()[7:]))
-                else:
-                    f.write("{}=\"{}\"\n".format(name.lower()[7:], str(value)))
-
 
 def main():
     parser = argparse.ArgumentParser(
@@ -547,14 +643,14 @@ def main():
     if description["target"] != "kasli_soc":
         raise ValueError("Description is for a different target")
 
-    if description["base"] == "standalone":
+    if description["drtio_role"] == "standalone":
         cls = GenericStandalone
-    elif description["base"] == "master":
+    elif description["drtio_role"] == "master":
         cls = GenericMaster
-    elif description["base"] == "satellite":
+    elif description["drtio_role"] == "satellite":
         cls = GenericSatellite
     else:
-        raise ValueError("Invalid base")
+        raise ValueError("Invalid DRTIO role")
 
     soc = cls(description, acpki=args.acpki)
     soc.finalize()

src/gateware/test_dma.py

@@ -168,7 +168,7 @@ class FullStackTB(Module):
         bus = axi.Interface(ws*8)
         self.memory = AXIMemorySim(bus, sequence)
         self.submodules.dut = dma.DMA(bus)
-        self.submodules.tsc = rtio.TSC("async")
+        self.submodules.tsc = rtio.TSC()
         self.submodules.rtio = rtio.Core(self.tsc, rtio_channels)
         self.comb += self.dut.cri.connect(self.rtio.cri)
 
@@ -229,7 +229,7 @@ class TestDMA(unittest.TestCase):
             do_dma(tb.dut, 0), monitor(),
             (None for _ in range(70)),
             tb.memory.ar(), tb.memory.r()
-        ]}, {"sys": 8, "rsys": 8, "rtio": 8, "rio": 8, "rio_phy": 8})
+        ]}, {"sys": 8, "rsys": 8, "rio": 8, "rio_phy": 8})
 
         correct_changes = [(timestamp + 11, channel)
                            for channel, timestamp, _, _ in test_writes_full_stack]

src/gateware/zc706.py

@@ -10,12 +10,11 @@ from migen.genlib.cdc import MultiReg
 from migen_axi.integration.soc_core import SoCCore
 from migen_axi.platforms import zc706
 from misoc.interconnect.csr import *
-from misoc.integration import cpu_interface
 from misoc.cores import gpio
 
 from artiq.gateware import rtio, nist_clock, nist_qc2
 from artiq.gateware.rtio.phy import ttl_simple, ttl_serdes_7series, dds, spi2, edge_counter
-from artiq.gateware.rtio.xilinx_clocking import RTIOClockMultiplier, fix_serdes_timing_path
+from artiq.gateware.rtio.xilinx_clocking import fix_serdes_timing_path
 from artiq.gateware.drtio.transceiver import gtx_7series
 from artiq.gateware.drtio.siphaser import SiPhaser7Series
 from artiq.gateware.drtio.rx_synchronizer import XilinxRXSynchronizer
@@ -25,51 +24,8 @@ import dma
 import analyzer
 import acpki
 import drtio_aux_controller
-
+import zynq_clocking
-
+from config import write_csr_file, write_mem_file, write_rustc_cfg_file
-class RTIOCRG(Module, AutoCSR):
-    def __init__(self, platform, rtio_internal_clk):
-        self.clock_sel = CSRStorage()
-        self.pll_reset = CSRStorage(reset=1)
-        self.pll_locked = CSRStatus()
-        self.clock_domains.cd_rtio = ClockDomain()
-        self.clock_domains.cd_rtiox4 = ClockDomain(reset_less=True)
-
-        rtio_external_clk = Signal()
-        user_sma_clock = platform.request("user_sma_clock")
-        platform.add_period_constraint(user_sma_clock.p, 8.0)
-        self.specials += Instance("IBUFDS",
-                                  i_I=user_sma_clock.p, i_IB=user_sma_clock.n,
-                                  o_O=rtio_external_clk)
-
-        pll_locked = Signal()
-        rtio_clk = Signal()
-        rtiox4_clk = Signal()
-        self.specials += [
-            Instance("PLLE2_ADV",
-                     p_STARTUP_WAIT="FALSE", o_LOCKED=pll_locked,
-
-                     p_REF_JITTER1=0.01,
-                     p_CLKIN1_PERIOD=8.0, p_CLKIN2_PERIOD=8.0,
-                     i_CLKIN1=rtio_internal_clk, i_CLKIN2=rtio_external_clk,
-                     # Warning: CLKINSEL=0 means CLKIN2 is selected
-                     i_CLKINSEL=~self.clock_sel.storage,
-
-                     # VCO @ 1GHz when using 125MHz input
-                     p_CLKFBOUT_MULT=8, p_DIVCLK_DIVIDE=1,
-                     i_CLKFBIN=self.cd_rtio.clk,
-                     i_RST=self.pll_reset.storage,
-
-                     o_CLKFBOUT=rtio_clk,
-
-                     p_CLKOUT0_DIVIDE=2, p_CLKOUT0_PHASE=0.0,
-                     o_CLKOUT0=rtiox4_clk),
-            Instance("BUFG", i_I=rtio_clk, o_O=self.cd_rtio.clk),
-            Instance("BUFG", i_I=rtiox4_clk, o_O=self.cd_rtiox4.clk),
-            AsyncResetSynchronizer(self.cd_rtio, ~pll_locked),
-            MultiReg(pll_locked, self.pll_locked.status)
-        ]
-
 
 class SMAClkinForward(Module):
     def __init__(self, platform):
@@ -84,6 +40,37 @@ class SMAClkinForward(Module):
         ]
 
 
+class CLK200BootstrapClock(Module):
+    def __init__(self, platform, freq=125e6):
+        self.clock_domains.cd_bootstrap = ClockDomain(reset_less=True)
+        self.cd_bootstrap.clk.attr.add("keep")
+
+        clk200 = platform.request("clk200")
+        clk200_se = Signal()
+
+        pll_fb = Signal()
+        pll_clkout = Signal()
+        assert freq in [125e6, 100e6]
+        divide = int(1e9/freq)
+        self.specials += [
+            Instance("IBUFDS",
+                i_I=clk200.p, i_IB=clk200.n, o_O=clk200_se),
+            Instance("PLLE2_BASE",
+                p_CLKIN1_PERIOD=5.0,
+                i_CLKIN1=clk200_se,
+                i_CLKFBIN=pll_fb,
+                o_CLKFBOUT=pll_fb,
+
+                # VCO @ 1GHz
+                p_CLKFBOUT_MULT=5, p_DIVCLK_DIVIDE=1,
+
+                # 125MHz/100MHz for bootstrap
+                p_CLKOUT1_DIVIDE=divide, p_CLKOUT1_PHASE=0.0, o_CLKOUT1=pll_clkout,
+            ),
+            Instance("BUFG", i_I=pll_clkout, o_O=self.cd_bootstrap.clk)
+        ]
+
+
 # The NIST backplanes require setting VADJ to 3.3V by reprogramming the power supply.
 # This also changes the I/O standard for some on-board LEDs.
 leds_fmc33 = [
@@ -135,14 +122,10 @@ def prepare_zc706_platform(platform):
     platform.toolchain.bitstream_commands.extend([
         "set_property BITSTREAM.GENERAL.COMPRESS True [current_design]",
     ])
-    platform.add_platform_command("create_clock -name clk_fpga_0 -period 8 [get_pins \"PS7/FCLKCLK[0]\"]")
-    platform.add_platform_command("set_input_jitter clk_fpga_0 0.24")
-
 
 class ZC706(SoCCore):
     def __init__(self, acpki=False):
         self.acpki = acpki
-        self.rustc_cfg = dict()
 
         platform = zc706.Platform()
         prepare_zc706_platform(platform)
@@ -150,30 +133,49 @@ class ZC706(SoCCore):
         ident = self.__class__.__name__
         if self.acpki:
             ident = "acpki_" + ident
-        SoCCore.__init__(self, platform=platform, csr_data_width=32, ident=ident)
+        SoCCore.__init__(self, platform=platform, csr_data_width=32, ident=ident, ps_cd_sys=False)
 
-        self.submodules.rtio_crg = RTIOCRG(self.platform, self.ps7.cd_sys.clk)
+        platform.add_extension(si5324_fmc33)
-        self.csr_devices.append("rtio_crg")
+        self.comb += platform.request("si5324_33").rst_n.eq(1)
-        self.rustc_cfg["has_rtio_crg_clock_sel"] = None
+
-        self.platform.add_period_constraint(self.rtio_crg.cd_rtio.clk, 8.)
+        cdr_clk = Signal()
-        self.platform.add_false_path_constraints(
+        cdr_clk_buf = Signal()
-            self.ps7.cd_sys.clk,
+        si5324_out = platform.request("si5324_clkout")
-            self.rtio_crg.cd_rtio.clk)
+        platform.add_period_constraint(si5324_out.p, 8.0)
+        self.specials += [
+            Instance("IBUFDS_GTE2",
+                i_CEB=0,
+                i_I=si5324_out.p, i_IB=si5324_out.n,
+                o_O=cdr_clk,
+                p_CLKCM_CFG="TRUE",
+                p_CLKRCV_TRST="TRUE",
+                p_CLKSWING_CFG=3),
+            Instance("BUFG", i_I=cdr_clk, o_O=cdr_clk_buf)
+        ]
+        self.config["HAS_SI5324"] = None
+        self.config["SI5324_AS_SYNTHESIZER"] = None
+        self.config["SI5324_SOFT_RESET"] = None
+        
+        self.submodules.bootstrap = CLK200BootstrapClock(platform)
+        self.submodules.sys_crg = zynq_clocking.SYSCRG(self.platform, self.ps7, cdr_clk_buf)
+        platform.add_false_path_constraints(
+            self.bootstrap.cd_bootstrap.clk, self.sys_crg.cd_sys.clk)
+        self.csr_devices.append("sys_crg")
 
     def add_rtio(self, rtio_channels):
-        self.submodules.rtio_tsc = rtio.TSC("async", glbl_fine_ts_width=3)
+        self.submodules.rtio_tsc = rtio.TSC(glbl_fine_ts_width=3)
         self.submodules.rtio_core = rtio.Core(self.rtio_tsc, rtio_channels)
         self.csr_devices.append("rtio_core")
 
         if self.acpki:
-            self.rustc_cfg["ki_impl"] = "acp"
+            self.config["KI_IMPL"] = "acp"
             self.submodules.rtio = acpki.KernelInitiator(self.rtio_tsc,
                                                          bus=self.ps7.s_axi_acp,
                                                          user=self.ps7.s_axi_acp_user,
                                                          evento=self.ps7.event.o)
             self.csr_devices.append("rtio")
         else:
-            self.rustc_cfg["ki_impl"] = "csr"
+            self.config["KI_IMPL"] = "csr"
             self.submodules.rtio = rtio.KernelInitiator(self.rtio_tsc, now64=True)
             self.csr_devices.append("rtio")
 
@@ -196,22 +198,18 @@ class ZC706(SoCCore):
 class _MasterBase(SoCCore):
     def __init__(self, acpki=False, drtio100mhz=False):
         self.acpki = acpki
-        self.rustc_cfg = dict()
+
+        clk_freq = 100e6 if drtio100mhz else 125e6
 
         platform = zc706.Platform()
         prepare_zc706_platform(platform)
         ident = self.__class__.__name__
         if self.acpki:
             ident = "acpki_" + ident
-        SoCCore.__init__(self, platform=platform, csr_data_width=32, ident=ident)
+        SoCCore.__init__(self, platform=platform, csr_data_width=32, ident=ident, ps_cd_sys=False)
 
         platform.add_extension(si5324_fmc33)
 
-        self.sys_clk_freq = 125e6
-        rtio_clk_freq = 100e6 if drtio100mhz else self.sys_clk_freq
-
-        platform = self.platform
-
         self.comb += platform.request("sfp_tx_disable_n").eq(1)
         data_pads = [
             platform.request("sfp"),
@@ -221,20 +219,38 @@ class _MasterBase(SoCCore):
         self.submodules += SMAClkinForward(self.platform)
 
         # 1000BASE_BX10 Ethernet compatible, 125MHz RTIO clock
-        self.submodules.drtio_transceiver = gtx_7series.GTX(
+        self.submodules.gt_drtio = gtx_7series.GTX(
             clock_pads=platform.request("si5324_clkout"),
             pads=data_pads,
-            sys_clk_freq=self.sys_clk_freq,
+            clk_freq=clk_freq)
-            rtio_clk_freq=rtio_clk_freq)
+        self.csr_devices.append("gt_drtio")
-        self.csr_devices.append("drtio_transceiver")
 
-        self.submodules.rtio_tsc = rtio.TSC("async", glbl_fine_ts_width=3)
+        self.submodules.rtio_tsc = rtio.TSC(glbl_fine_ts_width=3)
+        ext_async_rst = Signal()
+        txout_buf = Signal()
+        gtx0 = self.gt_drtio.gtxs[0]
+        self.specials += Instance("BUFG", i_I=gtx0.txoutclk, o_O=txout_buf)
+        self.submodules.bootstrap = CLK200BootstrapClock(platform, clk_freq)
+        self.submodules.sys_crg = zynq_clocking.SYSCRG(
+            self.platform, 
+            self.ps7, 
+            txout_buf,
+            clk_sw=self.gt_drtio.stable_clkin.storage,
+            clk_sw_status=gtx0.tx_init.done,
+            ext_async_rst=ext_async_rst,
+            freq=clk_freq)
+        platform.add_false_path_constraints(
+            self.bootstrap.cd_bootstrap.clk, self.sys_crg.cd_sys.clk)
+        self.csr_devices.append("sys_crg")
+
+        self.comb += ext_async_rst.eq(self.sys_crg.clk_sw_fsm.o_clk_sw & ~gtx0.tx_init.done)
+        self.specials += MultiReg(self.sys_crg.clk_sw_fsm.o_clk_sw & self.sys_crg.mmcm_locked, self.gt_drtio.clk_path_ready, odomain="bootstrap")
 
         drtio_csr_group = []
         drtioaux_csr_group = []
         drtioaux_memory_group = []
         self.drtio_cri = []
-        for i in range(len(self.drtio_transceiver.channels)):
+        for i in range(len(self.gt_drtio.channels)):
             core_name = "drtio" + str(i)
             coreaux_name = "drtioaux" + str(i)
             memory_name = "drtioaux" + str(i) + "_mem"
@@ -245,7 +261,7 @@ class _MasterBase(SoCCore):
             cdr = ClockDomainsRenamer({"rtio_rx": "rtio_rx" + str(i)})
 
             core = cdr(DRTIOMaster(
-                self.rtio_tsc, self.drtio_transceiver.channels[i]))
+                self.rtio_tsc, self.gt_drtio.channels[i]))
             setattr(self.submodules, core_name, core)
             self.drtio_cri.append(core.cri)
             self.csr_devices.append(core_name)
@@ -258,53 +274,45 @@ class _MasterBase(SoCCore):
             memory_address = self.axi2csr.register_port(coreaux.get_tx_port(), mem_size)
             self.axi2csr.register_port(coreaux.get_rx_port(), mem_size)
             self.add_memory_region(memory_name, self.mem_map["csr"] + memory_address, mem_size * 2)
-        self.rustc_cfg["has_drtio"] = None
+        self.config["HAS_DRTIO"] = None
-        self.rustc_cfg["has_drtio_routing"] = None
+        self.config["HAS_DRTIO_ROUTING"] = None
         self.add_csr_group("drtio", drtio_csr_group)
         self.add_csr_group("drtioaux", drtioaux_csr_group)
         self.add_memory_group("drtioaux_mem", drtioaux_memory_group)
 
-        self.rustc_cfg["rtio_frequency"] = str(self.drtio_transceiver.rtio_clk_freq/1e6)
+        self.config["RTIO_FREQUENCY"] = str(self.gt_drtio.rtio_clk_freq/1e6)
 
         self.submodules.si5324_rst_n = gpio.GPIOOut(platform.request("si5324_33").rst_n)
         self.csr_devices.append("si5324_rst_n")
-        self.rustc_cfg["has_si5324"] = None
+        self.config["HAS_SI5324"] = None
-        self.rustc_cfg["si5324_as_synthesizer"] = None
+        self.config["SI5324_AS_SYNTHESIZER"] = None
 
-        rtio_clk_period = 1e9/self.drtio_transceiver.rtio_clk_freq
         # Constrain TX & RX timing for the first transceiver channel
         # (First channel acts as master for phase alignment for all channels' TX)
-        gtx0 = self.drtio_transceiver.gtxs[0]
-        platform.add_period_constraint(gtx0.txoutclk, rtio_clk_period)
-        platform.add_period_constraint(gtx0.rxoutclk, rtio_clk_period)
         platform.add_false_path_constraints(
-            self.ps7.cd_sys.clk,
             gtx0.txoutclk, gtx0.rxoutclk)
         # Constrain RX timing for the each transceiver channel
         # (Each channel performs single-lane phase alignment for RX)
-        for gtx in self.drtio_transceiver.gtxs[1:]:
+        for gtx in self.gt_drtio.gtxs[1:]:
-            platform.add_period_constraint(gtx.rxoutclk, rtio_clk_period)
             platform.add_false_path_constraints(
-                self.ps7.cd_sys.clk, gtx0.txoutclk, gtx.rxoutclk)
+                gtx0.txoutclk, gtx.rxoutclk)
 
-        self.submodules.rtio_crg = RTIOClockMultiplier(self.sys_clk_freq)
+        fix_serdes_timing_path(platform)
-        self.csr_devices.append("rtio_crg")
-        fix_serdes_timing_path(self.platform)
 
     def add_rtio(self, rtio_channels):
-        self.submodules.rtio_tsc = rtio.TSC("async", glbl_fine_ts_width=3)
+        self.submodules.rtio_tsc = rtio.TSC(glbl_fine_ts_width=3)
         self.submodules.rtio_core = rtio.Core(self.rtio_tsc, rtio_channels)
         self.csr_devices.append("rtio_core")
 
         if self.acpki:
-            self.rustc_cfg["ki_impl"] = "acp"
+            self.config["KI_IMPL"] = "acp"
             self.submodules.rtio = acpki.KernelInitiator(self.rtio_tsc,
                                                          bus=self.ps7.s_axi_acp,
                                                          user=self.ps7.s_axi_acp_user,
                                                          evento=self.ps7.event.o)
             self.csr_devices.append("rtio")
         else:
-            self.rustc_cfg["ki_impl"] = "csr"
+            self.config["KI_IMPL"] = "csr"
             self.submodules.rtio = rtio.KernelInitiator(self.rtio_tsc, now64=True)
             self.csr_devices.append("rtio")
 
@@ -314,13 +322,13 @@ class _MasterBase(SoCCore):
         self.submodules.cri_con = rtio.CRIInterconnectShared(
             [self.rtio.cri, self.rtio_dma.cri],
             [self.rtio_core.cri] + self.drtio_cri,
-            mode="sync", enable_routing=True)
+            enable_routing=True)
         self.csr_devices.append("cri_con")
 
         self.submodules.rtio_moninj = rtio.MonInj(rtio_channels)
         self.csr_devices.append("rtio_moninj")
 
-        self.submodules.rtio_analyzer = analyzer.Analyzer(self.rtio_tsc, self.cri_con.switch.slave,
+        self.submodules.rtio_analyzer = analyzer.Analyzer(self.rtio_tsc, self.rtio_core.cri,
                                                           self.ps7.s_axi_hp1)
         self.csr_devices.append("rtio_analyzer")
 
@@ -331,21 +339,18 @@ class _MasterBase(SoCCore):
 class _SatelliteBase(SoCCore):
     def __init__(self, acpki=False, drtio100mhz=False):
         self.acpki = acpki
-        self.rustc_cfg = dict()
+
+        clk_freq = 100e6 if drtio100mhz else 125e6
 
         platform = zc706.Platform()
         prepare_zc706_platform(platform)
         ident = self.__class__.__name__
         if self.acpki:
             ident = "acpki_" + ident
-        SoCCore.__init__(self, platform=platform, csr_data_width=32, ident=ident)
+        SoCCore.__init__(self, platform=platform, csr_data_width=32, ident=ident, ps_cd_sys=False)
 
         platform.add_extension(si5324_fmc33)
 
-        self.sys_clk_freq = 125e6
-        rtio_clk_freq = 100e6 if drtio100mhz else self.sys_clk_freq
-        platform = self.platform
-
         # SFP
         self.comb += platform.request("sfp_tx_disable_n").eq(0)
         data_pads = [
@@ -353,21 +358,44 @@ class _SatelliteBase(SoCCore):
             platform.request("user_sma_mgt")
         ]
 
-        self.submodules.rtio_tsc = rtio.TSC("sync", glbl_fine_ts_width=3)
+        self.submodules.rtio_tsc = rtio.TSC(glbl_fine_ts_width=3)
 
         # 1000BASE_BX10 Ethernet compatible, 125MHz RTIO clock
-        self.submodules.drtio_transceiver = gtx_7series.GTX(
+        self.submodules.gt_drtio = gtx_7series.GTX(
             clock_pads=platform.request("si5324_clkout"),
             pads=data_pads,
-            sys_clk_freq=self.sys_clk_freq,
+            clk_freq=clk_freq)
-            rtio_clk_freq=rtio_clk_freq)
+        self.csr_devices.append("gt_drtio")
-        self.csr_devices.append("drtio_transceiver")
+
+        ext_async_rst = Signal()
+        txout_buf = Signal()
+        txout_buf.attr.add("keep")
+        gtx0 = self.gt_drtio.gtxs[0]
+        self.specials += Instance(
+            "BUFG", 
+            i_I=gtx0.txoutclk, 
+            o_O=txout_buf)
+        self.submodules.bootstrap = CLK200BootstrapClock(platform, clk_freq)
+        self.submodules.sys_crg = zynq_clocking.SYSCRG(
+            self.platform, 
+            self.ps7, 
+            txout_buf,
+            clk_sw=self.gt_drtio.stable_clkin.storage,
+            clk_sw_status=gtx0.tx_init.done,
+            ext_async_rst=ext_async_rst,
+            freq=clk_freq)
+        platform.add_false_path_constraints(
+            self.bootstrap.cd_bootstrap.clk, self.sys_crg.cd_sys.clk)
+        self.csr_devices.append("sys_crg")
+
+        self.comb += ext_async_rst.eq(self.sys_crg.clk_sw_fsm.o_clk_sw & ~gtx0.tx_init.done)
+        self.specials += MultiReg(self.sys_crg.clk_sw_fsm.o_clk_sw & self.sys_crg.mmcm_locked, self.gt_drtio.clk_path_ready, odomain="bootstrap")
 
         drtioaux_csr_group = []
         drtioaux_memory_group = []
         drtiorep_csr_group = []
         self.drtio_cri = []
-        for i in range(len(self.drtio_transceiver.channels)):
+        for i in range(len(self.gt_drtio.channels)):
             coreaux_name = "drtioaux" + str(i)
             memory_name = "drtioaux" + str(i) + "_mem"
             drtioaux_csr_group.append(coreaux_name)
@@ -379,7 +407,7 @@ class _SatelliteBase(SoCCore):
             if i == 0:
                 self.submodules.rx_synchronizer = cdr(XilinxRXSynchronizer())
                 core = cdr(DRTIOSatellite(
-                    self.rtio_tsc, self.drtio_transceiver.channels[0], self.rx_synchronizer))
+                    self.rtio_tsc, self.gt_drtio.channels[0], self.rx_synchronizer))
                 self.submodules.drtiosat = core
                 self.csr_devices.append("drtiosat")
             # Repeaters
@@ -387,7 +415,7 @@ class _SatelliteBase(SoCCore):
                 corerep_name = "drtiorep" + str(i-1)
                 drtiorep_csr_group.append(corerep_name)
                 core = cdr(DRTIORepeater(
-                    self.rtio_tsc, self.drtio_transceiver.channels[i]))
+                    self.rtio_tsc, self.gt_drtio.channels[i]))
                 setattr(self.submodules, corerep_name, core)
                 self.drtio_cri.append(core.cri)
                 self.csr_devices.append(corerep_name)
@@ -405,73 +433,70 @@ class _SatelliteBase(SoCCore):
             # and registered in PS interface
             # manually, because software refers to rx/tx by halves of entire memory block, not names
             self.add_memory_region(memory_name, self.mem_map["csr"] + memory_address, mem_size * 2)
-        self.rustc_cfg["has_drtio"] = None
+        self.config["HAS_DRTIO"] = None
-        self.rustc_cfg["has_drtio_routing"] = None
+        self.config["HAS_DRTIO_ROUTING"] = None
         self.add_csr_group("drtioaux", drtioaux_csr_group)
         self.add_csr_group("drtiorep", drtiorep_csr_group)
         self.add_memory_group("drtioaux_mem", drtioaux_memory_group)
 
-        self.rustc_cfg["rtio_frequency"] = str(self.drtio_transceiver.rtio_clk_freq/1e6)
+        self.config["RTIO_FREQUENCY"] = str(self.gt_drtio.rtio_clk_freq/1e6)
 
         # Si5324 Phaser
         self.submodules.siphaser = SiPhaser7Series(
             si5324_clkin=platform.request("si5324_clkin"),
             rx_synchronizer=self.rx_synchronizer,
             ultrascale=False,
-            rtio_clk_freq=self.drtio_transceiver.rtio_clk_freq)
+            rtio_clk_freq=self.gt_drtio.rtio_clk_freq)
         platform.add_false_path_constraints(
-            self.ps7.cd_sys.clk, self.siphaser.mmcm_freerun_output)
+            self.sys_crg.cd_sys.clk, self.siphaser.mmcm_freerun_output)
         self.csr_devices.append("siphaser")
         self.submodules.si5324_rst_n = gpio.GPIOOut(platform.request("si5324_33").rst_n)
         self.csr_devices.append("si5324_rst_n")
-        self.rustc_cfg["has_si5324"] = None
+        self.config["HAS_SI5324"] = None
-        self.rustc_cfg["has_siphaser"] = None
 
-        rtio_clk_period = 1e9/self.drtio_transceiver.rtio_clk_freq
+        rtio_clk_period = 1e9/self.gt_drtio.rtio_clk_freq
         # Constrain TX & RX timing for the first transceiver channel
         # (First channel acts as master for phase alignment for all channels' TX)
-        gtx0 = self.drtio_transceiver.gtxs[0]
-        platform.add_period_constraint(gtx0.txoutclk, rtio_clk_period)
-        platform.add_period_constraint(gtx0.rxoutclk, rtio_clk_period)
         platform.add_false_path_constraints(
-            self.ps7.cd_sys.clk,
             gtx0.txoutclk, gtx0.rxoutclk)
         # Constrain RX timing for the each transceiver channel
         # (Each channel performs single-lane phase alignment for RX)
-        for gtx in self.drtio_transceiver.gtxs[1:]:
+        for gtx in self.gt_drtio.gtxs[1:]:
-            platform.add_period_constraint(gtx.rxoutclk, rtio_clk_period)
             platform.add_false_path_constraints(
-                self.ps7.cd_sys.clk, gtx.rxoutclk)
+                self.sys_crg.cd_sys.clk, gtx.rxoutclk)
 
-        self.submodules.rtio_crg = RTIOClockMultiplier(self.sys_clk_freq)
+        fix_serdes_timing_path(platform)
-        self.csr_devices.append("rtio_crg")
-        self.rustc_cfg["has_rtio_crg"] = None
-        fix_serdes_timing_path(self.platform)
 
     def add_rtio(self, rtio_channels):
         self.submodules.rtio_moninj = rtio.MonInj(rtio_channels)
         self.csr_devices.append("rtio_moninj")
-        self.rustc_cfg["has_rtio_moninj"] = None
 
         if self.acpki:
-            self.rustc_cfg["ki_impl"] = "acp"
+            self.config["KI_IMPL"] = "acp"
             self.submodules.rtio = acpki.KernelInitiator(self.rtio_tsc,
                                                          bus=self.ps7.s_axi_acp,
                                                          user=self.ps7.s_axi_acp_user,
                                                          evento=self.ps7.event.o)
             self.csr_devices.append("rtio")
         else:
-            self.rustc_cfg["ki_impl"] = "csr"
+            self.config["KI_IMPL"] = "csr"
             self.submodules.rtio = rtio.KernelInitiator(self.rtio_tsc, now64=True)
             self.csr_devices.append("rtio")
 
+        self.submodules.rtio_dma = dma.DMA(self.ps7.s_axi_hp0)
+        self.csr_devices.append("rtio_dma")
+
         self.submodules.local_io = SyncRTIO(self.rtio_tsc, rtio_channels)
         self.submodules.cri_con = rtio.CRIInterconnectShared(
-            [self.drtiosat.cri],
+            [self.drtiosat.cri, self.rtio_dma.cri, self.rtio.cri],
             [self.local_io.cri] + self.drtio_cri,
-            mode="sync", enable_routing=True)
+            enable_routing=True)
         self.csr_devices.append("cri_con")
 
+        self.submodules.rtio_analyzer = analyzer.Analyzer(self.rtio_tsc, self.local_io.cri,
+                                                          self.ps7.s_axi_hp1)
+        self.csr_devices.append("rtio_analyzer")
+
         self.submodules.routing_table = rtio.RoutingTableAccess(self.cri_con)
         self.csr_devices.append("routing_table")
 
@@ -626,6 +651,7 @@ class _NIST_QC2_RTIO:
 class NIST_CLOCK(ZC706, _NIST_CLOCK_RTIO):
     def __init__(self, acpki, drtio100mhz):
         ZC706.__init__(self, acpki)
+        self.submodules += SMAClkinForward(self.platform)
         _NIST_CLOCK_RTIO.__init__(self)
 
 class NIST_CLOCK_Master(_MasterBase, _NIST_CLOCK_RTIO):
@@ -641,6 +667,7 @@ class NIST_CLOCK_Satellite(_SatelliteBase, _NIST_CLOCK_RTIO):
 class NIST_QC2(ZC706, _NIST_QC2_RTIO):
     def __init__(self, acpki, drtio100mhz):
         ZC706.__init__(self, acpki)
+        self.submodules += SMAClkinForward(self.platform)
         _NIST_QC2_RTIO.__init__(self)
 
 class NIST_QC2_Master(_MasterBase, _NIST_QC2_RTIO):
@@ -656,27 +683,6 @@ class NIST_QC2_Satellite(_SatelliteBase, _NIST_QC2_RTIO):
 VARIANTS = {cls.__name__.lower(): cls for cls in [NIST_CLOCK, NIST_CLOCK_Master, NIST_CLOCK_Satellite,
                                                   NIST_QC2, NIST_QC2_Master, NIST_QC2_Satellite]}
 
-
-def write_csr_file(soc, filename):
-    with open(filename, "w") as f:
-        f.write(cpu_interface.get_csr_rust(
-            soc.get_csr_regions(), soc.get_csr_groups(), soc.get_constants()))
-
-def write_mem_file(soc, filename):
-    with open(filename, "w") as f:
-        f.write(cpu_interface.get_mem_rust(
-            soc.get_memory_regions(), soc.get_memory_groups(), None))
-
-
-def write_rustc_cfg_file(soc, filename):
-    with open(filename, "w") as f:
-        for k, v in sorted(soc.rustc_cfg.items(), key=itemgetter(0)):
-            if v is None:
-                f.write("{}\n".format(k))
-            else:
-                f.write("{}=\"{}\"\n".format(k, v))
-
-
 def main():
     parser = argparse.ArgumentParser(
         description="ARTIQ port to the ZC706 Zynq development kit")

src/gateware/zynq_clocking.py

@@ -0,0 +1,154 @@
+from migen import *
+from migen.genlib.cdc import MultiReg
+from migen.genlib.resetsync import AsyncResetSynchronizer
+from misoc.interconnect.csr import *
+
+
+class ClockSwitchFSM(Module):
+    def __init__(self):
+        self.i_clk_sw = Signal()
+
+        self.o_clk_sw = Signal()
+        self.o_reset = Signal()
+
+        ###
+
+        i_switch = Signal()
+        o_switch = Signal()
+        reset = Signal()
+
+        # at 125MHz bootstrap cd, will get around 0.5ms
+        delay_counter = Signal(16, reset=0xFFFF)
+
+        # register to prevent glitches
+        self.sync.bootstrap += [
+            self.o_clk_sw.eq(o_switch),
+            self.o_reset.eq(reset),
+        ]
+
+        self.o_clk_sw.attr.add("no_retiming")
+        self.o_reset.attr.add("no_retiming")
+        self.i_clk_sw.attr.add("no_retiming")
+        i_switch.attr.add("no_retiming")
+
+        self.specials += MultiReg(self.i_clk_sw, i_switch, "bootstrap")
+
+        fsm = ClockDomainsRenamer("bootstrap")(FSM(reset_state="START"))
+
+        self.submodules += fsm
+
+        fsm.act("START",
+            If(i_switch & ~o_switch,
+                NextState("RESET_START"))
+        )
+        
+        fsm.act("RESET_START",
+            reset.eq(1),
+            If(delay_counter == 0,
+                NextValue(delay_counter, 0xFFFF),
+                NextState("CLOCK_SWITCH")
+            ).Else(
+                NextValue(delay_counter, delay_counter-1),
+            )
+        )
+
+        fsm.act("CLOCK_SWITCH",
+            reset.eq(1),
+            NextValue(o_switch, 1),
+            NextValue(delay_counter, delay_counter-1),
+            If(delay_counter == 0,
+                NextState("END"))
+        )
+        fsm.act("END",
+            NextValue(o_switch, 1),
+            reset.eq(0))
+
+
+class SYSCRG(Module, AutoCSR):
+    def __init__(self, platform, ps7, main_clk, clk_sw=None, clk_sw_status=None, freq=125e6, ext_async_rst=None, ):
+        # assumes bootstrap clock is same freq as main and sys output
+        self.clock_domains.cd_sys = ClockDomain()
+        self.clock_domains.cd_sys4x = ClockDomain(reset_less=True)
+        self.clock_domains.cd_sys5x = ClockDomain(reset_less=True)
+        self.clock_domains.cd_clk200 = ClockDomain()
+
+        self.current_clock = CSRStatus()
+
+        self.cd_sys.clk.attr.add("keep")
+
+        bootstrap_clk = ClockSignal("bootstrap")
+
+        period = 1e9/freq
+
+        self.submodules.clk_sw_fsm = ClockSwitchFSM()
+
+        if clk_sw is None:
+            self.clock_switch = CSRStorage()
+            self.comb += self.clk_sw_fsm.i_clk_sw.eq(self.clock_switch.storage)
+        else:
+            self.comb += self.clk_sw_fsm.i_clk_sw.eq(clk_sw)
+
+        self.mmcm_locked = Signal()
+        mmcm_sys = Signal()
+        mmcm_sys4x = Signal()
+        mmcm_sys5x = Signal()
+        mmcm_clk208 = Signal()
+        mmcm_fb_clk = Signal()
+        self.specials += [
+            Instance("MMCME2_ADV",
+                p_STARTUP_WAIT="FALSE", o_LOCKED=self.mmcm_locked,
+                p_BANDWIDTH="HIGH",
+                p_REF_JITTER1=0.001,
+                p_CLKIN1_PERIOD=period, i_CLKIN1=main_clk,
+                p_CLKIN2_PERIOD=period, i_CLKIN2=bootstrap_clk,
+                i_CLKINSEL=self.clk_sw_fsm.o_clk_sw,
+
+                # VCO @ 1.25GHz
+                p_CLKFBOUT_MULT_F=10, p_DIVCLK_DIVIDE=1,
+                i_CLKFBIN=mmcm_fb_clk,
+                i_RST=self.clk_sw_fsm.o_reset,
+
+                o_CLKFBOUT=mmcm_fb_clk,
+
+                p_CLKOUT0_DIVIDE_F=2.5, p_CLKOUT0_PHASE=0.0, o_CLKOUT0=mmcm_sys4x,
+
+                # 125MHz
+                p_CLKOUT1_DIVIDE=10, p_CLKOUT1_PHASE=0.0, o_CLKOUT1=mmcm_sys,
+
+                # 625MHz
+                p_CLKOUT2_DIVIDE=2, p_CLKOUT2_PHASE=0.0, o_CLKOUT2=mmcm_sys5x,
+
+                # 208MHz
+                p_CLKOUT3_DIVIDE=6, p_CLKOUT3_PHASE=0.0, o_CLKOUT3=mmcm_clk208,
+            ),
+            Instance("BUFG", i_I=mmcm_sys5x, o_O=self.cd_sys5x.clk),
+            Instance("BUFG", i_I=mmcm_sys, o_O=self.cd_sys.clk),
+            Instance("BUFG", i_I=mmcm_sys4x, o_O=self.cd_sys4x.clk),
+            Instance("BUFG", i_I=mmcm_clk208, o_O=self.cd_clk200.clk),
+        ]
+
+        if ext_async_rst is not None:
+            self.specials += [
+                AsyncResetSynchronizer(self.cd_sys, ~self.mmcm_locked | ext_async_rst),
+                AsyncResetSynchronizer(self.cd_clk200, ~self.mmcm_locked | ext_async_rst),
+            ]
+        else:
+            self.specials += [
+                AsyncResetSynchronizer(self.cd_sys, ~self.mmcm_locked),
+                AsyncResetSynchronizer(self.cd_clk200, ~self.mmcm_locked),
+            ]
+
+        reset_counter = Signal(4, reset=15)
+        ic_reset = Signal(reset=1)
+        self.sync.clk200 += \
+            If(reset_counter != 0,
+                reset_counter.eq(reset_counter - 1)
+            ).Else(
+                ic_reset.eq(0)
+            )
+        self.specials += Instance("IDELAYCTRL", i_REFCLK=ClockSignal("clk200"), i_RST=ic_reset)
+
+        if clk_sw_status is None:
+            self.comb += self.current_clock.status.eq(self.clk_sw_fsm.o_clk_sw)
+        else:
+            self.comb += self.current_clock.status.eq(clk_sw_status)

src/libboard_artiq/Cargo.toml.tpl

@@ -10,6 +10,7 @@ name = "libboard_artiq"
 [features]
 target_zc706 = ["libboard_zynq/target_zc706", "libconfig/target_zc706"]
 target_kasli_soc = ["libboard_zynq/target_kasli_soc", "libconfig/target_kasli_soc"]
+calibrate_wrpll_skew = []
 
 [build-dependencies]
 build_zynq = { path = "../libbuild_zynq" }
@@ -24,8 +25,9 @@ nb = "1.0"
 void = { version = "1", default-features = false }
 
 io = { path = "../libio", features = ["byteorder"] }
-libboard_zynq = { git = "https://git.m-labs.hk/M-Labs/zynq-rs.git"}
+libboard_zynq = { path = "@@ZYNQ_RS@@/libboard_zynq" }
-libregister = { git = "https://git.m-labs.hk/M-Labs/zynq-rs.git" }
+libsupport_zynq = { path = "@@ZYNQ_RS@@/libsupport_zynq", default-features = false, features = ["alloc_core"] }
-libconfig = { git = "https://git.m-labs.hk/M-Labs/zynq-rs.git", features = ["fat_lfn"] }
+libregister = { path = "@@ZYNQ_RS@@/libregister" }
-libcortex_a9 = { git = "https://git.m-labs.hk/M-Labs/zynq-rs.git" }
+libconfig = { path = "@@ZYNQ_RS@@/libconfig", features = ["fat_lfn"] }
-libasync = { git = "https://git.m-labs.hk/M-Labs/zynq-rs.git" }
+libcortex_a9 = { path = "@@ZYNQ_RS@@/libcortex_a9" }
+libasync = { path = "@@ZYNQ_RS@@/libasync" }

src/libboard_artiq/src/drtio_eem.rs

@@ -0,0 +1,228 @@
+use embedded_hal::prelude::_embedded_hal_blocking_delay_DelayUs;
+use libboard_zynq::timer::GlobalTimer;
+use libconfig::Config;
+use libsupport_zynq::alloc::format;
+use log::{debug, error, info};
+
+use crate::pl;
+
+struct SerdesConfig {
+    pub delay: [u8; 4],
+}
+
+impl SerdesConfig {
+    pub fn as_bytes(&self) -> &[u8] {
+        unsafe {
+            core::slice::from_raw_parts(
+                (self as *const SerdesConfig) as *const u8,
+                core::mem::size_of::<SerdesConfig>(),
+            )
+        }
+    }
+}
+
+fn select_lane(lane_no: u8) {
+    unsafe {
+        pl::csr::eem_transceiver::lane_sel_write(lane_no);
+    }
+}
+
+fn apply_delay(tap: u8, timer: &mut GlobalTimer) {
+    unsafe {
+        pl::csr::eem_transceiver::dly_cnt_in_write(tap);
+        pl::csr::eem_transceiver::dly_ld_write(1);
+        timer.delay_us(1);
+        assert!(tap as u8 == pl::csr::eem_transceiver::dly_cnt_out_read());
+    }
+}
+
+fn apply_config(config: &SerdesConfig, timer: &mut GlobalTimer) {
+    for lane_no in 0..4 {
+        select_lane(lane_no as u8);
+        apply_delay(config.delay[lane_no], timer);
+    }
+}
+
+unsafe fn assign_delay(timer: &mut GlobalTimer) -> SerdesConfig {
+    // Select an appropriate delay for lane 0
+    select_lane(0);
+
+    //
+
+    let mut best_dly = None;
+
+    loop {
+        let mut prev = None;
+        for curr_dly in 0..32 {
+            //let read_align = read_align_fn(curr_dly, timer);
+            let curr_low_rate = read_align(curr_dly, timer);
+
+            if let Some(prev_low_rate) = prev {
+                // This is potentially a crossover position
+                if prev_low_rate <= curr_low_rate && curr_low_rate >= 0.5 {
+                    let prev_dev = 0.5 - prev_low_rate;
+                    let curr_dev = curr_low_rate - 0.5;
+                    let selected_idx = if prev_dev < curr_dev { curr_dly - 1 } else { curr_dly };
+
+                    // The setup setup/hold calibration timing (even with
+                    // tolerance) might be invalid in other lanes due to skew.
+                    // 5 taps is very conservative, generally it is 1 or 2
+                    if selected_idx < 5 {
+                        prev = None;
+                        continue;
+                    } else {
+                        best_dly = Some(selected_idx);
+                        break;
+                    }
+                }
+            }
+
+            // Only rising slope from <= 0.5 can result in a rising low rate
+            // crossover at 50%.
+            if curr_low_rate <= 0.5 {
+                prev = Some(curr_low_rate);
+            }
+        }
+
+        if best_dly.is_none() {
+            error!("setup/hold timing calibration failed, retry in 1s...");
+            timer.delay_us(1_000_000);
+        } else {
+            break;
+        }
+    }
+
+    let best_dly = best_dly.unwrap();
+
+    apply_delay(best_dly, timer);
+    let mut delay_list = [best_dly; 4];
+
+    // Assign delay for other lanes
+    for lane_no in 1..=3 {
+        select_lane(lane_no as u8);
+
+        let mut min_deviation = 0.5;
+        let mut min_idx = 0;
+        for dly_delta in -3..=3 {
+            let index = (best_dly as isize + dly_delta) as u8;
+            let low_rate = read_align(index, timer);
+            // abs() from f32 is not available in core library
+            let deviation = if low_rate < 0.5 { 0.5 - low_rate } else { low_rate - 0.5 };
+
+            if deviation < min_deviation {
+                min_deviation = deviation;
+                min_idx = index;
+            }
+        }
+
+        apply_delay(min_idx, timer);
+        delay_list[lane_no] = min_idx;
+    }
+
+    debug!("setup/hold timing calibration: {:?}", delay_list);
+
+    SerdesConfig { delay: delay_list }
+}
+
+fn read_align(dly: u8, timer: &mut GlobalTimer) -> f32 {
+    unsafe {
+        apply_delay(dly, timer);
+        pl::csr::eem_transceiver::counter_reset_write(1);
+
+        pl::csr::eem_transceiver::counter_enable_write(1);
+        timer.delay_us(2000);
+        pl::csr::eem_transceiver::counter_enable_write(0);
+
+        let (high, low) = (
+            pl::csr::eem_transceiver::counter_high_count_read(),
+            pl::csr::eem_transceiver::counter_low_count_read(),
+        );
+        if pl::csr::eem_transceiver::counter_overflow_read() == 1 {
+            panic!("Unexpected phase detector counter overflow");
+        }
+
+        low as f32 / (low + high) as f32
+    }
+}
+
+unsafe fn align_comma(timer: &mut GlobalTimer) {
+    loop {
+        for slip in 1..=10 {
+            // The soft transceiver has 2 8b10b decoders, which receives lane
+            // 0/1 and lane 2/3 respectively. The decoder are time-multiplexed
+            // to decode exactly 1 lane each sysclk cycle.
+            //
+            // The decoder decodes lane 0/2 data on odd sysclk cycles, buffer
+            // on even cycles, and vice versa for lane 1/3. Data/Clock latency
+            // could change timing. The extend bit flips the decoding timing,
+            // so lane 0/2 data are decoded on even cycles, and lane 1/3 data
+            // are decoded on odd cycles.
+            //
+            // This is needed because transmitting/receiving a 8b10b character
+            // takes 2 sysclk cycles. Adjusting bitslip only via ISERDES
+            // limits the range to 1 cycle. The wordslip bit extends the range
+            // to 2 sysclk cycles.
+            pl::csr::eem_transceiver::wordslip_write((slip > 5) as u8);
+
+            // Apply a double bitslip since the ISERDES is 2x oversampled.
+            // Bitslip is used for comma alignment purposes once setup/hold
+            // timing is met.
+            pl::csr::eem_transceiver::bitslip_write(1);
+            pl::csr::eem_transceiver::bitslip_write(1);
+            timer.delay_us(1);
+
+            pl::csr::eem_transceiver::comma_align_reset_write(1);
+            timer.delay_us(100);
+
+            if pl::csr::eem_transceiver::comma_read() == 1 {
+                debug!("comma alignment completed after {} bitslips", slip);
+                return;
+            }
+        }
+
+        error!("comma alignment failed, retrying in 1s...");
+        timer.delay_us(1_000_000);
+    }
+}
+
+pub fn init(timer: &mut GlobalTimer, cfg: &Config) {
+    for trx_no in 0..pl::csr::CONFIG_EEM_DRTIO_COUNT {
+        unsafe {
+            pl::csr::eem_transceiver::transceiver_sel_write(trx_no as u8);
+        }
+
+        let key = format!("eem_drtio_delay{}", trx_no);
+
+        let cfg_read = cfg.read(&key);
+        match cfg_read {
+            Ok(record) => {
+                info!("loading calibrated timing values from sd card");
+                unsafe {
+                    apply_config(&*(record.as_ptr() as *const SerdesConfig), timer);
+                }
+            }
+            Err(_) => {
+                info!("calibrating...");
+                let config = unsafe { assign_delay(timer) };
+
+                match cfg.write(&key, config.as_bytes().to_vec()) {
+                    Ok(()) => {
+                        info!("storing calibration timing values into sd card");
+                    }
+                    Err(e) => {
+                        error!(
+                            "calibration successful but calibration timing values cannot be stored into sd card. \
+                             Error:{}",
+                            e
+                        );
+                    }
+                };
+            }
+        }
+
+        unsafe {
+            align_comma(timer);
+            pl::csr::eem_transceiver::rx_ready_write(1);
+        }
+    }
+}

src/libboard_artiq/src/drtio_routing.rs

@@ -1,9 +1,10 @@
-use libconfig::Config;
-#[cfg(has_drtio_routing)]
-use crate::pl::csr;
 use core::fmt;
 
-use log::{warn, info};
+use libconfig::Config;
+use log::{info, warn};
+
+#[cfg(has_drtio_routing)]
+use crate::pl::csr;
 
 #[cfg(has_drtio_routing)]
 pub const DEST_COUNT: usize = 256;
@@ -18,7 +19,7 @@ impl RoutingTable {
     // default routing table is for star topology with no repeaters
     pub fn default_master(default_n_links: usize) -> RoutingTable {
         let mut ret = RoutingTable([[INVALID_HOP; MAX_HOPS]; DEST_COUNT]);
-        let n_entries = default_n_links + 1;  // include local RTIO
+        let n_entries = default_n_links + 1; // include local RTIO
         for i in 0..n_entries {
             ret.0[i][0] = i as u8;
         }
@@ -58,10 +59,10 @@ impl fmt::Display for RoutingTable {
 pub fn config_routing_table(default_n_links: usize, cfg: &Config) -> RoutingTable {
     let mut ret = RoutingTable::default_master(default_n_links);
     if let Ok(data) = cfg.read("routing_table") {
-        if data.len() == DEST_COUNT*MAX_HOPS {
+        if data.len() == DEST_COUNT * MAX_HOPS {
             for i in 0..DEST_COUNT {
                 for j in 0..MAX_HOPS {
-                    ret.0[i][j] = data[i*MAX_HOPS+j];
+                    ret.0[i][j] = data[i * MAX_HOPS + j];
                 }
             }
         } else {

src/libboard_artiq/src/drtioaux.rs

@@ -1,14 +1,13 @@
-use crc;
+use core::slice;
 
-use core_io::{ErrorKind as IoErrorKind, Error as IoError};
+use core_io::{Error as IoError, ErrorKind as IoErrorKind};
-use io::{proto::ProtoRead, proto::ProtoWrite, Cursor};
+use crc;
-use libboard_zynq::{timer::GlobalTimer, time::Milliseconds};
+use io::{proto::{ProtoRead, ProtoWrite},
-use libcortex_a9::asm::dmb;
+         Cursor};
-use crate::mem::mem::DRTIOAUX_MEM;
+use libboard_zynq::{time::Milliseconds, timer::GlobalTimer};
-use crate::pl::csr::DRTIOAUX;
-use crate::drtioaux_proto::Error as ProtocolError;
 
 pub use crate::drtioaux_proto::Packet;
+use crate::{drtioaux_proto::Error as ProtocolError, mem::mem::DRTIOAUX_MEM, pl::csr::DRTIOAUX};
 
 #[derive(Debug)]
 pub enum Error {
@@ -21,7 +20,7 @@ pub enum Error {
 
     RoutingError,
 
-    Protocol(ProtocolError)
+    Protocol(ProtocolError),
 }
 
 impl From<ProtocolError> for Error {
@@ -58,31 +57,14 @@ pub fn has_rx_error(linkno: u8) -> bool {
     }
 }
 
-pub fn copy_work_buffer(src: *mut u32, dst: *mut u32, len: isize) {
-    // AXI writes must be 4-byte aligned (drtio proto doesn't care for that),
-    // and AXI burst reads/writes are not implemented yet in gateware
-    // thus the need for a work buffer for transmitting and copying it over
-    unsafe {
-        for i in 0..(len/4) {
-            *dst.offset(i) = *src.offset(i);
-            //data memory barrier to prevent bursts
-            dmb();
-        }
-    }
-}
-
 fn receive<F, T>(linkno: u8, f: F) -> Result<Option<T>, Error>
-    where F: FnOnce(&[u8]) -> Result<T, Error>
+where F: FnOnce(&[u8]) -> Result<T, Error> {
-{
     let linkidx = linkno as usize;
     unsafe {
         if (DRTIOAUX[linkidx].aux_rx_present_read)() == 1 {
-            let ptr = (DRTIOAUX_MEM[linkidx].base + DRTIOAUX_MEM[linkidx].size / 2) as *mut u32;
+            let read_ptr = (DRTIOAUX[linkidx].aux_read_pointer_read)() as usize;
-            let len = (DRTIOAUX[linkidx].aux_rx_length_read)() as usize;
+            let ptr = (DRTIOAUX_MEM[linkidx].base + DRTIOAUX_MEM[linkidx].size / 2 + read_ptr * 0x400) as *mut u32;
-            // work buffer to accomodate axi burst reads
+            let result = f(slice::from_raw_parts(ptr as *mut u8, 0x400 as usize));
-            let mut buf: [u8; 1024] = [0; 1024];
-            copy_work_buffer(ptr, buf.as_mut_ptr() as *mut u32, len as isize);
-            let result = f(&buf[0..len]);
             (DRTIOAUX[linkidx].aux_rx_present_write)(1);
             Ok(Some(result?))
         } else {
@@ -93,31 +75,29 @@ fn receive<F, T>(linkno: u8, f: F) -> Result<Option<T>, Error>
 
 pub fn recv(linkno: u8) -> Result<Option<Packet>, Error> {
     if has_rx_error(linkno) {
-        return Err(Error::GatewareError)
+        return Err(Error::GatewareError);
     }
 
     receive(linkno, |buffer| {
         if buffer.len() < 8 {
-            return Err(IoError::new(IoErrorKind::UnexpectedEof, "Unexpected end").into())
+            return Err(IoError::new(IoErrorKind::UnexpectedEof, "Unexpected end").into());
         }
 
         let mut reader = Cursor::new(buffer);
 
-        let checksum_at = buffer.len() - 4;
+        let packet = Packet::read_from(&mut reader)?;
+        let padding = (12 - (reader.position() % 8)) % 8;
+        let checksum_at = reader.position() + padding;
         let checksum = crc::crc32::checksum_ieee(&reader.get_ref()[0..checksum_at]);
         reader.set_position(checksum_at);
         if reader.read_u32()? != checksum {
-            return Err(Error::CorruptedPacket)
+            return Err(Error::CorruptedPacket);
         }
-        reader.set_position(0);
+        Ok(packet)
-
-        Ok(Packet::read_from(&mut reader)?)
     })
 }
 
-pub fn recv_timeout(linkno: u8, timeout_ms: Option<u64>,
+pub fn recv_timeout(linkno: u8, timeout_ms: Option<u64>, timer: GlobalTimer) -> Result<Packet, Error> {
-    timer: GlobalTimer) -> Result<Packet, Error> 
-{
     let timeout_ms = Milliseconds(timeout_ms.unwrap_or(10));
     let limit = timer.get_time() + timeout_ms;
     while timer.get_time() < limit {
@@ -130,17 +110,12 @@ pub fn recv_timeout(linkno: u8, timeout_ms: Option<u64>,
 }
 
 fn transmit<F>(linkno: u8, f: F) -> Result<(), Error>
-    where F: FnOnce(&mut [u8]) -> Result<usize, Error>
+where F: FnOnce(&mut [u8]) -> Result<usize, Error> {
-{
     let linkno = linkno as usize;
     unsafe {
         while (DRTIOAUX[linkno].aux_tx_read)() != 0 {}
         let ptr = DRTIOAUX_MEM[linkno].base as *mut u32;
-        let len = DRTIOAUX_MEM[linkno].size / 2;
+        let len = f(slice::from_raw_parts_mut(ptr as *mut u8, 0x400 as usize))?;
-        // work buffer, works with unaligned mem access
-        let mut buf: [u8; 1024] = [0; 1024]; 
-        let len = f(&mut buf[0..len])?;
-        copy_work_buffer(buf.as_mut_ptr() as *mut u32, ptr, len as isize);
         (DRTIOAUX[linkno].aux_tx_length_write)(len as u16);
         (DRTIOAUX[linkno].aux_tx_write)(1);
         Ok(())

src/libboard_artiq/src/drtioaux_async.rs

@@ -1,18 +1,18 @@
+use core::slice;
+
+use core_io::{Error as IoError, ErrorKind as IoErrorKind};
 use crc;
-
+use io::{proto::{ProtoRead, ProtoWrite},
-use core_io::{ErrorKind as IoErrorKind, Error as IoError};
+         Cursor};
-use void::Void;
+use libasync::{block_async, task};
+use libboard_zynq::{time::Milliseconds, timer::GlobalTimer};
 use nb;
-
+use void::Void;
-use libboard_zynq::{timer::GlobalTimer, time::Milliseconds};
-use libasync::{task, block_async};
-
-use io::{proto::ProtoRead, proto::ProtoWrite, Cursor};
-use crate::mem::mem::DRTIOAUX_MEM;
-use crate::pl::csr::DRTIOAUX;
-use crate::drtioaux::{Error, has_rx_error, copy_work_buffer};
 
 pub use crate::drtioaux_proto::Packet;
+use crate::{drtioaux::{has_rx_error, Error},
+            mem::mem::DRTIOAUX_MEM,
+            pl::csr::DRTIOAUX};
 
 pub async fn reset(linkno: u8) {
     let linkno = linkno as usize;
@@ -29,25 +29,20 @@ fn tx_ready(linkno: usize) -> nb::Result<(), Void> {
     unsafe {
         if (DRTIOAUX[linkno].aux_tx_read)() != 0 {
             Err(nb::Error::WouldBlock)
-        }
+        } else {
-        else {
             Ok(())
         }
     }
 }
 
 async fn receive<F, T>(linkno: u8, f: F) -> Result<Option<T>, Error>
-    where F: FnOnce(&[u8]) -> Result<T, Error>
+where F: FnOnce(&[u8]) -> Result<T, Error> {
-{
     let linkidx = linkno as usize;
     unsafe {
         if (DRTIOAUX[linkidx].aux_rx_present_read)() == 1 {
-            let ptr = (DRTIOAUX_MEM[linkidx].base + DRTIOAUX_MEM[linkidx].size / 2) as *mut u32;
+            let read_ptr = (DRTIOAUX[linkidx].aux_read_pointer_read)() as usize;
-            let len = (DRTIOAUX[linkidx].aux_rx_length_read)() as usize;
+            let ptr = (DRTIOAUX_MEM[linkidx].base + DRTIOAUX_MEM[linkidx].size / 2 + read_ptr * 0x400) as *mut u32;
-            // work buffer to accomodate axi burst reads
+            let result = f(slice::from_raw_parts(ptr as *mut u8, 0x400 as usize));
-            let mut buf: [u8; 1024] = [0; 1024];
-            copy_work_buffer(ptr, buf.as_mut_ptr() as *mut u32, len as isize);
-            let result = f(&buf[0..len]);
             (DRTIOAUX[linkidx].aux_rx_present_write)(1);
             Ok(Some(result?))
         } else {
@@ -58,31 +53,30 @@ async fn receive<F, T>(linkno: u8, f: F) -> Result<Option<T>, Error>
 
 pub async fn recv(linkno: u8) -> Result<Option<Packet>, Error> {
     if has_rx_error(linkno) {
-        return Err(Error::GatewareError)
+        return Err(Error::GatewareError);
     }
 
     receive(linkno, |buffer| {
         if buffer.len() < 8 {
-            return Err(IoError::new(IoErrorKind::UnexpectedEof, "Unexpected end").into())
+            return Err(IoError::new(IoErrorKind::UnexpectedEof, "Unexpected end").into());
         }
 
         let mut reader = Cursor::new(buffer);
 
-        let checksum_at = buffer.len() - 4;
+        let packet = Packet::read_from(&mut reader)?;
+        let padding = (12 - (reader.position() % 8)) % 8;
+        let checksum_at = reader.position() + padding;
         let checksum = crc::crc32::checksum_ieee(&reader.get_ref()[0..checksum_at]);
         reader.set_position(checksum_at);
         if reader.read_u32()? != checksum {
-            return Err(Error::CorruptedPacket)
+            return Err(Error::CorruptedPacket);
         }
-        reader.set_position(0);
+        Ok(packet)
-
+    })
-        Ok(Packet::read_from(&mut reader)?)
+    .await
-    }).await
 }
 
-pub async fn recv_timeout(linkno: u8, timeout_ms: Option<u64>,
+pub async fn recv_timeout(linkno: u8, timeout_ms: Option<u64>, timer: GlobalTimer) -> Result<Packet, Error> {
-    timer: GlobalTimer) -> Result<Packet, Error> 
-{
     let timeout_ms = Milliseconds(timeout_ms.unwrap_or(10));
     let limit = timer.get_time() + timeout_ms;
     let mut would_block = false;
@@ -93,7 +87,9 @@ pub async fn recv_timeout(linkno: u8, timeout_ms: Option<u64>,
             task::r#yield().await;
         }
         match recv(linkno).await? {
-            None => { would_block = true; },
+            None => {
+                would_block = true;
+            }
             Some(packet) => return Ok(packet),
         }
     }
@@ -101,17 +97,12 @@ pub async fn recv_timeout(linkno: u8, timeout_ms: Option<u64>,
 }
 
 async fn transmit<F>(linkno: u8, f: F) -> Result<(), Error>
-    where F: FnOnce(&mut [u8]) -> Result<usize, Error>
+where F: FnOnce(&mut [u8]) -> Result<usize, Error> {
-{
     let linkno = linkno as usize;
     unsafe {
         let _ = block_async!(tx_ready(linkno)).await;
         let ptr = DRTIOAUX_MEM[linkno].base as *mut u32;
-        let len = DRTIOAUX_MEM[linkno].size / 2;
+        let len = f(slice::from_raw_parts_mut(ptr as *mut u8, 0x400 as usize))?;
-        // work buffer, works with unaligned mem access
-        let mut buf: [u8; 1024] = [0; 1024]; 
-        let len = f(&mut buf[0..len])?;
-        copy_work_buffer(buf.as_mut_ptr() as *mut u32, ptr, len as isize);
         (DRTIOAUX[linkno].aux_tx_length_write)(len as u16);
         (DRTIOAUX[linkno].aux_tx_write)(1);
         Ok(())
@@ -134,5 +125,6 @@ pub async fn send(linkno: u8, packet: &Packet) -> Result<(), Error> {
         writer.write_u32(checksum)?;
 
         Ok(writer.position())
-    }).await
+    })
+    .await
 }

src/libboard_artiq/src/fiq.rs

@@ -0,0 +1,22 @@
+use libboard_zynq::{println, stdio};
+use libcortex_a9::{interrupt_handler, regs::MPIDR};
+use libregister::RegisterR;
+
+#[cfg(has_si549)]
+use crate::si549;
+
+interrupt_handler!(FIQ, fiq, __irq_stack0_start, __irq_stack1_start, {
+    match MPIDR.read().cpu_id() {
+        0 => {
+            // nFIQ is driven directly and bypass GIC
+            #[cfg(has_si549)]
+            si549::wrpll::interrupt_handler();
+            return;
+        }
+        _ => {}
+    };
+
+    stdio::drop_uart();
+    println!("FIQ");
+    loop {}
+});

src/libboard_artiq/src/io_expander.rs

@@ -1,6 +1,7 @@
 use libboard_zynq::i2c;
 use log::info;
 
+#[cfg(has_virtual_leds)]
 use crate::pl::csr;
 
 // Only the bare minimum registers. Bits/IO connections equivalent between IC types.
@@ -19,11 +20,15 @@ const IODIR_OUT_SFP_LED: u8 = 0x40;
 const IODIR_OUT_SFP0_LED: u8 = 0x40;
 #[cfg(hw_rev = "v1.1")]
 const IODIR_OUT_SFP0_LED: u8 = 0x80;
+#[cfg(has_si549)]
+const IODIR_CLK_SEL: u8 = 0x80; // out
+#[cfg(has_si5324)]
+const IODIR_CLK_SEL: u8 = 0x00; // in
 
 //IO expander port direction
 const IODIR0: [u8; 2] = [
     0xFF & !IODIR_OUT_SFP_TX_DISABLE & !IODIR_OUT_SFP0_LED,
-    0xFF & !IODIR_OUT_SFP_TX_DISABLE & !IODIR_OUT_SFP_LED,
+    0xFF & !IODIR_OUT_SFP_TX_DISABLE & !IODIR_OUT_SFP_LED & !IODIR_CLK_SEL,
 ];
 
 const IODIR1: [u8; 2] = [
@@ -33,6 +38,7 @@ const IODIR1: [u8; 2] = [
 
 pub struct IoExpander {
     address: u8,
+    #[cfg(has_virtual_leds)]
     virtual_led_mapping: &'static [(u8, u8, u8)],
     iodir: [u8; 2],
     out_current: [u8; 2],
@@ -42,17 +48,18 @@ pub struct IoExpander {
 
 impl IoExpander {
     pub fn new(i2c: &mut i2c::I2c, index: u8) -> Result<Self, &'static str> {
-        #[cfg(hw_rev = "v1.0")]
+        #[cfg(all(hw_rev = "v1.0", has_virtual_leds))]
         const VIRTUAL_LED_MAPPING0: [(u8, u8, u8); 2] = [(0, 0, 6), (1, 1, 6)];
-        #[cfg(hw_rev = "v1.1")]
+        #[cfg(all(hw_rev = "v1.1", has_virtual_leds))]
         const VIRTUAL_LED_MAPPING0: [(u8, u8, u8); 2] = [(0, 0, 7), (1, 1, 6)];
-
+        #[cfg(has_virtual_leds)]
         const VIRTUAL_LED_MAPPING1: [(u8, u8, u8); 2] = [(2, 0, 6), (3, 1, 6)];
 
         // Both expanders on SHARED I2C bus
         let mut io_expander = match index {
             0 => IoExpander {
                 address: 0x40,
+                #[cfg(has_virtual_leds)]
                 virtual_led_mapping: &VIRTUAL_LED_MAPPING0,
                 iodir: IODIR0,
                 out_current: [0; 2],
@@ -66,6 +73,7 @@ impl IoExpander {
             },
             1 => IoExpander {
                 address: 0x42,
+                #[cfg(has_virtual_leds)]
                 virtual_led_mapping: &VIRTUAL_LED_MAPPING1,
                 iodir: IODIR1,
                 out_current: [0; 2],
@@ -80,10 +88,7 @@ impl IoExpander {
             _ => return Err("incorrect I/O expander index"),
         };
         if !io_expander.check_ack(i2c)? {
-            info!(
+            info!("MCP23017 io expander {} not found. Checking for PCA9539.", index);
-                "MCP23017 io expander {} not found. Checking for PCA9539.",
-                index
-            );
             io_expander.address += 0xa8; // translate to PCA9539 addresses (see schematic)
             io_expander.registers = Registers {
                 iodira: 0x06,
@@ -130,6 +135,7 @@ impl IoExpander {
 
     pub fn init(&mut self, i2c: &mut i2c::I2c) -> Result<(), &'static str> {
         self.select(i2c)?;
+
         self.update_iodir(i2c)?;
 
         self.out_current[0] = 0x00;

src/libboard_artiq/src/lib.rs

@@ -1,40 +1,46 @@
 #![no_std]
 #![feature(never_type)]
+#![feature(naked_functions)]
+#![feature(asm)]
 
-extern crate log;
+extern crate core_io;
 extern crate crc;
 extern crate embedded_hal;
-extern crate core_io;
 extern crate io;
+extern crate libasync;
 extern crate libboard_zynq;
-extern crate libregister;
 extern crate libconfig;
 extern crate libcortex_a9;
-extern crate libasync;
+extern crate libregister;
+extern crate log;
 extern crate log_buffer;
 
-#[path = "../../../build/pl.rs"]
-pub mod pl;
-pub mod drtioaux_proto;
 pub mod drtio_routing;
-pub mod logger;
-#[cfg(has_si5324)]
-pub mod si5324;
 #[cfg(has_drtio)]
 pub mod drtioaux;
 #[cfg(has_drtio)]
 pub mod drtioaux_async;
+pub mod drtioaux_proto;
+pub mod fiq;
+#[cfg(feature = "target_kasli_soc")]
+pub mod io_expander;
+pub mod logger;
 #[cfg(has_drtio)]
+#[rustfmt::skip]
 #[path = "../../../build/mem.rs"]
 pub mod mem;
-#[cfg(all(feature = "target_kasli_soc", has_drtio))]
+#[rustfmt::skip]
-pub mod io_expander;
+#[path = "../../../build/pl.rs"]
+pub mod pl;
+#[cfg(has_drtio_eem)]
+pub mod drtio_eem;
 #[cfg(has_grabber)]
 pub mod grabber;
-
+#[cfg(has_si5324)]
+pub mod si5324;
+#[cfg(has_si549)]
+pub mod si549;
 use core::{cmp, str};
-use libboard_zynq::slcr;
-use libregister::RegisterW;
 
 pub fn identifier_read(buf: &mut [u8]) -> &str {
     unsafe {
@@ -48,26 +54,3 @@ pub fn identifier_read(buf: &mut [u8]) -> &str {
         str::from_utf8_unchecked(&buf[..len as usize])
     }
 }
-
-pub fn init_gateware() {
-    // Set up PS->PL clocks
-    slcr::RegisterBlock::unlocked(|slcr| {
-        // As we are touching the mux, the clock may glitch, so reset the PL.
-        slcr.fpga_rst_ctrl.write(
-            slcr::FpgaRstCtrl::zeroed()
-                .fpga0_out_rst(true)
-                .fpga1_out_rst(true)
-                .fpga2_out_rst(true)
-                .fpga3_out_rst(true)
-        );
-        slcr.fpga0_clk_ctrl.write(
-            slcr::Fpga0ClkCtrl::zeroed()
-                .src_sel(slcr::PllSource::IoPll)
-                .divisor0(8)
-                .divisor1(1)
-        );
-        slcr.fpga_rst_ctrl.write(
-            slcr::FpgaRstCtrl::zeroed()
-        );
-    });
-}

src/libboard_artiq/src/logger.rs

@@ -1,13 +1,13 @@
-use core::cell::Cell;
+use core::{cell::Cell, fmt::Write};
-use core::fmt::Write;
+
-use log::{Log, LevelFilter};
-use log_buffer::LogBuffer;
-use libcortex_a9::mutex::{Mutex, MutexGuard};
 use libboard_zynq::{println, timer::GlobalTimer};
+use libcortex_a9::mutex::{Mutex, MutexGuard};
+use log::{LevelFilter, Log};
+use log_buffer::LogBuffer;
 
 pub struct LogBufferRef<'a> {
-    buffer:        MutexGuard<'a, LogBuffer<&'static mut [u8]>>,
+    buffer: MutexGuard<'a, LogBuffer<&'static mut [u8]>>,
-    old_log_level: LevelFilter
+    old_log_level: LevelFilter,
 }
 
 impl<'a> LogBufferRef<'a> {
@@ -37,7 +37,7 @@ impl<'a> Drop for LogBufferRef<'a> {
 }
 
 pub struct BufferLogger {
-    buffer:      Mutex<LogBuffer<&'static mut [u8]>>,
+    buffer: Mutex<LogBuffer<&'static mut [u8]>>,
     uart_filter: Cell<LevelFilter>,
     buffer_filter: Cell<LevelFilter>,
 }
@@ -56,8 +56,7 @@ impl BufferLogger {
     pub fn register(self) {
         unsafe {
             LOGGER = Some(self);
-            log::set_logger(LOGGER.as_ref().unwrap())
+            log::set_logger(LOGGER.as_ref().unwrap()).expect("global logger can only be initialized once");
-                .expect("global logger can only be initialized once");
         }
     }
 
@@ -66,9 +65,7 @@ impl BufferLogger {
     }
 
     pub fn buffer<'a>(&'a self) -> Option<LogBufferRef<'a>> {
-        self.buffer
+        self.buffer.try_lock().map(LogBufferRef::new)
-            .try_lock()
-            .map(LogBufferRef::new)
     }
 
     pub fn uart_log_level(&self) -> LevelFilter {
@@ -99,25 +96,36 @@ impl Log for BufferLogger {
 
     fn log(&self, record: &log::Record) {
         if self.enabled(record.metadata()) {
-            let timestamp = unsafe {
+            let timestamp = unsafe { GlobalTimer::get() }.get_us().0;
-                GlobalTimer::get()
+            let seconds = timestamp / 1_000_000;
-            }.get_us().0;
+            let micros = timestamp % 1_000_000;
-            let seconds   = timestamp / 1_000_000;
-            let micros    = timestamp % 1_000_000;
 
             if record.level() <= self.buffer_log_level() {
                 let mut buffer = self.buffer.lock();
-                writeln!(buffer, "[{:6}.{:06}s] {:>5}({}): {}", seconds, micros,
+                writeln!(
-                         record.level(), record.target(), record.args()).unwrap();
+                    buffer,
+                    "[{:6}.{:06}s] {:>5}({}): {}",
+                    seconds,
+                    micros,
+                    record.level(),
+                    record.target(),
+                    record.args()
+                )
+                .unwrap();
             }
 
             if record.level() <= self.uart_log_level() {
-                println!("[{:6}.{:06}s] {:>5}({}): {}", seconds, micros,
+                println!(
-                         record.level(), record.target(), record.args());
+                    "[{:6}.{:06}s] {:>5}({}): {}",
+                    seconds,
+                    micros,
+                    record.level(),
+                    record.target(),
+                    record.args()
+                );
             }
         }
     }
 
-    fn flush(&self) {
+    fn flush(&self) {}
-    }
 }

src/libboard_artiq/src/si5324.rs

@@ -1,7 +1,9 @@
 use core::result;
-use log::info;
+
-use libboard_zynq::{i2c::I2c, timer::GlobalTimer, time::Milliseconds};
 use embedded_hal::blocking::delay::DelayUs;
+use libboard_zynq::{i2c::I2c, time::Milliseconds, timer::GlobalTimer};
+use log::info;
+
 #[cfg(not(si5324_soft_reset))]
 use crate::pl::csr;
 
@@ -11,9 +13,13 @@ const ADDRESS: u8 = 0x68;
 
 #[cfg(not(si5324_soft_reset))]
 fn hard_reset(timer: &mut GlobalTimer) {
-    unsafe { csr::si5324_rst_n::out_write(0); }
+    unsafe {
+        csr::si5324_rst_n::out_write(0);
+    }
     timer.delay_us(1_000);
-    unsafe { csr::si5324_rst_n::out_write(1); }
+    unsafe {
+        csr::si5324_rst_n::out_write(1);
+    }
     timer.delay_us(10_000);
 }
 
@@ -29,7 +35,7 @@ pub struct FrequencySettings {
     pub n31: u32,
     pub n32: u32,
     pub bwsel: u8,
-    pub crystal_ref: bool
+    pub crystal_as_ckin2: bool,
 }
 
 pub enum Input {
@@ -39,52 +45,52 @@ pub enum Input {
 
 fn map_frequency_settings(settings: &FrequencySettings) -> Result<FrequencySettings> {
     if settings.nc1_ls != 0 && (settings.nc1_ls % 2) == 1 {
-        return Err("NC1_LS must be 0 or even")
+        return Err("NC1_LS must be 0 or even");
     }
     if settings.nc1_ls > (1 << 20) {
-        return Err("NC1_LS is too high")
+        return Err("NC1_LS is too high");
     }
     if (settings.n2_ls % 2) == 1 {
-        return Err("N2_LS must be even")
+        return Err("N2_LS must be even");
     }
     if settings.n2_ls > (1 << 20) {
-        return Err("N2_LS is too high")
+        return Err("N2_LS is too high");
     }
     if settings.n31 > (1 << 19) {
-        return Err("N31 is too high")
+        return Err("N31 is too high");
     }
     if settings.n32 > (1 << 19) {
-        return Err("N32 is too high")
+        return Err("N32 is too high");
     }
     let r = FrequencySettings {
         n1_hs: match settings.n1_hs {
-            4  => 0b000,
+            4 => 0b000,
-            5  => 0b001,
+            5 => 0b001,
-            6  => 0b010,
+            6 => 0b010,
-            7  => 0b011,
+            7 => 0b011,
-            8  => 0b100,
+            8 => 0b100,
-            9  => 0b101,
+            9 => 0b101,
             10 => 0b110,
             11 => 0b111,
-            _  => return Err("N1_HS has an invalid value")
+            _ => return Err("N1_HS has an invalid value"),
         },
         nc1_ls: settings.nc1_ls - 1,
         n2_hs: match settings.n2_hs {
-            4  => 0b000,
+            4 => 0b000,
-            5  => 0b001,
+            5 => 0b001,
-            6  => 0b010,
+            6 => 0b010,
-            7  => 0b011,
+            7 => 0b011,
-            8  => 0b100,
+            8 => 0b100,
-            9  => 0b101,
+            9 => 0b101,
             10 => 0b110,
             11 => 0b111,
-            _  => return Err("N2_HS has an invalid value")
+            _ => return Err("N2_HS has an invalid value"),
         },
         n2_ls: settings.n2_ls - 1,
         n31: settings.n31 - 1,
         n32: settings.n32 - 1,
         bwsel: settings.bwsel,
-        crystal_ref: settings.crystal_ref
+        crystal_as_ckin2: settings.crystal_as_ckin2,
     };
     Ok(r)
 }
@@ -92,13 +98,13 @@ fn map_frequency_settings(settings: &FrequencySettings) -> Result<FrequencySetti
 fn write(i2c: &mut I2c, reg: u8, val: u8) -> Result<()> {
     i2c.start().unwrap();
     if !i2c.write(ADDRESS << 1).unwrap() {
-        return Err("Si5324 failed to ack write address")
+        return Err("Si5324 failed to ack write address");
     }
     if !i2c.write(reg).unwrap() {
-        return Err("Si5324 failed to ack register")
+        return Err("Si5324 failed to ack register");
     }
     if !i2c.write(val).unwrap() {
-        return Err("Si5324 failed to ack value")
+        return Err("Si5324 failed to ack value");
     }
     i2c.stop().unwrap();
     Ok(())
@@ -108,10 +114,10 @@ fn write(i2c: &mut I2c, reg: u8, val: u8) -> Result<()> {
 fn write_no_ack_value(i2c: &mut I2c, reg: u8, val: u8) -> Result<()> {
     i2c.start().unwrap();
     if !i2c.write(ADDRESS << 1).unwrap() {
-        return Err("Si5324 failed to ack write address")
+        return Err("Si5324 failed to ack write address");
     }
     if !i2c.write(reg).unwrap() {
-        return Err("Si5324 failed to ack register")
+        return Err("Si5324 failed to ack register");
     }
     i2c.write(val).unwrap();
     i2c.stop().unwrap();
@@ -121,22 +127,22 @@ fn write_no_ack_value(i2c: &mut I2c, reg: u8, val: u8) -> Result<()> {
 fn read(i2c: &mut I2c, reg: u8) -> Result<u8> {
     i2c.start().unwrap();
     if !i2c.write(ADDRESS << 1).unwrap() {
-        return Err("Si5324 failed to ack write address")
+        return Err("Si5324 failed to ack write address");
     }
     if !i2c.write(reg).unwrap() {
-        return Err("Si5324 failed to ack register")
+        return Err("Si5324 failed to ack register");
     }
     i2c.restart().unwrap();
     if !i2c.write((ADDRESS << 1) | 1).unwrap() {
-        return Err("Si5324 failed to ack read address")
+        return Err("Si5324 failed to ack read address");
     }
     let val = i2c.read(false).unwrap();
     i2c.stop().unwrap();
     Ok(val)
 }
 
-fn rmw<F>(i2c: &mut I2c, reg: u8, f: F) -> Result<()> where
+fn rmw<F>(i2c: &mut I2c, reg: u8, f: F) -> Result<()>
-        F: Fn(u8) -> u8 {
+where F: Fn(u8) -> u8 {
     let value = read(i2c, reg)?;
     write(i2c, reg, f(value))?;
     Ok(())
@@ -155,18 +161,18 @@ fn soft_reset(i2c: &mut I2c, timer: &mut GlobalTimer) -> Result<()> {
 }
 
 fn has_xtal(i2c: &mut I2c) -> Result<bool> {
-    Ok((read(i2c, 129)? & 0x01) == 0)  // LOSX_INT=0
+    Ok((read(i2c, 129)? & 0x01) == 0) // LOSX_INT=0
 }
 
 fn has_ckin(i2c: &mut I2c, input: Input) -> Result<bool> {
     match input {
-        Input::Ckin1 => Ok((read(i2c, 129)? & 0x02) == 0),  // LOS1_INT=0
+        Input::Ckin1 => Ok((read(i2c, 129)? & 0x02) == 0), // LOS1_INT=0
-        Input::Ckin2 => Ok((read(i2c, 129)? & 0x04) == 0),  // LOS2_INT=0
+        Input::Ckin2 => Ok((read(i2c, 129)? & 0x04) == 0), // LOS2_INT=0
     }
 }
 
 fn locked(i2c: &mut I2c) -> Result<bool> {
-    Ok((read(i2c, 130)? & 0x01) == 0)  // LOL_INT=0
+    Ok((read(i2c, 130)? & 0x01) == 0) // LOL_INT=0
 }
 
 fn monitor_lock(i2c: &mut I2c, timer: &mut GlobalTimer) -> Result<()> {
@@ -211,11 +217,11 @@ pub fn bypass(i2c: &mut I2c, input: Input, timer: &mut GlobalTimer) -> Result<()
         Input::Ckin2 => 0b01,
     };
     init(i2c, timer)?;
-    rmw(i2c, 21,  |v| v & 0xfe)?;                        // CKSEL_PIN=0
+    rmw(i2c, 21, |v| v & 0xfe)?; // CKSEL_PIN=0
-    rmw(i2c, 3,   |v| (v & 0x3f) | (cksel_reg << 6))?;   // CKSEL_REG
+    rmw(i2c, 3, |v| (v & 0x3f) | (cksel_reg << 6))?; // CKSEL_REG
-    rmw(i2c, 4,   |v| (v & 0x3f) | (0b00 << 6))?;        // AUTOSEL_REG=b00
+    rmw(i2c, 4, |v| (v & 0x3f) | (0b00 << 6))?; // AUTOSEL_REG=b00
-    rmw(i2c, 6,   |v| (v & 0xc0) | 0b111111)?;           // SFOUT2_REG=b111 SFOUT1_REG=b111
+    rmw(i2c, 6, |v| (v & 0xc0) | 0b111111)?; // SFOUT2_REG=b111 SFOUT1_REG=b111
-    rmw(i2c, 0,   |v| (v & 0xfd) | 0x02)?;               // BYPASS_REG=1
+    rmw(i2c, 0, |v| (v & 0xfd) | 0x02)?; // BYPASS_REG=1
     Ok(())
 }
 
@@ -227,32 +233,32 @@ pub fn setup(i2c: &mut I2c, settings: &FrequencySettings, input: Input, timer: &
     };
 
     init(i2c, timer)?;
-    if settings.crystal_ref {
+    if settings.crystal_as_ckin2 {
-        rmw(i2c, 0,   |v| v | 0x40)?;                     // FREE_RUN=1
+        rmw(i2c, 0, |v| v | 0x40)?; // FREE_RUN=1
     }
-    rmw(i2c, 2,   |v| (v & 0x0f) | (s.bwsel << 4))?;
+    rmw(i2c, 2, |v| (v & 0x0f) | (s.bwsel << 4))?;
-    rmw(i2c, 21,  |v| v & 0xfe)?;                        // CKSEL_PIN=0
+    rmw(i2c, 21, |v| v & 0xfe)?; // CKSEL_PIN=0
-    rmw(i2c, 3,   |v| (v & 0x2f) | (cksel_reg << 6) | 0x10)?;  // CKSEL_REG, SQ_ICAL=1
+    rmw(i2c, 3, |v| (v & 0x2f) | (cksel_reg << 6) | 0x10)?; // CKSEL_REG, SQ_ICAL=1
-    rmw(i2c, 4,   |v| (v & 0x3f) | (0b00 << 6))?;        // AUTOSEL_REG=b00
+    rmw(i2c, 4, |v| (v & 0x3f) | (0b00 << 6))?; // AUTOSEL_REG=b00
-    rmw(i2c, 6,   |v| (v & 0xc0) | 0b111111)?;           // SFOUT2_REG=b111 SFOUT1_REG=b111
+    rmw(i2c, 6, |v| (v & 0xc0) | 0b111111)?; // SFOUT2_REG=b111 SFOUT1_REG=b111
-    write(i2c, 25,  (s.n1_hs  << 5 ) as u8)?;
+    write(i2c, 25, (s.n1_hs << 5) as u8)?;
-    write(i2c, 31,  (s.nc1_ls >> 16) as u8)?;
+    write(i2c, 31, (s.nc1_ls >> 16) as u8)?;
-    write(i2c, 32,  (s.nc1_ls >> 8 ) as u8)?;
+    write(i2c, 32, (s.nc1_ls >> 8) as u8)?;
-    write(i2c, 33,  (s.nc1_ls)       as u8)?;
+    write(i2c, 33, (s.nc1_ls) as u8)?;
-    write(i2c, 34,  (s.nc1_ls >> 16) as u8)?;            // write to NC2_LS as well
+    write(i2c, 34, (s.nc1_ls >> 16) as u8)?; // write to NC2_LS as well
-    write(i2c, 35,  (s.nc1_ls >> 8 ) as u8)?;
+    write(i2c, 35, (s.nc1_ls >> 8) as u8)?;
-    write(i2c, 36,  (s.nc1_ls)       as u8)?;
+    write(i2c, 36, (s.nc1_ls) as u8)?;
-    write(i2c, 40,  (s.n2_hs  << 5 ) as u8 | (s.n2_ls  >> 16) as u8)?;
+    write(i2c, 40, (s.n2_hs << 5) as u8 | (s.n2_ls >> 16) as u8)?;
-    write(i2c, 41,  (s.n2_ls  >> 8 ) as u8)?;
+    write(i2c, 41, (s.n2_ls >> 8) as u8)?;
-    write(i2c, 42,  (s.n2_ls)        as u8)?;
+    write(i2c, 42, (s.n2_ls) as u8)?;
-    write(i2c, 43,  (s.n31    >> 16) as u8)?;
+    write(i2c, 43, (s.n31 >> 16) as u8)?;
-    write(i2c, 44,  (s.n31    >> 8)  as u8)?;
+    write(i2c, 44, (s.n31 >> 8) as u8)?;
-    write(i2c, 45,  (s.n31)          as u8)?;
+    write(i2c, 45, (s.n31) as u8)?;
-    write(i2c, 46,  (s.n32    >> 16) as u8)?;
+    write(i2c, 46, (s.n32 >> 16) as u8)?;
-    write(i2c, 47,  (s.n32    >> 8)  as u8)?;
+    write(i2c, 47, (s.n32 >> 8) as u8)?;
-    write(i2c, 48,  (s.n32)          as u8)?;
+    write(i2c, 48, (s.n32) as u8)?;
-    rmw(i2c, 137, |v| v | 0x01)?;                       // FASTLOCK=1
+    rmw(i2c, 137, |v| v | 0x01)?; // FASTLOCK=1
-    rmw(i2c, 136, |v| v | 0x40)?;                       // ICAL=1
+    rmw(i2c, 136, |v| v | 0x40)?; // ICAL=1
 
     if !has_xtal(i2c)? {
         return Err("Si5324 misses XA/XB signal");
@@ -270,7 +276,7 @@ pub fn select_input(i2c: &mut I2c, input: Input, timer: &mut GlobalTimer) -> Res
         Input::Ckin1 => 0b00,
         Input::Ckin2 => 0b01,
     };
-    rmw(i2c, 3,   |v| (v & 0x3f) | (cksel_reg << 6))?;
+    rmw(i2c, 3, |v| (v & 0x3f) | (cksel_reg << 6))?;
     if !has_ckin(i2c, input)? {
         return Err("Si5324 misses clock input signal");
     }
@@ -285,12 +291,12 @@ pub mod siphaser {
 
     pub fn select_recovered_clock(i2c: &mut I2c, rc: bool, timer: &mut GlobalTimer) -> Result<()> {
         let val = read(i2c, 3)?;
-        write(i2c, 3,   (val & 0xdf) | (1 << 5))?;  // DHOLD=1
+        write(i2c, 3, (val & 0xdf) | (1 << 5))?; // DHOLD=1
         unsafe {
             csr::siphaser::switch_clocks_write(if rc { 1 } else { 0 });
         }
         let val = read(i2c, 3)?;
-        write(i2c, 3,   (val & 0xdf) | (0 << 5))?;  // DHOLD=0
+        write(i2c, 3, (val & 0xdf) | (0 << 5))?; // DHOLD=0
         monitor_lock(i2c, timer)?;
         Ok(())
     }
@@ -309,9 +315,7 @@ pub mod siphaser {
             csr::siphaser::error_write(1);
         }
         timer.delay_us(5_000);
-        unsafe {
+        unsafe { csr::siphaser::error_read() != 0 }
-            csr::siphaser::error_read() != 0
-        }
     }
 
     fn find_edge(target: bool, timer: &mut GlobalTimer) -> Result<u32> {
@@ -340,11 +344,16 @@ pub mod siphaser {
         }
         let width = find_edge(true, timer)? + jitter_margin;
         // width is 360 degrees (one full rotation of the phase between s/h limits) minus jitter
-        info!("calibration successful, lead: {}, width: {} ({}deg)", lead, width, width*360/(56*8));
+        info!(
+            "calibration successful, lead: {}, width: {} ({}deg)",
+            lead,
+            width,
+            width * 360 / (56 * 8)
+        );
 
         // Apply reverse phase shift for half the width to get into the
         // middle of the working region.
-        for _ in 0..width/2 {
+        for _ in 0..width / 2 {
             phase_shift(0, timer);
         }
 

src/libboard_artiq/src/si549.rs

@@ -0,0 +1,854 @@
+use embedded_hal::prelude::_embedded_hal_blocking_delay_DelayUs;
+use libboard_zynq::timer::GlobalTimer;
+use log::info;
+
+use crate::pl::csr;
+
+#[cfg(feature = "target_kasli_soc")]
+const ADDRESS: u8 = 0x67;
+
+const ADPLL_MAX: i32 = (950.0 / 0.0001164) as i32;
+
+pub struct DividerConfig {
+    pub hsdiv: u16,
+    pub lsdiv: u8,
+    pub fbdiv: u64,
+}
+
+pub struct FrequencySetting {
+    pub main: DividerConfig,
+    pub helper: DividerConfig,
+}
+
+mod i2c {
+    use super::*;
+
+    #[derive(Clone, Copy)]
+    pub enum DCXO {
+        Main,
+        Helper,
+    }
+
+    fn half_period(timer: &mut GlobalTimer) {
+        timer.delay_us(1)
+    }
+
+    fn sda_i(dcxo: DCXO) -> bool {
+        match dcxo {
+            DCXO::Main => unsafe { csr::wrpll::main_dcxo_sda_in_read() == 1 },
+            DCXO::Helper => unsafe { csr::wrpll::helper_dcxo_sda_in_read() == 1 },
+        }
+    }
+
+    fn sda_oe(dcxo: DCXO, oe: bool) {
+        let val = if oe { 1 } else { 0 };
+        match dcxo {
+            DCXO::Main => unsafe { csr::wrpll::main_dcxo_sda_oe_write(val) },
+            DCXO::Helper => unsafe { csr::wrpll::helper_dcxo_sda_oe_write(val) },
+        };
+    }
+
+    fn sda_o(dcxo: DCXO, o: bool) {
+        let val = if o { 1 } else { 0 };
+        match dcxo {
+            DCXO::Main => unsafe { csr::wrpll::main_dcxo_sda_out_write(val) },
+            DCXO::Helper => unsafe { csr::wrpll::helper_dcxo_sda_out_write(val) },
+        };
+    }
+
+    fn scl_oe(dcxo: DCXO, oe: bool) {
+        let val = if oe { 1 } else { 0 };
+        match dcxo {
+            DCXO::Main => unsafe { csr::wrpll::main_dcxo_scl_oe_write(val) },
+            DCXO::Helper => unsafe { csr::wrpll::helper_dcxo_scl_oe_write(val) },
+        };
+    }
+
+    fn scl_o(dcxo: DCXO, o: bool) {
+        let val = if o { 1 } else { 0 };
+        match dcxo {
+            DCXO::Main => unsafe { csr::wrpll::main_dcxo_scl_out_write(val) },
+            DCXO::Helper => unsafe { csr::wrpll::helper_dcxo_scl_out_write(val) },
+        };
+    }
+
+    pub fn init(dcxo: DCXO, timer: &mut GlobalTimer) -> Result<(), &'static str> {
+        // Set SCL as output, and high level
+        scl_o(dcxo, true);
+        scl_oe(dcxo, true);
+        // Prepare a zero level on SDA so that sda_oe pulls it down
+        sda_o(dcxo, false);
+        // Release SDA
+        sda_oe(dcxo, false);
+
+        // Check the I2C bus is ready
+        half_period(timer);
+        half_period(timer);
+        if !sda_i(dcxo) {
+            // Try toggling SCL a few times
+            for _bit in 0..8 {
+                scl_o(dcxo, false);
+                half_period(timer);
+                scl_o(dcxo, true);
+                half_period(timer);
+            }
+        }
+
+        if !sda_i(dcxo) {
+            return Err("SDA is stuck low and doesn't get unstuck");
+        }
+        Ok(())
+    }
+
+    pub fn start(dcxo: DCXO, timer: &mut GlobalTimer) {
+        // Set SCL high then SDA low
+        scl_o(dcxo, true);
+        half_period(timer);
+        sda_oe(dcxo, true);
+        half_period(timer);
+    }
+
+    pub fn stop(dcxo: DCXO, timer: &mut GlobalTimer) {
+        // First, make sure SCL is low, so that the target releases the SDA line
+        scl_o(dcxo, false);
+        half_period(timer);
+        // Set SCL high then SDA high
+        sda_oe(dcxo, true);
+        scl_o(dcxo, true);
+        half_period(timer);
+        sda_oe(dcxo, false);
+        half_period(timer);
+    }
+
+    pub fn write(dcxo: DCXO, data: u8, timer: &mut GlobalTimer) -> bool {
+        // MSB first
+        for bit in (0..8).rev() {
+            // Set SCL low and set our bit on SDA
+            scl_o(dcxo, false);
+            sda_oe(dcxo, data & (1 << bit) == 0);
+            half_period(timer);
+            // Set SCL high ; data is shifted on the rising edge of SCL
+            scl_o(dcxo, true);
+            half_period(timer);
+        }
+        // Check ack
+        // Set SCL low, then release SDA so that the I2C target can respond
+        scl_o(dcxo, false);
+        half_period(timer);
+        sda_oe(dcxo, false);
+        // Set SCL high and check for ack
+        scl_o(dcxo, true);
+        half_period(timer);
+        // returns true if acked (I2C target pulled SDA low)
+        !sda_i(dcxo)
+    }
+
+    pub fn read(dcxo: DCXO, ack: bool, timer: &mut GlobalTimer) -> u8 {
+        // Set SCL low first, otherwise setting SDA as input may cause a transition
+        // on SDA with SCL high which will be interpreted as START/STOP condition.
+        scl_o(dcxo, false);
+        half_period(timer); // make sure SCL has settled low
+        sda_oe(dcxo, false);
+
+        let mut data: u8 = 0;
+
+        // MSB first
+        for bit in (0..8).rev() {
+            scl_o(dcxo, false);
+            half_period(timer);
+            // Set SCL high and shift data
+            scl_o(dcxo, true);
+            half_period(timer);
+            if sda_i(dcxo) {
+                data |= 1 << bit
+            }
+        }
+        // Send ack
+        // Set SCL low and pull SDA low when acking
+        scl_o(dcxo, false);
+        if ack {
+            sda_oe(dcxo, true)
+        }
+        half_period(timer);
+        // then set SCL high
+        scl_o(dcxo, true);
+        half_period(timer);
+
+        data
+    }
+}
+
+fn write(dcxo: i2c::DCXO, reg: u8, val: u8, timer: &mut GlobalTimer) -> Result<(), &'static str> {
+    i2c::start(dcxo, timer);
+    if !i2c::write(dcxo, ADDRESS << 1, timer) {
+        return Err("Si549 failed to ack write address");
+    }
+    if !i2c::write(dcxo, reg, timer) {
+        return Err("Si549 failed to ack register");
+    }
+    if !i2c::write(dcxo, val, timer) {
+        return Err("Si549 failed to ack value");
+    }
+    i2c::stop(dcxo, timer);
+    Ok(())
+}
+
+fn read(dcxo: i2c::DCXO, reg: u8, timer: &mut GlobalTimer) -> Result<u8, &'static str> {
+    i2c::start(dcxo, timer);
+    if !i2c::write(dcxo, ADDRESS << 1, timer) {
+        return Err("Si549 failed to ack write address");
+    }
+    if !i2c::write(dcxo, reg, timer) {
+        return Err("Si549 failed to ack register");
+    }
+    i2c::stop(dcxo, timer);
+
+    i2c::start(dcxo, timer);
+    if !i2c::write(dcxo, (ADDRESS << 1) | 1, timer) {
+        return Err("Si549 failed to ack read address");
+    }
+    let val = i2c::read(dcxo, false, timer);
+    i2c::stop(dcxo, timer);
+    Ok(val)
+}
+
+fn setup(dcxo: i2c::DCXO, config: &DividerConfig, timer: &mut GlobalTimer) -> Result<(), &'static str> {
+    i2c::init(dcxo, timer)?;
+
+    write(dcxo, 255, 0x00, timer)?; // PAGE
+    write(dcxo, 69, 0x00, timer)?; // Disable FCAL override.
+    write(dcxo, 17, 0x00, timer)?; // Synchronously disable output
+
+    // The Si549 has no ID register, so we check that it responds correctly
+    // by writing values to a RAM-like register and reading them back.
+    for test_value in 0..255 {
+        write(dcxo, 23, test_value, timer)?;
+        let readback = read(dcxo, 23, timer)?;
+        if readback != test_value {
+            return Err("Si549 detection failed");
+        }
+    }
+
+    write(dcxo, 23, config.hsdiv as u8, timer)?;
+    write(dcxo, 24, (config.hsdiv >> 8) as u8 | (config.lsdiv << 4), timer)?;
+    write(dcxo, 26, config.fbdiv as u8, timer)?;
+    write(dcxo, 27, (config.fbdiv >> 8) as u8, timer)?;
+    write(dcxo, 28, (config.fbdiv >> 16) as u8, timer)?;
+    write(dcxo, 29, (config.fbdiv >> 24) as u8, timer)?;
+    write(dcxo, 30, (config.fbdiv >> 32) as u8, timer)?;
+    write(dcxo, 31, (config.fbdiv >> 40) as u8, timer)?;
+
+    write(dcxo, 7, 0x08, timer)?; // Start FCAL
+    timer.delay_us(30_000); // Internal FCAL VCO calibration
+    write(dcxo, 17, 0x01, timer)?; // Synchronously enable output
+
+    Ok(())
+}
+
+pub fn main_setup(timer: &mut GlobalTimer, settings: &FrequencySetting) -> Result<(), &'static str> {
+    unsafe {
+        csr::wrpll::main_dcxo_bitbang_enable_write(1);
+        csr::wrpll::main_dcxo_i2c_address_write(ADDRESS);
+    }
+
+    setup(i2c::DCXO::Main, &settings.main, timer)?;
+
+    // Si549 maximum settling time for large frequency change.
+    timer.delay_us(40_000);
+
+    unsafe {
+        csr::wrpll::main_dcxo_bitbang_enable_write(0);
+    }
+
+    info!("Main Si549 started");
+    Ok(())
+}
+
+pub fn helper_setup(timer: &mut GlobalTimer, settings: &FrequencySetting) -> Result<(), &'static str> {
+    unsafe {
+        csr::wrpll::helper_reset_write(1);
+        csr::wrpll::helper_dcxo_bitbang_enable_write(1);
+        csr::wrpll::helper_dcxo_i2c_address_write(ADDRESS);
+    }
+
+    setup(i2c::DCXO::Helper, &settings.helper, timer)?;
+
+    // Si549 maximum settling time for large frequency change.
+    timer.delay_us(40_000);
+
+    unsafe {
+        csr::wrpll::helper_reset_write(0);
+        csr::wrpll::helper_dcxo_bitbang_enable_write(0);
+    }
+    info!("Helper Si549 started");
+    Ok(())
+}
+
+fn set_adpll(dcxo: i2c::DCXO, adpll: i32) -> Result<(), &'static str> {
+    if adpll.abs() > ADPLL_MAX {
+        return Err("adpll is too large");
+    }
+
+    match dcxo {
+        i2c::DCXO::Main => unsafe {
+            if csr::wrpll::main_dcxo_bitbang_enable_read() == 1 {
+                return Err("Main si549 bitbang mode is active when using gateware i2c");
+            }
+
+            while csr::wrpll::main_dcxo_adpll_busy_read() == 1 {}
+            if csr::wrpll::main_dcxo_nack_read() == 1 {
+                return Err("Main si549 failed to ack adpll write");
+            }
+
+            csr::wrpll::main_dcxo_i2c_address_write(ADDRESS);
+            csr::wrpll::main_dcxo_adpll_write(adpll as u32);
+
+            csr::wrpll::main_dcxo_adpll_stb_write(1);
+        },
+        i2c::DCXO::Helper => unsafe {
+            if csr::wrpll::helper_dcxo_bitbang_enable_read() == 1 {
+                return Err("Helper si549 bitbang mode is active when using gateware i2c");
+            }
+
+            while csr::wrpll::helper_dcxo_adpll_busy_read() == 1 {}
+            if csr::wrpll::helper_dcxo_nack_read() == 1 {
+                return Err("Helper si549 failed to ack adpll write");
+            }
+
+            csr::wrpll::helper_dcxo_i2c_address_write(ADDRESS);
+            csr::wrpll::helper_dcxo_adpll_write(adpll as u32);
+
+            csr::wrpll::helper_dcxo_adpll_stb_write(1);
+        },
+    };
+
+    Ok(())
+}
+
+#[cfg(has_wrpll)]
+pub mod wrpll {
+
+    use super::*;
+
+    const BEATING_PERIOD: i32 = 0x8000;
+    const BEATING_HALFPERIOD: i32 = 0x4000;
+    const COUNTER_WIDTH: u32 = 24;
+    const DIV_WIDTH: u32 = 2;
+
+    // y[n] = b0*x[n] + b1*x[n-1] + b2*x[n-2] - a1*y[n-1] - a2*y[n-2]
+    struct FilterParameters {
+        pub b0: f64,
+        pub b1: f64,
+        pub b2: f64,
+        pub a1: f64,
+        pub a2: f64,
+    }
+
+    #[cfg(rtio_frequency = "100.0")]
+    const LPF: FilterParameters = FilterParameters {
+        b0: 0.03967479060647884,
+        b1: 0.07934958121295768,
+        b2: 0.03967479060647884,
+        a1: -1.3865593741228928,
+        a2: 0.5452585365488082,
+    };
+
+    #[cfg(rtio_frequency = "125.0")]
+    const LPF: FilterParameters = FilterParameters {
+        b0: 0.07209205036273991,
+        b1: 0.14418410072547982,
+        b2: 0.07209205036273991,
+        a1: -0.6114078511562919,
+        a2: -0.10022394739274834,
+    };
+
+    static mut H_ADPLL1: i32 = 0;
+    static mut H_ADPLL2: i32 = 0;
+    static mut PERIOD_ERR1: i32 = 0;
+    static mut PERIOD_ERR2: i32 = 0;
+
+    static mut M_ADPLL1: i32 = 0;
+    static mut M_ADPLL2: i32 = 0;
+    static mut PHASE_ERR1: i32 = 0;
+    static mut PHASE_ERR2: i32 = 0;
+
+    static mut BASE_ADPLL: i32 = 0;
+
+    #[derive(Clone, Copy)]
+    pub enum ISR {
+        RefTag,
+        MainTag,
+    }
+
+    mod tag_collector {
+        use super::*;
+
+        #[cfg(wrpll_ref_clk = "GT_CDR")]
+        static mut TAG_OFFSET: u32 = 8382;
+        #[cfg(wrpll_ref_clk = "SMA_CLKIN")]
+        static mut TAG_OFFSET: u32 = 0;
+        static mut REF_TAG: u32 = 0;
+        static mut REF_TAG_READY: bool = false;
+        static mut MAIN_TAG: u32 = 0;
+        static mut MAIN_TAG_READY: bool = false;
+
+        pub fn reset() {
+            clear_phase_diff_ready();
+            unsafe {
+                REF_TAG = 0;
+                MAIN_TAG = 0;
+            }
+        }
+
+        pub fn clear_phase_diff_ready() {
+            unsafe {
+                REF_TAG_READY = false;
+                MAIN_TAG_READY = false;
+            }
+        }
+
+        pub fn collect_tags(interrupt: ISR) {
+            match interrupt {
+                ISR::RefTag => unsafe {
+                    REF_TAG = csr::wrpll::ref_tag_read();
+                    REF_TAG_READY = true;
+                },
+                ISR::MainTag => unsafe {
+                    MAIN_TAG = csr::wrpll::main_tag_read();
+                    MAIN_TAG_READY = true;
+                },
+            }
+        }
+
+        pub fn phase_diff_ready() -> bool {
+            unsafe { REF_TAG_READY && MAIN_TAG_READY }
+        }
+
+        #[cfg(feature = "calibrate_wrpll_skew")]
+        pub fn set_tag_offset(offset: u32) {
+            unsafe {
+                TAG_OFFSET = offset;
+            }
+        }
+
+        #[cfg(feature = "calibrate_wrpll_skew")]
+        pub fn get_tag_offset() -> u32 {
+            unsafe { TAG_OFFSET }
+        }
+
+        pub fn get_period_error() -> i32 {
+            // n * BEATING_PERIOD - REF_TAG(n) mod BEATING_PERIOD
+            let mut period_error = unsafe { REF_TAG.overflowing_neg().0.rem_euclid(BEATING_PERIOD as u32) as i32 };
+            // mapping tags from [0, 2π] -> [-π, π]
+            if period_error > BEATING_HALFPERIOD {
+                period_error -= BEATING_PERIOD
+            }
+            period_error
+        }
+
+        pub fn get_phase_error() -> i32 {
+            // MAIN_TAG(n) - REF_TAG(n) - TAG_OFFSET mod BEATING_PERIOD
+            let mut phase_error = unsafe {
+                MAIN_TAG
+                    .overflowing_sub(REF_TAG + TAG_OFFSET)
+                    .0
+                    .rem_euclid(BEATING_PERIOD as u32) as i32
+            };
+
+            // mapping tags from [0, 2π] -> [-π, π]
+            if phase_error > BEATING_HALFPERIOD {
+                phase_error -= BEATING_PERIOD
+            }
+            phase_error
+        }
+    }
+
+    fn set_isr(en: bool) {
+        let val = if en { 1 } else { 0 };
+        unsafe {
+            csr::wrpll::ref_tag_ev_enable_write(val);
+            csr::wrpll::main_tag_ev_enable_write(val);
+        }
+    }
+
+    fn set_base_adpll() -> Result<(), &'static str> {
+        let count2adpll =
+            |error: i32| ((error as f64 * 1e6) / (0.0001164 * (1 << (COUNTER_WIDTH - DIV_WIDTH)) as f64)) as i32;
+
+        let (ref_count, main_count) = get_freq_counts();
+        unsafe {
+            BASE_ADPLL = count2adpll(ref_count as i32 - main_count as i32);
+            set_adpll(i2c::DCXO::Main, BASE_ADPLL)?;
+            set_adpll(i2c::DCXO::Helper, BASE_ADPLL)?;
+        }
+        Ok(())
+    }
+
+    fn get_freq_counts() -> (u32, u32) {
+        unsafe {
+            csr::wrpll::frequency_counter_update_write(1);
+            while csr::wrpll::frequency_counter_busy_read() == 1 {}
+            #[cfg(wrpll_ref_clk = "GT_CDR")]
+            let ref_count = csr::wrpll::frequency_counter_counter_rtio_rx0_read();
+            #[cfg(wrpll_ref_clk = "SMA_CLKIN")]
+            let ref_count = csr::wrpll::frequency_counter_counter_ref_read();
+            let main_count = csr::wrpll::frequency_counter_counter_sys_read();
+
+            (ref_count, main_count)
+        }
+    }
+
+    fn reset_plls(timer: &mut GlobalTimer) -> Result<(), &'static str> {
+        unsafe {
+            H_ADPLL1 = 0;
+            H_ADPLL2 = 0;
+            PERIOD_ERR1 = 0;
+            PERIOD_ERR2 = 0;
+            M_ADPLL1 = 0;
+            M_ADPLL2 = 0;
+            PHASE_ERR1 = 0;
+            PHASE_ERR2 = 0;
+        }
+        set_adpll(i2c::DCXO::Main, 0)?;
+        set_adpll(i2c::DCXO::Helper, 0)?;
+        // wait for adpll to transfer and DCXO to settle
+        timer.delay_us(200);
+        Ok(())
+    }
+
+    fn clear_pending(interrupt: ISR) {
+        match interrupt {
+            ISR::RefTag => unsafe { csr::wrpll::ref_tag_ev_pending_write(1) },
+            ISR::MainTag => unsafe { csr::wrpll::main_tag_ev_pending_write(1) },
+        };
+    }
+
+    fn is_pending(interrupt: ISR) -> bool {
+        match interrupt {
+            ISR::RefTag => unsafe { csr::wrpll::ref_tag_ev_pending_read() == 1 },
+            ISR::MainTag => unsafe { csr::wrpll::main_tag_ev_pending_read() == 1 },
+        }
+    }
+
+    pub fn interrupt_handler() {
+        if is_pending(ISR::RefTag) {
+            tag_collector::collect_tags(ISR::RefTag);
+            clear_pending(ISR::RefTag);
+            helper_pll().expect("failed to run helper DCXO PLL");
+        }
+
+        if is_pending(ISR::MainTag) {
+            tag_collector::collect_tags(ISR::MainTag);
+            clear_pending(ISR::MainTag);
+        }
+
+        if tag_collector::phase_diff_ready() {
+            main_pll().expect("failed to run main DCXO PLL");
+            tag_collector::clear_phase_diff_ready();
+        }
+    }
+
+    fn helper_pll() -> Result<(), &'static str> {
+        let period_err = tag_collector::get_period_error();
+        unsafe {
+            let adpll = ((LPF.b0 * period_err as f64) + (LPF.b1 * PERIOD_ERR1 as f64) + (LPF.b2 * PERIOD_ERR2 as f64)
+                - (LPF.a1 * H_ADPLL1 as f64)
+                - (LPF.a2 * H_ADPLL2 as f64)) as i32;
+            set_adpll(i2c::DCXO::Helper, BASE_ADPLL + adpll)?;
+            H_ADPLL2 = H_ADPLL1;
+            PERIOD_ERR2 = PERIOD_ERR1;
+            H_ADPLL1 = adpll;
+            PERIOD_ERR1 = period_err;
+        };
+        Ok(())
+    }
+
+    fn main_pll() -> Result<(), &'static str> {
+        let phase_err = tag_collector::get_phase_error();
+        unsafe {
+            let adpll = ((LPF.b0 * phase_err as f64) + (LPF.b1 * PHASE_ERR1 as f64) + (LPF.b2 * PHASE_ERR2 as f64)
+                - (LPF.a1 * M_ADPLL1 as f64)
+                - (LPF.a2 * M_ADPLL2 as f64)) as i32;
+            set_adpll(i2c::DCXO::Main, BASE_ADPLL + adpll)?;
+            M_ADPLL2 = M_ADPLL1;
+            PHASE_ERR2 = PHASE_ERR1;
+            M_ADPLL1 = adpll;
+            PHASE_ERR1 = phase_err;
+        };
+        Ok(())
+    }
+
+    #[cfg(wrpll_ref_clk = "GT_CDR")]
+    fn test_skew(timer: &mut GlobalTimer) -> Result<(), &'static str> {
+        // wait for PLL to stabilize
+        timer.delay_us(20_000);
+
+        info!("testing the skew of SYS CLK...");
+        if has_timing_error(timer) {
+            return Err("the skew cannot satisfy setup/hold time constraint of RX synchronizer");
+        }
+        info!("the skew of SYS CLK met the timing constraint");
+        Ok(())
+    }
+
+    #[cfg(wrpll_ref_clk = "GT_CDR")]
+    fn has_timing_error(timer: &mut GlobalTimer) -> bool {
+        unsafe {
+            csr::wrpll_skewtester::error_write(1);
+        }
+        timer.delay_us(5_000);
+        unsafe { csr::wrpll_skewtester::error_read() == 1 }
+    }
+
+    #[cfg(feature = "calibrate_wrpll_skew")]
+    fn find_edge(target: bool, timer: &mut GlobalTimer) -> Result<u32, &'static str> {
+        const STEP: u32 = 8;
+        const STABLE_THRESHOLD: u32 = 10;
+
+        enum FSM {
+            Init,
+            WaitEdge,
+            GotEdge,
+        }
+
+        let mut state: FSM = FSM::Init;
+        let mut offset: u32 = tag_collector::get_tag_offset();
+        let mut median_edge: u32 = 0;
+        let mut stable_counter: u32 = 0;
+
+        for _ in 0..(BEATING_PERIOD as u32 / STEP) as usize {
+            tag_collector::set_tag_offset(offset);
+            offset += STEP;
+            // wait for PLL to stabilize
+            timer.delay_us(20_000);
+
+            let error = has_timing_error(timer);
+            // A median edge deglitcher
+            match state {
+                FSM::Init => {
+                    if error != target {
+                        stable_counter += 1;
+                    } else {
+                        stable_counter = 0;
+                    }
+
+                    if stable_counter >= STABLE_THRESHOLD {
+                        state = FSM::WaitEdge;
+                        stable_counter = 0;
+                    }
+                }
+                FSM::WaitEdge => {
+                    if error == target {
+                        state = FSM::GotEdge;
+                        median_edge = offset;
+                    }
+                }
+                FSM::GotEdge => {
+                    if error != target {
+                        median_edge += STEP;
+                        stable_counter = 0;
+                    } else {
+                        stable_counter += 1;
+                    }
+
+                    if stable_counter >= STABLE_THRESHOLD {
+                        return Ok(median_edge);
+                    }
+                }
+            }
+        }
+        return Err("failed to find timing error edge");
+    }
+
+    #[cfg(feature = "calibrate_wrpll_skew")]
+    fn calibrate_skew(timer: &mut GlobalTimer) -> Result<(), &'static str> {
+        info!("calibrating skew to meet timing constraint...");
+
+        // clear calibrated value
+        tag_collector::set_tag_offset(0);
+        let rising = find_edge(true, timer)? as i32;
+        let falling = find_edge(false, timer)? as i32;
+
+        let width = BEATING_PERIOD - (falling - rising);
+        let result = falling + width / 2;
+        tag_collector::set_tag_offset(result as u32);
+
+        info!(
+            "calibration successful, error zone: {} -> {}, width: {} ({}deg), middle of working region: {}",
+            rising,
+            falling,
+            width,
+            360 * width / BEATING_PERIOD,
+            result,
+        );
+
+        Ok(())
+    }
+
+    pub fn select_recovered_clock(rc: bool, timer: &mut GlobalTimer) {
+        set_isr(false);
+
+        if rc {
+            tag_collector::reset();
+            reset_plls(timer).expect("failed to reset main and helper PLL");
+
+            // get within capture range
+            set_base_adpll().expect("failed to set base adpll");
+
+            // clear gateware pending flag
+            clear_pending(ISR::RefTag);
+            clear_pending(ISR::MainTag);
+
+            // use nFIQ to avoid IRQ being disabled by mutex lock and mess up PLL
+            set_isr(true);
+            info!("WRPLL interrupt enabled");
+
+            #[cfg(feature = "calibrate_wrpll_skew")]
+            calibrate_skew(timer).expect("failed to set the correct skew");
+
+            #[cfg(wrpll_ref_clk = "GT_CDR")]
+            test_skew(timer).expect("skew test failed");
+        }
+    }
+}
+
+#[cfg(has_wrpll_refclk)]
+pub mod wrpll_refclk {
+    use super::*;
+
+    pub struct MmcmSetting {
+        pub clkout0_reg1: u16,  //0x08
+        pub clkout0_reg2: u16,  //0x09
+        pub clkfbout_reg1: u16, //0x14
+        pub clkfbout_reg2: u16, //0x15
+        pub div_reg: u16,       //0x16
+        pub lock_reg1: u16,     //0x18
+        pub lock_reg2: u16,     //0x19
+        pub lock_reg3: u16,     //0x1A
+        pub power_reg: u16,     //0x28
+        pub filt_reg1: u16,     //0x4E
+        pub filt_reg2: u16,     //0x4F
+    }
+
+    fn one_clock_cycle() {
+        unsafe {
+            csr::wrpll_refclk::mmcm_dclk_write(1);
+            csr::wrpll_refclk::mmcm_dclk_write(0);
+        }
+    }
+
+    fn set_addr(address: u8) {
+        unsafe {
+            csr::wrpll_refclk::mmcm_daddr_write(address);
+        }
+    }
+
+    fn set_data(value: u16) {
+        unsafe {
+            csr::wrpll_refclk::mmcm_din_write(value);
+        }
+    }
+
+    fn set_enable(en: bool) {
+        unsafe {
+            let val = if en { 1 } else { 0 };
+            csr::wrpll_refclk::mmcm_den_write(val);
+        }
+    }
+
+    fn set_write_enable(en: bool) {
+        unsafe {
+            let val = if en { 1 } else { 0 };
+            csr::wrpll_refclk::mmcm_dwen_write(val);
+        }
+    }
+
+    fn get_data() -> u16 {
+        unsafe { csr::wrpll_refclk::mmcm_dout_read() }
+    }
+
+    fn drp_ready() -> bool {
+        unsafe { csr::wrpll_refclk::mmcm_dready_read() == 1 }
+    }
+
+    #[allow(dead_code)]
+    fn read(address: u8) -> u16 {
+        set_addr(address);
+        set_enable(true);
+        // Set DADDR on the mmcm and assert DEN for one clock cycle
+        one_clock_cycle();
+
+        set_enable(false);
+        while !drp_ready() {
+            // keep the clock signal until data is ready
+            one_clock_cycle();
+        }
+        get_data()
+    }
+
+    fn write(address: u8, value: u16) {
+        set_addr(address);
+        set_data(value);
+        set_write_enable(true);
+        set_enable(true);
+        // Set DADDR, DI on the mmcm and assert DWE, DEN for one clock cycle
+        one_clock_cycle();
+
+        set_write_enable(false);
+        set_enable(false);
+        while !drp_ready() {
+            // keep the clock signal until write is finished
+            one_clock_cycle();
+        }
+    }
+
+    fn reset(rst: bool) {
+        unsafe {
+            let val = if rst { 1 } else { 0 };
+            csr::wrpll_refclk::mmcm_reset_write(val)
+        }
+    }
+
+    pub fn setup(timer: &mut GlobalTimer, settings: MmcmSetting, mmcm_bypass: bool) -> Result<(), &'static str> {
+        unsafe {
+            csr::wrpll_refclk::refclk_reset_write(1);
+        }
+
+        if mmcm_bypass {
+            info!("Bypassing mmcm");
+            unsafe {
+                csr::wrpll_refclk::mmcm_bypass_write(1);
+            }
+        } else {
+            // Based on "DRP State Machine" from XAPP888
+            // hold reset HIGH during mmcm config
+            reset(true);
+            write(0x08, settings.clkout0_reg1);
+            write(0x09, settings.clkout0_reg2);
+            write(0x14, settings.clkfbout_reg1);
+            write(0x15, settings.clkfbout_reg2);
+            write(0x16, settings.div_reg);
+            write(0x18, settings.lock_reg1);
+            write(0x19, settings.lock_reg2);
+            write(0x1A, settings.lock_reg3);
+            write(0x28, settings.power_reg);
+            write(0x4E, settings.filt_reg1);
+            write(0x4F, settings.filt_reg2);
+            reset(false);
+
+            // wait for the mmcm to lock
+            timer.delay_us(100);
+
+            let locked = unsafe { csr::wrpll_refclk::mmcm_locked_read() == 1 };
+            if !locked {
+                return Err("mmcm failed to generate 125MHz ref clock from SMA CLKIN");
+            }
+        }
+
+        unsafe {
+            csr::wrpll_refclk::refclk_reset_write(0);
+        }
+
+        Ok(())
+    }
+}

src/libbuild_zynq/lib.rs

@@ -1,8 +1,7 @@
-use std::env;
+use std::{env,
-use std::fs::File;
+          fs::File,
-use std::io::{BufRead, BufReader, Write};
+          io::{BufRead, BufReader, Write},
-use std::path::PathBuf;
+          path::PathBuf};
-
 
 pub fn add_linker_script() {
     // Put the linker script somewhere the linker can find it

src/libbuild_zynq/link.x

@@ -10,7 +10,9 @@ SECTIONS
     __text_start = .;
     .text :
     {
+        __exceptions_start = .;
         KEEP(*(.text.exceptions));
+        __exceptions_end = .;
         *(.text.boot);
         *(.text .text.*);
     } > SDRAM

src/libc/Cargo.toml.tpl

@@ -6,7 +6,7 @@ edition = "2018"
 build = "build.rs"
 
 [dependencies]
-libboard_zynq = { git = "https://git.m-labs.hk/M-Labs/zynq-rs.git" }
+libboard_zynq = { path = "@@ZYNQ_RS@@/libboard_zynq" }
 
 [build-dependencies]
 cc = { version = "1.0.1" }

src/libc/build.rs

@@ -4,8 +4,7 @@ fn main() {
 }
 
 mod libc {
-    use std::path::Path;
+    use std::{env, path::Path};
-    use std::env;
 
     pub fn compile() {
         let cfg = &mut cc::Build::new();
@@ -32,9 +31,7 @@ mod libc {
         cfg.flag("-U_FORTIFY_SOURCE");
         cfg.define("_FORTIFY_SOURCE", Some("0"));
 
-        let sources = vec![
+        let sources = vec!["printf.c"];
-            "printf.c"
-        ];
 
         let root = Path::new("./");
         for src in sources {

src/libdwarf/src/eh.rs

@@ -11,10 +11,12 @@
 #![allow(non_upper_case_globals)]
 #![allow(unused)]
 
-use crate::DwarfReader;
 use core::mem;
+
 use cslice::CSlice;
 
+use crate::DwarfReader;
+
 pub const DW_EH_PE_omit: u8 = 0xFF;
 pub const DW_EH_PE_absptr: u8 = 0x00;
 
@@ -160,15 +162,11 @@ pub unsafe fn find_eh_action(
                         if ar_filter == 0 {
                             saw_cleanup = true;
                         } else if ar_filter > 0 {
-                            let catch_type = get_ttype_entry(
+                            let catch_type =
-                                ar_filter as usize,
+                                get_ttype_entry(ar_filter as usize, ttype_encoding, ttype_base, ttype_table)?;
-                                ttype_encoding,
-                                ttype_base,
-                                ttype_table,
-                            )?;
                             match catch_type {
                                 Some(clause_ptr) if *(clause_ptr as *const u32) == id => {
-                                    return Ok(EHAction::Catch(lpad))
+                                    return Ok(EHAction::Catch(lpad));
                                 }
                                 None => return Ok(EHAction::Catch(lpad)),
                                 _ => {}
@@ -251,19 +249,11 @@ fn get_base(encoding: u8, context: &EHContext<'_>) -> Result<usize, ()> {
     }
 }
 
-unsafe fn read_encoded_pointer(
+unsafe fn read_encoded_pointer(reader: &mut DwarfReader, context: &EHContext<'_>, encoding: u8) -> Result<usize, ()> {
-    reader: &mut DwarfReader,
-    context: &EHContext<'_>,
-    encoding: u8,
-) -> Result<usize, ()> {
     read_encoded_pointer_with_base(reader, encoding, get_base(encoding, context)?)
 }
 
-unsafe fn read_encoded_pointer_with_base(
+unsafe fn read_encoded_pointer_with_base(reader: &mut DwarfReader, encoding: u8, base: usize) -> Result<usize, ()> {
-    reader: &mut DwarfReader,
-    encoding: u8,
-    base: usize,
-) -> Result<usize, ()> {
     if encoding == DW_EH_PE_omit {
         return Err(());
     }

src/libdyld/Cargo.toml.tpl

@@ -8,4 +8,4 @@ name = "dyld"
 
 [dependencies]
 log = "0.4"
-libcortex_a9 = { git = "https://git.m-labs.hk/M-Labs/zynq-rs.git" }
+libcortex_a9 = { path = "@@ZYNQ_RS@@/libcortex_a9" }

src/libdyld/src/elf.rs

@@ -1451,8 +1451,7 @@ pub const R_AARCH64_TLSDESC_CALL: usize = 569;
 pub const R_AARCH64_TLSLE_LDST128_TPREL_LO12: usize = 570;
 pub const R_AARCH64_TLSLE_LDST128_TPREL_LO12_NC: usize = 571;
 pub const R_AARCH64_TLSLD_LDST128_DTPREL_LO12: usize = 572;
-pub const R_AARCH64_TLSLD_LDST128_DTPREL_LO12_NC: usize =
+pub const R_AARCH64_TLSLD_LDST128_DTPREL_LO12_NC: usize = 573;
-    573;
 pub const R_AARCH64_COPY: usize = 1024;
 pub const R_AARCH64_GLOB_DAT: usize = 1025;
 pub const R_AARCH64_JUMP_SLOT: usize = 1026;
@@ -2267,7 +2266,9 @@ pub struct Elf32_Ehdr {
     pub e_shstrndx: Elf32_Half,
 }
 impl Clone for Elf32_Ehdr {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2288,7 +2289,9 @@ pub struct Elf64_Ehdr {
     pub e_shstrndx: Elf64_Half,
 }
 impl Clone for Elf64_Ehdr {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2305,7 +2308,9 @@ pub struct Elf32_Shdr {
     pub sh_entsize: Elf32_Word,
 }
 impl Clone for Elf32_Shdr {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2322,7 +2327,9 @@ pub struct Elf64_Shdr {
     pub sh_entsize: Elf64_Xword,
 }
 impl Clone for Elf64_Shdr {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2335,7 +2342,9 @@ pub struct Elf32_Sym {
     pub st_shndx: Elf32_Section,
 }
 impl Clone for Elf32_Sym {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2348,7 +2357,9 @@ pub struct Elf64_Sym {
     pub st_size: Elf64_Xword,
 }
 impl Clone for Elf64_Sym {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2357,7 +2368,9 @@ pub struct Elf32_Syminfo {
     pub si_flags: Elf32_Half,
 }
 impl Clone for Elf32_Syminfo {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2366,7 +2379,9 @@ pub struct Elf64_Syminfo {
     pub si_flags: Elf64_Half,
 }
 impl Clone for Elf64_Syminfo {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2375,7 +2390,9 @@ pub struct Elf32_Rel {
     pub r_info: Elf32_Word,
 }
 impl Clone for Elf32_Rel {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2384,7 +2401,9 @@ pub struct Elf64_Rel {
     pub r_info: Elf64_Xword,
 }
 impl Clone for Elf64_Rel {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2394,7 +2413,9 @@ pub struct Elf32_Rela {
     pub r_addend: Elf32_Sword,
 }
 impl Clone for Elf32_Rela {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2404,7 +2425,9 @@ pub struct Elf64_Rela {
     pub r_addend: Elf64_Sxword,
 }
 impl Clone for Elf64_Rela {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2419,7 +2442,9 @@ pub struct Elf32_Phdr {
     pub p_align: Elf32_Word,
 }
 impl Clone for Elf32_Phdr {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2434,7 +2459,9 @@ pub struct Elf64_Phdr {
     pub p_align: Elf64_Xword,
 }
 impl Clone for Elf64_Phdr {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Copy)]
@@ -2449,10 +2476,14 @@ pub union Elf32_Dyn__bindgen_ty_1 {
     pub d_ptr: Elf32_Addr,
 }
 impl Clone for Elf32_Dyn__bindgen_ty_1 {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 impl Clone for Elf32_Dyn {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Copy)]
@@ -2467,10 +2498,14 @@ pub union Elf64_Dyn__bindgen_ty_1 {
     pub d_ptr: Elf64_Addr,
 }
 impl Clone for Elf64_Dyn__bindgen_ty_1 {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 impl Clone for Elf64_Dyn {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2484,7 +2519,9 @@ pub struct Elf32_Verdef {
     pub vd_next: Elf32_Word,
 }
 impl Clone for Elf32_Verdef {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2498,7 +2535,9 @@ pub struct Elf64_Verdef {
     pub vd_next: Elf64_Word,
 }
 impl Clone for Elf64_Verdef {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2507,7 +2546,9 @@ pub struct Elf32_Verdaux {
     pub vda_next: Elf32_Word,
 }
 impl Clone for Elf32_Verdaux {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2516,7 +2557,9 @@ pub struct Elf64_Verdaux {
     pub vda_next: Elf64_Word,
 }
 impl Clone for Elf64_Verdaux {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2528,7 +2571,9 @@ pub struct Elf32_Verneed {
     pub vn_next: Elf32_Word,
 }
 impl Clone for Elf32_Verneed {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2540,7 +2585,9 @@ pub struct Elf64_Verneed {
     pub vn_next: Elf64_Word,
 }
 impl Clone for Elf64_Verneed {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2552,7 +2599,9 @@ pub struct Elf32_Vernaux {
     pub vna_next: Elf32_Word,
 }
 impl Clone for Elf32_Vernaux {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2564,7 +2613,9 @@ pub struct Elf64_Vernaux {
     pub vna_next: Elf64_Word,
 }
 impl Clone for Elf64_Vernaux {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Copy)]
@@ -2578,10 +2629,14 @@ pub union Elf32_auxv_t__bindgen_ty_1 {
     pub a_val: u32,
 }
 impl Clone for Elf32_auxv_t__bindgen_ty_1 {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 impl Clone for Elf32_auxv_t {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Copy)]
@@ -2595,10 +2650,14 @@ pub union Elf64_auxv_t__bindgen_ty_1 {
     pub a_val: u64,
 }
 impl Clone for Elf64_auxv_t__bindgen_ty_1 {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 impl Clone for Elf64_auxv_t {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2608,7 +2667,9 @@ pub struct Elf32_Nhdr {
     pub n_type: Elf32_Word,
 }
 impl Clone for Elf32_Nhdr {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2618,7 +2679,9 @@ pub struct Elf64_Nhdr {
     pub n_type: Elf64_Word,
 }
 impl Clone for Elf64_Nhdr {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2630,7 +2693,9 @@ pub struct Elf32_Move {
     pub m_stride: Elf32_Half,
 }
 impl Clone for Elf32_Move {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2642,7 +2707,9 @@ pub struct Elf64_Move {
     pub m_stride: Elf64_Half,
 }
 impl Clone for Elf64_Move {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Copy)]
@@ -2657,7 +2724,9 @@ pub struct Elf32_gptab__bindgen_ty_1 {
     pub gt_unused: Elf32_Word,
 }
 impl Clone for Elf32_gptab__bindgen_ty_1 {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2666,10 +2735,14 @@ pub struct Elf32_gptab__bindgen_ty_2 {
     pub gt_bytes: Elf32_Word,
 }
 impl Clone for Elf32_gptab__bindgen_ty_2 {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 impl Clone for Elf32_gptab {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2679,7 +2752,9 @@ pub struct Elf32_RegInfo {
     pub ri_gp_value: Elf32_Sword,
 }
 impl Clone for Elf32_RegInfo {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2690,7 +2765,9 @@ pub struct Elf_Options {
     pub info: Elf32_Word,
 }
 impl Clone for Elf_Options {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2699,7 +2776,9 @@ pub struct Elf_Options_Hw {
     pub hwp_flags2: Elf32_Word,
 }
 impl Clone for Elf_Options_Hw {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2711,7 +2790,9 @@ pub struct Elf32_Lib {
     pub l_flags: Elf32_Word,
 }
 impl Clone for Elf32_Lib {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 #[repr(C)]
 #[derive(Debug, Copy)]
@@ -2723,17 +2804,31 @@ pub struct Elf64_Lib {
     pub l_flags: Elf64_Word,
 }
 impl Clone for Elf64_Lib {
-    fn clone(&self) -> Self { *self }
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 pub type Elf32_Conflict = Elf32_Addr;
 #[repr(C)]
 #[derive(Clone, Copy)]
 pub struct EXIDX_Entry(u32, u32);
 
-pub fn ELF32_R_SYM(info: Elf32_Word) -> Elf32_Word { info >> 8 }
+pub fn ELF32_R_SYM(info: Elf32_Word) -> Elf32_Word {
-pub fn ELF32_R_TYPE(info: Elf32_Word) -> u8 { info as u8 }
+    info >> 8
-pub fn ELF32_R_INFO(sym: Elf32_Word, ty: u8) -> Elf32_Word { sym << 8 | ty as Elf32_Word }
+}
+pub fn ELF32_R_TYPE(info: Elf32_Word) -> u8 {
+    info as u8
+}
+pub fn ELF32_R_INFO(sym: Elf32_Word, ty: u8) -> Elf32_Word {
+    sym << 8 | ty as Elf32_Word
+}
 
-pub fn ELF32_ST_BIND(info: u8) -> u8 { info >> 4 }
+pub fn ELF32_ST_BIND(info: u8) -> u8 {
-pub fn ELF32_ST_TYPE(info: u8) -> u8 { info & 0xf }
+    info >> 4
-pub fn ELF32_ST_INFO(bind: u8, ty: u8) -> u8 { (bind << 4) | (ty & 0xf) }
+}
+pub fn ELF32_ST_TYPE(info: u8) -> u8 {
+    info & 0xf
+}
+pub fn ELF32_ST_INFO(bind: u8, ty: u8) -> u8 {
+    (bind << 4) | (ty & 0xf)
+}

src/libdyld/src/file.rs

@@ -1,8 +1,8 @@
-use core::{mem, ptr, ops::{Deref, Range}};
+use core::{mem,
-use super::{
+           ops::{Deref, Range},
-    Arch,
+           ptr};
-    elf::*,
+
-};
+use super::{elf::*, Arch};
 
 fn read_unaligned<T: Copy>(data: &[u8], offset: usize) -> Option<T> {
     if data.len() < offset + mem::size_of::<T>() {
@@ -31,14 +31,40 @@ impl<'a> File<'a> {
 
     pub fn arch(&self) -> Option<Arch> {
         const IDENT_OPENRISC: [u8; EI_NIDENT] = [
-            ELFMAG0,    ELFMAG1,     ELFMAG2,    ELFMAG3,
+            ELFMAG0,
-            ELFCLASS32, ELFDATA2MSB, EV_CURRENT, ELFOSABI_NONE,
+            ELFMAG1,
-            /* ABI version */ 0, /* padding */ 0, 0, 0, 0, 0, 0, 0
+            ELFMAG2,
+            ELFMAG3,
+            ELFCLASS32,
+            ELFDATA2MSB,
+            EV_CURRENT,
+            ELFOSABI_NONE,
+            /* ABI version */ 0,
+            /* padding */ 0,
+            0,
+            0,
+            0,
+            0,
+            0,
+            0,
         ];
         const IDENT_ARM: [u8; EI_NIDENT] = [
-            ELFMAG0,    ELFMAG1,     ELFMAG2,    ELFMAG3,
+            ELFMAG0,
-            ELFCLASS32, ELFDATA2LSB, EV_CURRENT, ELFOSABI_NONE,
+            ELFMAG1,
-            /* ABI version */ 0, /* padding */ 0, 0, 0, 0, 0, 0, 0
+            ELFMAG2,
+            ELFMAG3,
+            ELFCLASS32,
+            ELFDATA2LSB,
+            EV_CURRENT,
+            ELFOSABI_NONE,
+            /* ABI version */ 0,
+            /* padding */ 0,
+            0,
+            0,
+            0,
+            0,
+            0,
+            0,
         ];
 
         match (self.ehdr.e_ident, self.ehdr.e_machine) {
@@ -48,16 +74,14 @@ impl<'a> File<'a> {
         }
     }
 
-    pub fn program_headers<'b>(&'b self) -> impl Iterator<Item = Option<Elf32_Phdr>> + 'b
+    pub fn program_headers<'b>(&'b self) -> impl Iterator<Item = Option<Elf32_Phdr>> + 'b {
-    {
         (0..self.ehdr.e_phnum).map(move |i| {
             let phdr_off = self.ehdr.e_phoff as usize + mem::size_of::<Elf32_Phdr>() * i as usize;
             self.read_unaligned::<Elf32_Phdr>(phdr_off)
         })
     }
 
-    pub fn section_headers<'b>(&'b self) -> impl Iterator<Item = Option<Elf32_Shdr>> + 'b
+    pub fn section_headers<'b>(&'b self) -> impl Iterator<Item = Option<Elf32_Shdr>> + 'b {
-    {
         (0..self.ehdr.e_shnum).map(move |i| {
             let shdr_off = self.ehdr.e_shoff as usize + mem::size_of::<Elf32_Shdr>() * i as usize;
             self.read_unaligned::<Elf32_Shdr>(shdr_off)

src/libdyld/src/image.rs

@@ -1,13 +1,9 @@
-use core::{
-    ops::{Deref, DerefMut, Range},
-    mem,
-    slice,
-};
 use alloc::alloc::{alloc_zeroed, dealloc, Layout, LayoutError};
-use super::{
+use core::{mem,
-    elf::*,
+           ops::{Deref, DerefMut, Range},
-    Error,
+           slice};
-};
+
+use super::{elf::*, Error};
 
 pub struct DynamicSection {
     pub strtab: Range<usize>,
@@ -34,17 +30,12 @@ impl Image {
             slice::from_raw_parts_mut(ptr, size)
         };
 
-        Ok(Image {
+        Ok(Image { layout, data })
-            layout,
-            data,
-        })
     }
 
     /// assumes that self.data is properly aligned
     pub(crate) fn get_ref<T>(&self, offset: usize) -> Option<&T>
-    where
+    where T: Copy {
-        T: Copy,
-    {
         if self.data.len() < offset + mem::size_of::<T>() {
             None
         } else if (self.data.as_ptr() as usize + offset) & (mem::align_of::<T>() - 1) != 0 {
@@ -66,55 +57,53 @@ impl Image {
         unsafe { slice::from_raw_parts(ptr, len) }
     }
 
-    fn dyn_headers<'a>(&'a self, range: Range<usize>) ->
+    fn dyn_headers<'a>(&'a self, range: Range<usize>) -> impl Iterator<Item = &'a Elf32_Dyn> + 'a {
-        impl Iterator<Item = &'a Elf32_Dyn> + 'a
-    {
         range
             .step_by(mem::size_of::<Elf32_Dyn>())
-            .filter_map(move |offset| {
+            .filter_map(move |offset| self.get_ref::<Elf32_Dyn>(offset))
-                self.get_ref::<Elf32_Dyn>(offset)
-            })
             .take_while(|d| unsafe { d.d_un.d_val } as i32 != DT_NULL)
     }
 
     pub fn dyn_section(&self, range: Range<usize>) -> Result<DynamicSection, Error> {
         let (mut strtab_off, mut strtab_sz) = (0, 0);
-        let (mut rel_off,    mut rel_sz)    = (0, 0);
+        let (mut rel_off, mut rel_sz) = (0, 0);
-        let (mut rela_off,   mut rela_sz)   = (0, 0);
+        let (mut rela_off, mut rela_sz) = (0, 0);
         let (mut pltrel_off, mut pltrel_sz) = (0, 0);
-        let (mut hash_off,   mut hash_sz)   = (0, 0);
+        let (mut hash_off, mut hash_sz) = (0, 0);
         let mut symtab_off = 0;
-        let mut sym_ent    = 0;
+        let mut sym_ent = 0;
-        let mut rel_ent    = 0;
+        let mut rel_ent = 0;
-        let mut rela_ent   = 0;
+        let mut rela_ent = 0;
-        let mut nbucket    = 0;
+        let mut nbucket = 0;
-        let mut nchain     = 0;
+        let mut nchain = 0;
 
         for dyn_header in self.dyn_headers(range) {
             let val = unsafe { dyn_header.d_un.d_val } as usize;
             match dyn_header.d_tag {
-                DT_NULL     => break,
+                DT_NULL => break,
-                DT_STRTAB   => strtab_off = val,
+                DT_STRTAB => strtab_off = val,
-                DT_STRSZ    => strtab_sz  = val,
+                DT_STRSZ => strtab_sz = val,
-                DT_SYMTAB   => symtab_off = val,
+                DT_SYMTAB => symtab_off = val,
-                DT_SYMENT   => sym_ent    = val,
+                DT_SYMENT => sym_ent = val,
-                DT_REL      => rel_off    = val,
+                DT_REL => rel_off = val,
-                DT_RELSZ    => rel_sz     = val,
+                DT_RELSZ => rel_sz = val,
-                DT_RELENT   => rel_ent    = val,
+                DT_RELENT => rel_ent = val,
-                DT_RELA     => rela_off   = val,
+                DT_RELA => rela_off = val,
-                DT_RELASZ   => rela_sz    = val,
+                DT_RELASZ => rela_sz = val,
-                DT_RELAENT  => rela_ent   = val,
+                DT_RELAENT => rela_ent = val,
-                DT_JMPREL   => pltrel_off = val,
+                DT_JMPREL => pltrel_off = val,
-                DT_PLTRELSZ => pltrel_sz  = val,
+                DT_PLTRELSZ => pltrel_sz = val,
-                DT_HASH     => {
+                DT_HASH => {
-                    nbucket  = *self.get_ref::<Elf32_Word>(val + 0)
+                    nbucket = *self
+                        .get_ref::<Elf32_Word>(val + 0)
                         .ok_or("cannot read hash bucket count")? as usize;
-                    nchain   = *self.get_ref::<Elf32_Word>(val + 4)
+                    nchain = *self
+                        .get_ref::<Elf32_Word>(val + 4)
                         .ok_or("cannot read hash chain count")? as usize;
                     hash_off = val + 8;
-                    hash_sz  = (nbucket + nchain) * mem::size_of::<Elf32_Word>();
+                    hash_sz = (nbucket + nchain) * mem::size_of::<Elf32_Word>();
                 }
-                _ => ()
+                _ => (),
             }
         }
 
@@ -123,28 +112,28 @@ impl Image {
         let symtab_sz = nchain * mem::size_of::<Elf32_Sym>();
 
         if strtab_off + strtab_sz > self.data.len() {
-            return Err("invalid strtab offset/size")?
+            return Err("invalid strtab offset/size")?;
         }
         if symtab_off + symtab_sz > self.data.len() {
-            return Err("invalid symtab offset/size")?
+            return Err("invalid symtab offset/size")?;
         }
         if sym_ent != mem::size_of::<Elf32_Sym>() {
-            return Err("incorrect symbol entry size")?
+            return Err("incorrect symbol entry size")?;
         }
         if rel_off + rel_sz > self.data.len() {
-            return Err("invalid rel offset/size")?
+            return Err("invalid rel offset/size")?;
         }
         if rel_ent != 0 && rel_ent != mem::size_of::<Elf32_Rel>() {
-            return Err("incorrect relocation entry size")?
+            return Err("incorrect relocation entry size")?;
         }
         if rela_off + rela_sz > self.data.len() {
-            return Err("invalid rela offset/size")?
+            return Err("invalid rela offset/size")?;
         }
         if rela_ent != 0 && rela_ent != mem::size_of::<Elf32_Rela>() {
-            return Err("incorrect relocation entry size")?
+            return Err("incorrect relocation entry size")?;
         }
         if pltrel_off + pltrel_sz > self.data.len() {
-            return Err("invalid pltrel offset/size")?
+            return Err("invalid pltrel offset/size")?;
         }
 
         Ok(DynamicSection {
@@ -165,7 +154,7 @@ impl Image {
 
     pub fn write(&self, offset: usize, value: Elf32_Word) -> Result<(), Error> {
         if offset + mem::size_of::<Elf32_Addr>() > self.data.len() {
-            return Err("relocation out of image bounds")?
+            return Err("relocation out of image bounds")?;
         }
 
         let ptr = (self.data.as_ptr() as usize + offset) as *mut Elf32_Addr;

src/libdyld/src/lib.rs

@@ -1,13 +1,14 @@
 #![no_std]
 
 extern crate alloc;
-extern crate log;
 extern crate libcortex_a9;
+extern crate log;
 
-use core::{convert, fmt, ops::Range, str};
 use alloc::string::String;
-use log::{debug, trace};
+use core::{convert, fmt, ops::Range, str};
+
 use elf::*;
+use log::{debug, trace};
 
 pub mod elf;
 mod file;
@@ -21,11 +22,10 @@ pub enum Arch {
     OpenRisc,
 }
 
-
 #[derive(Debug)]
 pub enum Error {
     Parsing(&'static str),
-    Lookup(String)
+    Lookup(String),
 }
 
 impl convert::From<&'static str> for Error {
@@ -37,10 +37,8 @@ impl convert::From<&'static str> for Error {
 impl fmt::Display for Error {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         match self {
-            &Error::Parsing(desc) =>
+            &Error::Parsing(desc) => write!(f, "parse error: {}", desc),
-                write!(f, "parse error: {}", desc),
+            &Error::Lookup(ref sym) => write!(f, "symbol lookup error: {}", sym),
-            &Error::Lookup(ref sym) =>
-                write!(f, "symbol lookup error: {}", sym),
         }
     }
 }
@@ -103,20 +101,22 @@ impl Library {
         let mut index = self.hash_bucket()[hash as usize % self.hash_bucket().len()] as usize;
 
         loop {
-            if index == STN_UNDEF { return None }
+            if index == STN_UNDEF {
+                return None;
+            }
 
             let sym = &self.symtab()[index];
             let sym_name_off = sym.st_name as usize;
             match self.strtab().get(sym_name_off..sym_name_off + name.len()) {
                 Some(sym_name) if sym_name == name => {
                     if ELF32_ST_BIND(sym.st_info) & STB_GLOBAL == 0 {
-                        return None
+                        return None;
                     }
 
                     match sym.st_shndx {
                         SHN_UNDEF => return None,
                         SHN_ABS => return Some(self.image.ptr() as u32 + sym.st_value),
-                        _ => return Some(self.image.ptr() as u32 + sym.st_value)
+                        _ => return Some(self.image.ptr() as u32 + sym.st_value),
                     }
                 }
                 _ => (),
@@ -127,10 +127,16 @@ impl Library {
     }
 
     pub fn name_starting_at(&self, offset: usize) -> Result<&[u8], Error> {
-        let size = self.strtab().iter().skip(offset).position(|&x| x == 0)
+        let size = self
-                              .ok_or("symbol in symbol table not null-terminated")?;
+            .strtab()
-        Ok(self.strtab().get(offset..offset + size)
+            .iter()
-           .ok_or("cannot read symbol name")?)
+            .skip(offset)
+            .position(|&x| x == 0)
+            .ok_or("symbol in symbol table not null-terminated")?;
+        Ok(self
+            .strtab()
+            .get(offset..offset + size)
+            .ok_or("cannot read symbol name")?)
     }
 
     /// Rebind Rela by `name` to a new `addr`
@@ -143,53 +149,59 @@ impl Library {
     }
 }
 
-pub fn load(
+pub fn load(data: &[u8], resolve: &dyn Fn(&[u8]) -> Option<Elf32_Word>) -> Result<Library, Error> {
-    data: &[u8],
-    resolve: &dyn Fn(&[u8]) -> Option<Elf32_Word>
-) -> Result<Library, Error> {
     // validate ELF file
-    let file = file::File::new(data)
+    let file = file::File::new(data).ok_or("cannot read ELF header")?;
-        .ok_or("cannot read ELF header")?;
     if file.ehdr.e_type != ET_DYN {
-        return Err("not a shared library")?
+        return Err("not a shared library")?;
     }
-    let arch = file.arch()
+    let arch = file.arch().ok_or("not for a supported architecture")?;
-        .ok_or("not for a supported architecture")?;
 
     // prepare target memory
-    let image_size = file.program_headers()
+    let image_size = file
+        .program_headers()
         .filter_map(|phdr| phdr.map(|phdr| phdr.p_vaddr + phdr.p_memsz))
         .max()
         .unwrap_or(0) as usize;
-    let image_align = file.program_headers()
+    let image_align = file
-        .filter_map(|phdr| phdr.and_then(|phdr| {
+        .program_headers()
-            if phdr.p_type == PT_LOAD {
+        .filter_map(|phdr| {
-                Some(phdr.p_align)
+            phdr.and_then(|phdr| {
-            } else {
+                if phdr.p_type == PT_LOAD {
-                None
+                    Some(phdr.p_align)
-            }
+                } else {
-        }))
+                    None
+                }
+            })
+        })
         .max()
         .unwrap_or(4) as usize;
     // 1 image for all segments
-    let mut image = image::Image::new(image_size, image_align)
+    let mut image = image::Image::new(image_size, image_align).map_err(|_| "cannot allocate target image")?;
-        .map_err(|_| "cannot allocate target image")?;
+    debug!(
-    debug!("ELF target: {} bytes, align to {:X}, allocated at {:08X}", image_size, image_align, image.ptr() as usize);
+        "ELF target: {} bytes, align to {:X}, allocated at {:08X}",
+        image_size,
+        image_align,
+        image.ptr() as usize
+    );
 
     // LOAD
     for phdr in file.program_headers() {
         let phdr = phdr.ok_or("cannot read program header")?;
-        trace!("Program header: {:08X}+{:08X} to {:08X}",
+        trace!(
-              phdr.p_offset, phdr.p_filesz,
+            "Program header: {:08X}+{:08X} to {:08X}",
-              image.ptr() as u32
+            phdr.p_offset,
+            phdr.p_filesz,
+            image.ptr() as u32
         );
         let file_range = phdr.p_offset as usize..(phdr.p_offset + phdr.p_filesz) as usize;
         match phdr.p_type {
             PT_LOAD => {
-                let src = file.get(file_range)
+                let src = file
+                    .get(file_range)
                     .ok_or("program header requests an out of bounds load (in file)")?;
-                let dst = image.get_mut(phdr.p_vaddr as usize..
+                let dst = image
-                                        (phdr.p_vaddr + phdr.p_filesz) as usize)
+                    .get_mut(phdr.p_vaddr as usize..(phdr.p_vaddr + phdr.p_filesz) as usize)
                     .ok_or("program header requests an out of bounds load (in target)")?;
                 dst.copy_from_slice(src);
             }
@@ -203,9 +215,9 @@ pub fn load(
         let shdr = shdr.ok_or("cannot read section header")?;
         match shdr.sh_type as usize {
             SHT_ARM_EXIDX => {
-                let range = shdr.sh_addr as usize..
+                let range = shdr.sh_addr as usize..(shdr.sh_addr + shdr.sh_size) as usize;
-                    (shdr.sh_addr + shdr.sh_size) as usize;
+                let _ = image
-                let _ = image.get(range.clone())
+                    .get(range.clone())
                     .ok_or("section header specifies EXIDX outside of image (in target)")?;
                 exidx = Some(range);
             }
@@ -214,11 +226,14 @@ pub fn load(
     }
 
     // relocate DYNAMIC
-    let dyn_range = file.dyn_header_vaddr()
+    let dyn_range = file.dyn_header_vaddr().ok_or("cannot find a dynamic header")?;
-        .ok_or("cannot find a dynamic header")?;
     let dyn_section = image.dyn_section(dyn_range.clone())?;
-    debug!("Relocating {} rela, {} rel, {} pltrel",
+    debug!(
-           dyn_section.rela.len(), dyn_section.rel.len(), dyn_section.pltrel.len());
+        "Relocating {} rela, {} rel, {} pltrel",
+        dyn_section.rela.len(),
+        dyn_section.rel.len(),
+        dyn_section.pltrel.len()
+    );
     let lib = Library {
         arch,
         image,

src/libdyld/src/reloc.rs

@@ -1,16 +1,10 @@
 use alloc::string::String;
+
+use libcortex_a9::{asm::{dsb, isb},
+                   cache::{bpiall, dcci_slice, iciallu}};
 use log::trace;
-use super::{
+
-    Arch,
+use super::{elf::*, image::Image, Arch, Error, Library};
-    elf::*,
-    Error,
-    image::Image,
-    Library,
-};
-use libcortex_a9::{
-    cache::{dcci_slice, iciallu, bpiall},
-    asm::{dsb, isb},
-};
 
 pub trait Relocatable {
     fn offset(&self) -> usize;
@@ -66,25 +60,18 @@ enum RelType {
 impl RelType {
     pub fn new(arch: Arch, type_info: u8) -> Option<Self> {
         match type_info {
-            R_OR1K_NONE if arch == Arch::OpenRisc =>
+            R_OR1K_NONE if arch == Arch::OpenRisc => Some(RelType::None),
-                Some(RelType::None),
+            R_ARM_NONE if arch == Arch::Arm => Some(RelType::None),
-            R_ARM_NONE if arch == Arch::Arm =>
-                Some(RelType::None),
 
-            R_OR1K_RELATIVE if arch == Arch::OpenRisc =>
+            R_OR1K_RELATIVE if arch == Arch::OpenRisc => Some(RelType::Relative),
-                Some(RelType::Relative),
+            R_ARM_RELATIVE if arch == Arch::Arm => Some(RelType::Relative),
-            R_ARM_RELATIVE if arch == Arch::Arm =>
-                Some(RelType::Relative),
 
-            R_OR1K_32 | R_OR1K_GLOB_DAT | R_OR1K_JMP_SLOT
+            R_OR1K_32 | R_OR1K_GLOB_DAT | R_OR1K_JMP_SLOT if arch == Arch::OpenRisc => Some(RelType::LookupAbs),
-                if arch == Arch::OpenRisc => Some(RelType::LookupAbs),
+            R_ARM_GLOB_DAT | R_ARM_JUMP_SLOT | R_ARM_ABS32 if arch == Arch::Arm => Some(RelType::LookupAbs),
-            R_ARM_GLOB_DAT | R_ARM_JUMP_SLOT | R_ARM_ABS32
-                if arch == Arch::Arm => Some(RelType::LookupAbs),
 
             R_ARM_PREL31 if arch == Arch::Arm => Some(RelType::LookupRel),
 
-            _ =>
+            _ => None,
-                None
         }
     }
 }
@@ -96,22 +83,25 @@ fn format_sym_name(sym_name: &[u8]) -> String {
 }
 
 pub fn relocate<R: Relocatable>(
-    arch: Arch, lib: &Library,
+    arch: Arch,
-    rel: &R, resolve: &dyn Fn(&[u8]) -> Option<Elf32_Word>
+    lib: &Library,
+    rel: &R,
+    resolve: &dyn Fn(&[u8]) -> Option<Elf32_Word>,
 ) -> Result<(), Error> {
     let sym;
     if rel.sym_info() == 0 {
         sym = None;
     } else {
-        sym = Some(lib.symtab().get(rel.sym_info() as usize)
+        sym = Some(
-                   .ok_or("symbol out of bounds of symbol table")?)
+            lib.symtab()
+                .get(rel.sym_info() as usize)
+                .ok_or("symbol out of bounds of symbol table")?,
+        )
     }
 
-    let rel_type = RelType::new(arch, rel.type_info())
+    let rel_type = RelType::new(arch, rel.type_info()).ok_or("unsupported relocation type")?;
-        .ok_or("unsupported relocation type")?;
     let value = match rel_type {
-        RelType::None =>
+        RelType::None => return Ok(()),
-            return Ok(()),
 
         RelType::Relative => {
             let addend = rel.addend(&lib.image);
@@ -132,42 +122,48 @@ pub fn relocate<R: Relocatable>(
                 addr
             } else {
                 // We couldn't find it anywhere.
-                return Err(Error::Lookup(format_sym_name(sym_name)))
+                return Err(Error::Lookup(format_sym_name(sym_name)));
             };
 
             match rel_type {
                 RelType::LookupAbs => sym_addr,
-                RelType::LookupRel =>
+                RelType::LookupRel => {
-                    sym_addr.wrapping_sub(
+                    sym_addr.wrapping_sub(lib.image.ptr().wrapping_offset(rel.offset() as isize) as Elf32_Addr)
-                        lib.image.ptr().wrapping_offset(rel.offset() as isize) as Elf32_Addr),
+                }
-                _ => unreachable!()
+                _ => unreachable!(),
             }
         }
     };
 
     match rel.type_info() {
         R_ARM_PREL31 => {
-            let reloc_word = lib.image.get_ref::<Elf32_Word>(rel.offset())
+            let reloc_word = lib
-                                .ok_or("relocation offset cannot be read")?;
+                .image
-            lib.image.write(rel.offset(), (reloc_word & 0x80000000) | (value & 0x7FFFFFFF))
+                .get_ref::<Elf32_Word>(rel.offset())
-        },
+                .ok_or("relocation offset cannot be read")?;
+            lib.image
+                .write(rel.offset(), (reloc_word & 0x80000000) | (value & 0x7FFFFFFF))
+        }
 
         _ => lib.image.write(rel.offset(), value),
     }
 }
 
-pub fn rebind(
+pub fn rebind(arch: Arch, lib: &Library, name: &[u8], value: Elf32_Word) -> Result<(), Error> {
-    arch: Arch, lib: &Library, name: &[u8], value: Elf32_Word
-) -> Result<(), Error> {
     fn rebind_symbol_to_value<R: Relocatable>(
-        arch: Arch, lib: &Library,name: &[u8], value: Elf32_Word, relocs: &[R]
+        arch: Arch,
+        lib: &Library,
+        name: &[u8],
+        value: Elf32_Word,
+        relocs: &[R],
     ) -> Result<(), Error> {
         for reloc in relocs {
-            let rel_type = RelType::new(arch, reloc.type_info())
+            let rel_type = RelType::new(arch, reloc.type_info()).ok_or("unsupported relocation type")?;
-                .ok_or("unsupported relocation type")?;
             match rel_type {
                 RelType::LookupAbs => {
-                    let sym = lib.symtab().get(reloc.sym_info() as usize)
+                    let sym = lib
+                        .symtab()
+                        .get(reloc.sym_info() as usize)
                         .ok_or("symbol out of bounds of symbol table")?;
                     let sym_name = lib.name_starting_at(sym.st_name as usize)?;
 

src/libio/Cargo.toml.tpl

@@ -11,7 +11,7 @@ path = "lib.rs"
 core_io = { version = "0.1", features = ["collections"] }
 byteorder = { version = "1.0", default-features = false, optional = true }
 
-libsupport_zynq = { default-features = false, features = ["alloc_core"], git = "https://git.m-labs.hk/M-Labs/zynq-rs.git" }
+libsupport_zynq = { path = "@@ZYNQ_RS@@/libsupport_zynq", default-features = false, features = ["alloc_core"] }
 
 [features]
 alloc = []

src/libio/cursor.rs

@@ -1,9 +1,12 @@
-use core_io::{Read, Write, Error as IoError};
+#[cfg(feature = "alloc")]
+use alloc::vec::Vec;
+
+use core_io::{Error as IoError, Read, Write};
 
 #[derive(Debug, Clone)]
 pub struct Cursor<T> {
     inner: T,
-    pos:   usize
+    pos: usize,
 }
 
 impl<T> Cursor<T> {
@@ -39,22 +42,25 @@ impl<T> Cursor<T> {
 }
 
 impl<T: AsRef<[u8]>> Read for Cursor<T> {
-
     fn read(&mut self, buf: &mut [u8]) -> Result<usize, IoError> {
         let data = &self.inner.as_ref()[self.pos..];
         let len = buf.len().min(data.len());
-        buf[..len].copy_from_slice(&data[..len]);
+        // ``copy_from_slice`` generates AXI bursts, use a regular loop instead
+        for i in 0..len {
+            buf[i] = data[i];
+        }
         self.pos += len;
         Ok(len)
     }
 }
 
 impl Write for Cursor<&mut [u8]> {
-
     fn write(&mut self, buf: &[u8]) -> Result<usize, IoError> {
         let data = &mut self.inner[self.pos..];
-        let len  = buf.len().min(data.len());
+        let len = buf.len().min(data.len());
-        data[..len].copy_from_slice(&buf[..len]);
+        for i in 0..len {
+            data[i] = buf[i];
+        }
         self.pos += len;
         Ok(len)
     }
@@ -66,9 +72,7 @@ impl Write for Cursor<&mut [u8]> {
 }
 
 #[cfg(feature = "alloc")]
-impl Write for Cursor<::alloc::Vec<u8>> {
+impl Write for Cursor<Vec<u8>> {
-
-    #[inline]
     fn write(&mut self, buf: &[u8]) -> Result<usize, IoError> {
         self.inner.extend_from_slice(buf);
         Ok(buf.len())

src/libio/lib.rs

@@ -1,13 +1,10 @@
 #![no_std]
 #![feature(never_type)]
-#![cfg_attr(feature = "alloc", feature(alloc))]
 
+#[cfg(feature = "alloc")]
 extern crate alloc;
 extern crate core_io;
 
-#[cfg(feature = "alloc")]
-#[macro_use]
-use alloc;
 #[cfg(feature = "byteorder")]
 extern crate byteorder;
 
@@ -16,7 +13,7 @@ pub mod cursor;
 pub mod proto;
 
 pub use cursor::Cursor;
-#[cfg(feature = "byteorder")]
-pub use proto::{ProtoRead, ProtoWrite};
 #[cfg(all(feature = "byteorder", feature = "alloc"))]
 pub use proto::ReadStringError;
+#[cfg(feature = "byteorder")]
+pub use proto::{ProtoRead, ProtoWrite};

src/libio/proto.rs

@@ -1,15 +1,15 @@
+#[cfg(feature = "alloc")]
+use alloc::{string::String, vec};
 use core::str::Utf8Error;
-use byteorder::{ByteOrder, NativeEndian};
-use alloc::vec;
-use alloc::string::String;
 
-use core_io::{Read, Write, Error as IoError};
+use byteorder::{ByteOrder, NativeEndian};
+use core_io::{Error as IoError, Read, Write};
 
 #[allow(dead_code)]
 #[derive(Debug, Clone, PartialEq)]
 pub enum ReadStringError<T> {
     Utf8(Utf8Error),
-    Other(T)
+    Other(T),
 }
 
 pub trait ProtoRead {
@@ -51,7 +51,8 @@ pub trait ProtoRead {
     }
 
     #[inline]
-    fn read_bytes(&mut self) -> Result<::alloc::vec::Vec<u8>, Self::ReadError> {
+    #[cfg(feature = "alloc")]
+    fn read_bytes(&mut self) -> Result<vec::Vec<u8>, Self::ReadError> {
         let length = self.read_u32()?;
         let mut value = vec![0; length as usize];
         self.read_exact(&mut value)?;
@@ -59,7 +60,8 @@ pub trait ProtoRead {
     }
 
     #[inline]
-    fn read_string(&mut self) -> Result<::alloc::string::String, ReadStringError<Self::ReadError>> {
+    #[cfg(feature = "alloc")]
+    fn read_string(&mut self) -> Result<String, ReadStringError<Self::ReadError>> {
         let bytes = self.read_bytes().map_err(ReadStringError::Other)?;
         String::from_utf8(bytes).map_err(|err| ReadStringError::Utf8(err.utf8_error()))
     }
@@ -136,12 +138,15 @@ pub trait ProtoWrite {
     }
 
     #[inline]
+    #[cfg(feature = "alloc")]
     fn write_string(&mut self, value: &str) -> Result<(), Self::WriteError> {
         self.write_bytes(value.as_bytes())
     }
 }
 
-impl<T> ProtoRead for T where T: Read + ?Sized {
+impl<T> ProtoRead for T
+where T: Read + ?Sized
+{
     type ReadError = IoError;
 
     fn read_exact(&mut self, buf: &mut [u8]) -> Result<(), Self::ReadError> {
@@ -149,7 +154,9 @@ impl<T> ProtoRead for T where T: Read + ?Sized {
     }
 }
 
-impl<T> ProtoWrite for T where T: Write + ?Sized {
+impl<T> ProtoWrite for T
+where T: Write + ?Sized
+{
     type WriteError = IoError;
 
     fn write_all(&mut self, buf: &[u8]) -> Result<(), Self::WriteError> {

src/libksupport/Cargo.toml.tpl

@@ -0,0 +1,34 @@
+[package]
+name = "ksupport"
+description = "Kernel support for Zynq-based platforms"
+version = "0.1.0"
+authors = ["M-Labs"]
+edition = "2018"
+
+[build-dependencies]
+build_zynq = { path = "../libbuild_zynq" }
+
+[dependencies]
+cslice = "0.3"
+log = "0.4"
+nb = "0.1"
+core_io = { version = "0.1", features = ["collections"] }
+byteorder = { version = "1.3", default-features = false }
+void = { version = "1", default-features = false }
+log_buffer = { version = "1.2" }
+libm = { version = "0.2", features = ["unstable"] }
+vcell = "0.1"
+
+libboard_zynq = { path = "@@ZYNQ_RS@@/libboard_zynq", features = ["ipv6"]}
+libsupport_zynq = { path = "@@ZYNQ_RS@@/libsupport_zynq", default-features = false, features = ["alloc_core"] }
+libcortex_a9 = { path = "@@ZYNQ_RS@@/libcortex_a9" }
+libasync = { path = "@@ZYNQ_RS@@/libasync" }
+libregister = { path = "@@ZYNQ_RS@@/libregister" }
+libconfig = { path = "@@ZYNQ_RS@@/libconfig", features = ["fat_lfn", "ipv6"] }
+
+dyld = { path = "../libdyld" }
+dwarf = { path = "../libdwarf" }
+unwind = { path = "../libunwind" }
+libc = { path = "../libc" }
+io = { path = "../libio" }
+libboard_artiq = { path = "../libboard_artiq" }

src/libksupport/build.rs

@@ -0,0 +1,5 @@
+extern crate build_zynq;
+
+fn main() {
+    build_zynq::cfg();
+}

src/libksupport/src/eh_artiq.rs

@@ -13,14 +13,14 @@
 #![allow(non_camel_case_types)]
 
 use core::mem;
+
 use cslice::CSlice;
-use unwind as uw;
-use libc::{c_int, c_void, uintptr_t};
-use log::{trace, error};
-use crate::kernel::KERNEL_IMAGE;
-
 use dwarf::eh::{self, EHAction, EHContext};
+use libc::{c_int, c_void, uintptr_t};
+use log::{error, trace};
+use unwind as uw;
 
+use crate::kernel::KERNEL_IMAGE;
 
 const EXCEPTION_CLASS: uw::_Unwind_Exception_Class = 0x4d_4c_42_53_41_52_54_51; /* 'MLBSARTQ' */
 
@@ -32,13 +32,13 @@ const UNWIND_DATA_REG: (i32, i32) = (0, 1); // R0, R1
 #[repr(C)]
 #[derive(Clone, Copy)]
 pub struct Exception<'a> {
-    pub id:       u32,
+    pub id: u32,
-    pub file:     CSlice<'a, u8>,
+    pub file: CSlice<'a, u8>,
-    pub line:     u32,
+    pub line: u32,
-    pub column:   u32,
+    pub column: u32,
     pub function: CSlice<'a, u8>,
-    pub message:  CSlice<'a, u8>,
+    pub message: CSlice<'a, u8>,
-    pub param:    [i64; 3]
+    pub param: [i64; 3],
 }
 
 fn str_err(_: core::str::Utf8Error) -> core::fmt::Error {
@@ -55,12 +55,16 @@ fn exception_str<'a>(s: &'a CSlice<'a, u8>) -> Result<&'a str, core::str::Utf8Er
 
 impl<'a> core::fmt::Debug for Exception<'a> {
     fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
-        write!(f, "Exception {} from {} in {}:{}:{}, message: {}",
+        write!(
+            f,
+            "Exception {} from {} in {}:{}:{}, message: {}",
             self.id,
             exception_str(&self.function).map_err(str_err)?,
             exception_str(&self.file).map_err(str_err)?,
-            self.line, self.column,
+            self.line,
-            exception_str(&self.message).map_err(str_err)?)
+            self.column,
+            exception_str(&self.message).map_err(str_err)?
+        )
     }
 }
 
@@ -91,9 +95,9 @@ struct ExceptionBuffer {
 
 static mut EXCEPTION_BUFFER: ExceptionBuffer = ExceptionBuffer {
     uw_exceptions: [uw::_Unwind_Exception {
-        exception_class:   EXCEPTION_CLASS,
+        exception_class: EXCEPTION_CLASS,
         exception_cleanup: cleanup,
-        private:           [0; uw::unwinder_private_data_size],
+        private: [0; uw::unwinder_private_data_size],
     }; MAX_INFLIGHT_EXCEPTIONS],
     exceptions: [None; MAX_INFLIGHT_EXCEPTIONS + 1],
     exception_stack: [-1; MAX_INFLIGHT_EXCEPTIONS + 1],
@@ -102,17 +106,17 @@ static mut EXCEPTION_BUFFER: ExceptionBuffer = ExceptionBuffer {
     stack_pointers: [StackPointerBacktrace {
         stack_pointer: 0,
         initial_backtrace_size: 0,
-        current_backtrace_size: 0
+        current_backtrace_size: 0,
     }; MAX_INFLIGHT_EXCEPTIONS + 1],
-    exception_count: 0
+    exception_count: 0,
 };
 
-pub unsafe extern fn reset_exception_buffer() {
+pub unsafe extern "C" fn reset_exception_buffer() {
     trace!("reset exception buffer");
     EXCEPTION_BUFFER.uw_exceptions = [uw::_Unwind_Exception {
-        exception_class:   EXCEPTION_CLASS,
+        exception_class: EXCEPTION_CLASS,
         exception_cleanup: cleanup,
-        private:           [0; uw::unwinder_private_data_size],
+        private: [0; uw::unwinder_private_data_size],
     }; MAX_INFLIGHT_EXCEPTIONS];
     EXCEPTION_BUFFER.exceptions = [None; MAX_INFLIGHT_EXCEPTIONS + 1];
     EXCEPTION_BUFFER.exception_stack = [-1; MAX_INFLIGHT_EXCEPTIONS + 1];
@@ -120,26 +124,25 @@ pub unsafe extern fn reset_exception_buffer() {
     EXCEPTION_BUFFER.exception_count = 0;
 }
 
-type _Unwind_Stop_Fn = extern "C" fn(version: c_int,
+type _Unwind_Stop_Fn = extern "C" fn(
-                                     actions: i32,
+    version: c_int,
-                                     exception_class: uw::_Unwind_Exception_Class,
+    actions: i32,
-                                     exception_object: *mut uw::_Unwind_Exception,
+    exception_class: uw::_Unwind_Exception_Class,
-                                     context: *mut uw::_Unwind_Context,
+    exception_object: *mut uw::_Unwind_Exception,
-                                     stop_parameter: *mut c_void)
+    context: *mut uw::_Unwind_Context,
-                                    -> uw::_Unwind_Reason_Code;
+    stop_parameter: *mut c_void,
+) -> uw::_Unwind_Reason_Code;
 
-extern {
+extern "C" {
     // not defined in EHABI, but LLVM added it and is useful to us
-    fn _Unwind_ForcedUnwind(exception: *mut uw::_Unwind_Exception,
+    fn _Unwind_ForcedUnwind(
-                            stop_fn: _Unwind_Stop_Fn,
+        exception: *mut uw::_Unwind_Exception,
-                            stop_parameter: *mut c_void) -> uw::_Unwind_Reason_Code;
+        stop_fn: _Unwind_Stop_Fn,
+        stop_parameter: *mut c_void,
+    ) -> uw::_Unwind_Reason_Code;
 }
 
-unsafe fn find_eh_action(
+unsafe fn find_eh_action(context: *mut uw::_Unwind_Context, foreign_exception: bool, id: u32) -> Result<EHAction, ()> {
-    context: *mut uw::_Unwind_Context,
-    foreign_exception: bool,
-    id: u32,
-) -> Result<EHAction, ()> {
     let lsda = uw::_Unwind_GetLanguageSpecificData(context) as *const u8;
     let mut ip_before_instr: c_int = 0;
     let ip = uw::_Unwind_GetIPInfo(context, &mut ip_before_instr);
@@ -154,10 +157,11 @@ unsafe fn find_eh_action(
     eh::find_eh_action(lsda, &eh_context, foreign_exception, id)
 }
 
-pub unsafe fn artiq_personality(_state: uw::_Unwind_State,
+pub unsafe fn artiq_personality(
-                                exception_object: *mut uw::_Unwind_Exception,
+    _state: uw::_Unwind_State,
-                                context: *mut uw::_Unwind_Context)
+    exception_object: *mut uw::_Unwind_Exception,
-                                -> uw::_Unwind_Reason_Code {
+    context: *mut uw::_Unwind_Context,
+) -> uw::_Unwind_Reason_Code {
     // we will only do phase 2 forced unwinding now
     // The DWARF unwinder assumes that _Unwind_Context holds things like the function
     // and LSDA pointers, however ARM EHABI places them into the exception object.
@@ -165,9 +169,7 @@ pub unsafe fn artiq_personality(_state: uw::_Unwind_State,
     // take only the context pointer, GCC personality routines stash a pointer to
     // exception_object in the context, using location reserved for ARM's
     // "scratch register" (r12).
-    uw::_Unwind_SetGR(context,
+    uw::_Unwind_SetGR(context, uw::UNWIND_POINTER_REG, exception_object as uw::_Unwind_Ptr);
-                      uw::UNWIND_POINTER_REG,
-                      exception_object as uw::_Unwind_Ptr);
     // ...A more principled approach would be to provide the full definition of ARM's
     // _Unwind_Context in our libunwind bindings and fetch the required data from there
     // directly, bypassing DWARF compatibility functions.
@@ -186,10 +188,8 @@ pub unsafe fn artiq_personality(_state: uw::_Unwind_State,
     };
     match eh_action {
         EHAction::None => return continue_unwind(exception_object, context),
-        EHAction::Cleanup(lpad) |
+        EHAction::Cleanup(lpad) | EHAction::Catch(lpad) => {
-        EHAction::Catch(lpad) => {
+            uw::_Unwind_SetGR(context, UNWIND_DATA_REG.0, exception_object as uintptr_t);
-            uw::_Unwind_SetGR(context, UNWIND_DATA_REG.0,
-                              exception_object as uintptr_t);
             uw::_Unwind_SetGR(context, UNWIND_DATA_REG.1, exception as *const _ as uw::_Unwind_Word);
             uw::_Unwind_SetIP(context, lpad);
             return uw::_URC_INSTALL_CONTEXT;
@@ -199,9 +199,10 @@ pub unsafe fn artiq_personality(_state: uw::_Unwind_State,
 
     // On ARM EHABI the personality routine is responsible for actually
     // unwinding a single stack frame before returning (ARM EHABI Sec. 6.1).
-    unsafe fn continue_unwind(exception_object: *mut uw::_Unwind_Exception,
+    unsafe fn continue_unwind(
-                              context: *mut uw::_Unwind_Context)
+        exception_object: *mut uw::_Unwind_Exception,
-                              -> uw::_Unwind_Reason_Code {
+        context: *mut uw::_Unwind_Context,
+    ) -> uw::_Unwind_Reason_Code {
         let reason = __gnu_unwind_frame(exception_object, context);
         if reason == uw::_URC_NO_REASON {
             uw::_URC_CONTINUE_UNWIND
@@ -211,13 +212,14 @@ pub unsafe fn artiq_personality(_state: uw::_Unwind_State,
     }
     // defined in libgcc
     extern "C" {
-        fn __gnu_unwind_frame(exception_object: *mut uw::_Unwind_Exception,
+        fn __gnu_unwind_frame(
-                              context: *mut uw::_Unwind_Context)
+            exception_object: *mut uw::_Unwind_Exception,
-                              -> uw::_Unwind_Reason_Code;
+            context: *mut uw::_Unwind_Context,
+        ) -> uw::_Unwind_Reason_Code;
     }
 }
 
-pub unsafe extern fn raise(exception: *const Exception) -> ! {
+pub unsafe extern "C" fn raise(exception: *const Exception) -> ! {
     use cslice::AsCSlice;
 
     let count = EXCEPTION_BUFFER.exception_count;
@@ -244,8 +246,11 @@ pub unsafe extern fn raise(exception: *const Exception) -> ! {
             }
         }
         assert!(found);
-        let _result = _Unwind_ForcedUnwind(&mut EXCEPTION_BUFFER.uw_exceptions[stack[count - 1] as usize],
+        let _result = _Unwind_ForcedUnwind(
-                                           stop_fn, core::ptr::null_mut());
+            &mut EXCEPTION_BUFFER.uw_exceptions[stack[count - 1] as usize],
+            stop_fn,
+            core::ptr::null_mut(),
+        );
     } else {
         if count < MAX_INFLIGHT_EXCEPTIONS {
             trace!("raising exception at level {}", count);
@@ -253,34 +258,33 @@ pub unsafe extern fn raise(exception: *const Exception) -> ! {
             for (i, slot) in EXCEPTION_BUFFER.exceptions.iter_mut().enumerate() {
                 // we should always be able to find a slot
                 if slot.is_none() {
-                    *slot = Some(
+                    *slot = Some(*mem::transmute::<*const Exception, *const Exception<'static>>(
-                        *mem::transmute::<*const Exception, *const Exception<'static>>
+                        exception,
-                        (exception));
+                    ));
                     EXCEPTION_BUFFER.exception_stack[count] = i as isize;
-                    EXCEPTION_BUFFER.uw_exceptions[i].private =
+                    EXCEPTION_BUFFER.uw_exceptions[i].private = [0; uw::unwinder_private_data_size];
-                        [0; uw::unwinder_private_data_size];
                     EXCEPTION_BUFFER.stack_pointers[i] = StackPointerBacktrace {
                         stack_pointer: 0,
                         initial_backtrace_size: EXCEPTION_BUFFER.backtrace_size,
                         current_backtrace_size: 0,
                     };
                     EXCEPTION_BUFFER.exception_count += 1;
-                    let _result = _Unwind_ForcedUnwind(&mut EXCEPTION_BUFFER.uw_exceptions[i],
+                    let _result =
-                                                       stop_fn, core::ptr::null_mut());
+                        _Unwind_ForcedUnwind(&mut EXCEPTION_BUFFER.uw_exceptions[i], stop_fn, core::ptr::null_mut());
                 }
             }
         } else {
             error!("too many nested exceptions");
             // TODO: better reporting?
             let exception = Exception {
-                id:       get_exception_id("RuntimeError"),
+                id: get_exception_id("RuntimeError"),
-                file:     file!().as_c_slice(),
+                file: file!().as_c_slice(),
-                line:     line!(),
+                line: line!(),
-                column:   column!(),
+                column: column!(),
                 // https://github.com/rust-lang/rfcs/pull/1719
                 function: "__artiq_raise".as_c_slice(),
-                message:  "too many nested exceptions".as_c_slice(),
+                message: "too many nested exceptions".as_c_slice(),
-                param:    [0, 0, 0]
+                param: [0, 0, 0],
             };
             EXCEPTION_BUFFER.exceptions[MAX_INFLIGHT_EXCEPTIONS] = Some(mem::transmute(exception));
             EXCEPTION_BUFFER.stack_pointers[MAX_INFLIGHT_EXCEPTIONS] = Default::default();
@@ -291,17 +295,20 @@ pub unsafe extern fn raise(exception: *const Exception) -> ! {
     unreachable!();
 }
 
-pub unsafe extern fn resume() -> ! {
+pub unsafe extern "C" fn resume() -> ! {
     trace!("resume");
     assert!(EXCEPTION_BUFFER.exception_count != 0);
     let i = EXCEPTION_BUFFER.exception_stack[EXCEPTION_BUFFER.exception_count - 1];
     assert!(i != -1);
-    let _result = _Unwind_ForcedUnwind(&mut EXCEPTION_BUFFER.uw_exceptions[i as usize],
+    let _result = _Unwind_ForcedUnwind(
-                                       stop_fn, core::ptr::null_mut());
+        &mut EXCEPTION_BUFFER.uw_exceptions[i as usize],
+        stop_fn,
+        core::ptr::null_mut(),
+    );
     unreachable!()
 }
 
-pub unsafe extern fn end_catch() {
+pub unsafe extern "C" fn end_catch() {
     let mut count = EXCEPTION_BUFFER.exception_count;
     assert!(count != 0);
     // we remove all exceptions with SP <= current exception SP
@@ -309,8 +316,7 @@ pub unsafe extern fn end_catch() {
     let index = EXCEPTION_BUFFER.exception_stack[count - 1] as usize;
     EXCEPTION_BUFFER.exception_stack[count - 1] = -1;
     EXCEPTION_BUFFER.exceptions[index] = None;
-    let outer_sp = EXCEPTION_BUFFER.stack_pointers
+    let outer_sp = EXCEPTION_BUFFER.stack_pointers[index].stack_pointer;
-        [index].stack_pointer;
     count -= 1;
     for i in (0..count).rev() {
         let index = EXCEPTION_BUFFER.exception_stack[i];
@@ -334,8 +340,7 @@ pub unsafe extern fn end_catch() {
     };
 }
 
-extern fn cleanup(_unwind_code: uw::_Unwind_Reason_Code,
+extern "C" fn cleanup(_unwind_code: uw::_Unwind_Reason_Code, _uw_exception: *mut uw::_Unwind_Exception) {
-                  _uw_exception: *mut uw::_Unwind_Exception) {
     unimplemented!()
 }
 
@@ -345,18 +350,26 @@ fn uncaught_exception() -> ! {
         for i in 0..EXCEPTION_BUFFER.exception_count {
             if EXCEPTION_BUFFER.exception_stack[i] != i as isize {
                 // find the correct index
-                let index = EXCEPTION_BUFFER.exception_stack
+                let index = EXCEPTION_BUFFER
+                    .exception_stack
                     .iter()
-                    .position(|v| *v == i as isize).unwrap();
+                    .position(|v| *v == i as isize)
+                    .unwrap();
                 let a = EXCEPTION_BUFFER.exception_stack[index];
                 let b = EXCEPTION_BUFFER.exception_stack[i];
                 assert!(a != -1 && b != -1);
-                core::mem::swap(&mut EXCEPTION_BUFFER.exception_stack[index],
+                core::mem::swap(
-                                &mut EXCEPTION_BUFFER.exception_stack[i]);
+                    &mut EXCEPTION_BUFFER.exception_stack[index],
-                core::mem::swap(&mut EXCEPTION_BUFFER.exceptions[a as usize],
+                    &mut EXCEPTION_BUFFER.exception_stack[i],
-                                &mut EXCEPTION_BUFFER.exceptions[b as usize]);
+                );
-                core::mem::swap(&mut EXCEPTION_BUFFER.stack_pointers[a as usize],
+                core::mem::swap(
-                                &mut EXCEPTION_BUFFER.stack_pointers[b as usize]);
+                    &mut EXCEPTION_BUFFER.exceptions[a as usize],
+                    &mut EXCEPTION_BUFFER.exceptions[b as usize],
+                );
+                core::mem::swap(
+                    &mut EXCEPTION_BUFFER.stack_pointers[a as usize],
+                    &mut EXCEPTION_BUFFER.stack_pointers[b as usize],
+                );
             }
         }
     }
@@ -364,17 +377,20 @@ fn uncaught_exception() -> ! {
         crate::kernel::core1::terminate(
             EXCEPTION_BUFFER.exceptions[..EXCEPTION_BUFFER.exception_count].as_ref(),
             EXCEPTION_BUFFER.stack_pointers[..EXCEPTION_BUFFER.exception_count].as_ref(),
-            EXCEPTION_BUFFER.backtrace[..EXCEPTION_BUFFER.backtrace_size].as_mut())
+            EXCEPTION_BUFFER.backtrace[..EXCEPTION_BUFFER.backtrace_size].as_mut(),
+        )
     }
 }
 
 // stop function which would be executed when we unwind each frame
-extern fn stop_fn(_version: c_int,
+extern "C" fn stop_fn(
-                  actions: i32,
+    _version: c_int,
-                  _uw_exception_class: uw::_Unwind_Exception_Class,
+    actions: i32,
-                  _uw_exception: *mut uw::_Unwind_Exception,
+    _uw_exception_class: uw::_Unwind_Exception_Class,
-                  context: *mut uw::_Unwind_Context,
+    _uw_exception: *mut uw::_Unwind_Exception,
-                  _stop_parameter: *mut c_void) -> uw::_Unwind_Reason_Code {
+    context: *mut uw::_Unwind_Context,
+    _stop_parameter: *mut c_void,
+) -> uw::_Unwind_Reason_Code {
     unsafe {
         let load_addr = KERNEL_IMAGE.as_ref().unwrap().get_load_addr();
         let backtrace_size = EXCEPTION_BUFFER.backtrace_size;
@@ -406,7 +422,7 @@ extern fn stop_fn(_version: c_int,
 }
 
 // Must be kept in sync with preallocate_runtime_exception_names() in artiq/language/embedding_map.py
-static EXCEPTION_ID_LOOKUP: [(&str, u32); 11] = [
+static EXCEPTION_ID_LOOKUP: [(&str, u32); 12] = [
     ("RuntimeError", 0),
     ("RTIOUnderflow", 1),
     ("RTIOOverflow", 2),
@@ -418,12 +434,13 @@ static EXCEPTION_ID_LOOKUP: [(&str, u32); 11] = [
     ("ZeroDivisionError", 8),
     ("IndexError", 9),
     ("UnwrapNoneError", 10),
+    ("SubkernelError", 11),
 ];
 
 pub fn get_exception_id(name: &str) -> u32 {
     for (n, id) in EXCEPTION_ID_LOOKUP.iter() {
         if *n == name {
-            return *id
+            return *id;
         }
     }
     unimplemented!("unallocated internal exception id")
@@ -431,25 +448,24 @@ pub fn get_exception_id(name: &str) -> u32 {
 
 #[macro_export]
 macro_rules! artiq_raise {
-    ($name:expr, $message:expr, $param0:expr, $param1:expr, $param2:expr) => ({
+    ($name:expr, $message:expr, $param0:expr, $param1:expr, $param2:expr) => {{
         use cslice::AsCSlice;
         let name_id = $crate::eh_artiq::get_exception_id($name);
+        let message_cl = $message.clone();
         let exn = $crate::eh_artiq::Exception {
-            id:       name_id,
+            id: name_id,
-            file:     file!().as_c_slice(),
+            file: file!().as_c_slice(),
-            line:     line!(),
+            line: line!(),
-            column:   column!(),
+            column: column!(),
             // https://github.com/rust-lang/rfcs/pull/1719
             function: "(Rust function)".as_c_slice(),
-            message:  $message.as_c_slice(),
+            message: message_cl.as_c_slice(),
-            param:    [$param0, $param1, $param2]
+            param: [$param0, $param1, $param2],
         };
         #[allow(unused_unsafe)]
         unsafe {
             $crate::eh_artiq::raise(&exn)
         }
-    });
+    }};
-    ($name:expr, $message:expr) => ({
+    ($name:expr, $message:expr) => {{ artiq_raise!($name, $message, 0, 0, 0) }};
-        artiq_raise!($name, $message, 0, 0, 0)
-    });
 }

src/libksupport/src/i2c.rs

@@ -1,9 +1,10 @@
 use libboard_zynq;
+
 use crate::artiq_raise;
 
 pub static mut I2C_BUS: Option<libboard_zynq::i2c::I2c> = None;
 
-pub extern fn start(busno: i32) {
+pub extern "C" fn start(busno: i32) {
     if busno > 0 {
         artiq_raise!("I2CError", "I2C bus could not be accessed");
     }
@@ -14,7 +15,7 @@ pub extern fn start(busno: i32) {
     }
 }
 
-pub extern fn restart(busno: i32) {
+pub extern "C" fn restart(busno: i32) {
     if busno > 0 {
         artiq_raise!("I2CError", "I2C bus could not be accessed");
     }
@@ -25,7 +26,7 @@ pub extern fn restart(busno: i32) {
     }
 }
 
-pub extern fn stop(busno: i32) {
+pub extern "C" fn stop(busno: i32) {
     if busno > 0 {
         artiq_raise!("I2CError", "I2C bus could not be accessed");
     }
@@ -36,7 +37,7 @@ pub extern fn stop(busno: i32) {
     }
 }
 
-pub extern fn write(busno: i32, data: i32) -> bool {
+pub extern "C" fn write(busno: i32, data: i32) -> bool {
     if busno > 0 {
         artiq_raise!("I2CError", "I2C bus could not be accessed");
     }
@@ -48,7 +49,7 @@ pub extern fn write(busno: i32, data: i32) -> bool {
     }
 }
 
-pub extern fn read(busno: i32, ack: bool) -> i32 {
+pub extern "C" fn read(busno: i32, ack: bool) -> i32 {
     if busno > 0 {
         artiq_raise!("I2CError", "I2C bus could not be accessed");
     }
@@ -60,11 +61,12 @@ pub extern fn read(busno: i32, ack: bool) -> i32 {
     }
 }
 
-pub extern fn switch_select(busno: i32, address: i32, mask: i32) {
+pub extern "C" fn switch_select(busno: i32, address: i32, mask: i32) {
     if busno > 0 {
         artiq_raise!("I2CError", "I2C bus could not be accessed");
     }
-    let ch = match mask { //decode from mainline, PCA9548-centric API
+    let ch = match mask {
+        //decode from mainline, PCA9548-centric API
         0x00 => None,
         0x01 => Some(0),
         0x02 => Some(1),
@@ -74,10 +76,15 @@ pub extern fn switch_select(busno: i32, address: i32, mask: i32) {
         0x20 => Some(5),
         0x40 => Some(6),
         0x80 => Some(7),
-        _ => artiq_raise!("I2CError", "switch select supports only one channel")
+        _ => artiq_raise!("I2CError", "switch select supports only one channel"),
     };
     unsafe {
-        if (&mut I2C_BUS).as_mut().unwrap().pca954x_select(address as u8, ch).is_err() {
+        if (&mut I2C_BUS)
+            .as_mut()
+            .unwrap()
+            .pca954x_select(address as u8, ch)
+            .is_err()
+        {
             artiq_raise!("I2CError", "switch select failed");
         }
     }

src/libksupport/src/irq.rs

@@ -1,12 +1,9 @@
-use libboard_zynq::{gic, mpcore, println, stdio};
-use libcortex_a9::{
-    asm, interrupt_handler,
-    regs::MPIDR,
-    spin_lock_yield, notify_spin_lock
-};
-use libregister::RegisterR;
 use core::sync::atomic::{AtomicBool, Ordering};
 
+use libboard_zynq::{gic, mpcore, println, stdio};
+use libcortex_a9::{asm, interrupt_handler, notify_spin_lock, regs::MPIDR, spin_lock_yield};
+use libregister::RegisterR;
+
 extern "C" {
     static mut __stack1_start: u32;
     fn main_core1() -> !;

src/libksupport/src/kernel/api.rs

@@ -1,29 +1,27 @@
-use core::ffi::VaList;
+use alloc::vec;
-use core::ptr;
+use core::{ffi::VaList, ptr, str};
-use core::str;
+
 use libc::{c_char, c_int, size_t};
 use libm;
 use log::{info, warn};
 
-use alloc::vec;
+#[cfg(has_drtio)]
-
+use super::subkernel;
-use crate::eh_artiq;
+use super::{cache,
-use crate::rtio;
+            core1::rtio_get_destination_status,
-use crate::i2c;
+            dma,
-use super::rpc::{rpc_send, rpc_send_async, rpc_recv};
+            rpc::{rpc_recv, rpc_send, rpc_send_async}};
-use super::dma;
+use crate::{eh_artiq, i2c, rtio};
-use super::cache;
-use super::core1::rtio_get_destination_status;
 
 extern "C" {
     fn vsnprintf_(buffer: *mut c_char, count: size_t, format: *const c_char, va: VaList) -> c_int;
 }
 
-unsafe extern fn core_log(fmt: *const c_char, mut args: ...) {
+unsafe extern "C" fn core_log(fmt: *const c_char, mut args: ...) {
     let size = vsnprintf_(ptr::null_mut(), 0, fmt, args.as_va_list()) as usize;
     let mut buf = vec![0; size + 1];
     vsnprintf_(buf.as_mut_ptr() as *mut i8, size + 1, fmt, args.as_va_list());
-    let buf: &[u8] = &buf.as_slice()[..size-1]; // strip \n and NUL
+    let buf: &[u8] = &buf.as_slice()[..size - 1]; // strip \n and NUL
     match str::from_utf8(buf) {
         Ok(s) => info!("kernel: {}", s),
         Err(e) => {
@@ -33,14 +31,13 @@ unsafe extern fn core_log(fmt: *const c_char, mut args: ...) {
     }
 }
 
-unsafe extern fn rtio_log(fmt: *const c_char, mut args: ...) {
+unsafe extern "C" fn rtio_log(fmt: *const c_char, mut args: ...) {
     let size = vsnprintf_(ptr::null_mut(), 0, fmt, args.as_va_list()) as usize;
     let mut buf = vec![0; size + 1];
     vsnprintf_(buf.as_mut_ptr(), size + 1, fmt, args.as_va_list());
     rtio::write_log(buf.as_slice());
 }
 
-
 macro_rules! api {
     ($i:ident) => ({
         extern { static $i: u8; }
@@ -56,24 +53,25 @@ macro_rules! api {
 }
 
 macro_rules! api_libm_f64f64 {
-    ($i:ident) => ({
+    ($i:ident) => {{
-        extern fn $i(x: f64) -> f64 {
+        extern "C" fn $i(x: f64) -> f64 {
             libm::$i(x)
         }
         api!($i = $i)
-    })
+    }};
 }
 
 macro_rules! api_libm_f64f64f64 {
-    ($i:ident) => ({
+    ($i:ident) => {{
-        extern fn $i(x: f64, y: f64) -> f64 {
+        extern "C" fn $i(x: f64, y: f64) -> f64 {
             libm::$i(x, y)
         }
         api!($i = $i)
-    })
+    }};
 }
 
 pub fn resolve(required: &[u8]) -> Option<u32> {
+    #[rustfmt::skip]
     let api = &[
         // timing
         api!(now_mu = rtio::now_mu),
@@ -118,12 +116,23 @@ pub fn resolve(required: &[u8]) -> Option<u32> {
         api!(i2c_read = i2c::read),
         api!(i2c_switch_select = i2c::switch_select),
 
+        // subkernel
+        #[cfg(has_drtio)]
+        api!(subkernel_load_run = subkernel::load_run),
+        #[cfg(has_drtio)]
+        api!(subkernel_await_finish = subkernel::await_finish),
+        #[cfg(has_drtio)]
+        api!(subkernel_send_message = subkernel::send_message),
+        #[cfg(has_drtio)]
+        api!(subkernel_await_message = subkernel::await_message),
+
         // Double-precision floating-point arithmetic helper functions
         // RTABI chapter 4.1.2, Table 2
         api!(__aeabi_dadd),
         api!(__aeabi_ddiv),
         api!(__aeabi_dmul),
         api!(__aeabi_dsub),
+
         // Double-precision floating-point comparison helper functions
         // RTABI chapter 4.1.2, Table 3
         api!(__aeabi_dcmpeq),
@@ -133,12 +142,14 @@ pub fn resolve(required: &[u8]) -> Option<u32> {
         api!(__aeabi_dcmpge),
         api!(__aeabi_dcmpgt),
         api!(__aeabi_dcmpun),
+
         // Single-precision floating-point arithmetic helper functions
         // RTABI chapter 4.1.2, Table 4
         api!(__aeabi_fadd),
         api!(__aeabi_fdiv),
         api!(__aeabi_fmul),
         api!(__aeabi_fsub),
+
         // Single-precision floating-point comparison helper functions
         // RTABI chapter 4.1.2, Table 5
         api!(__aeabi_fcmpeq),
@@ -148,6 +159,7 @@ pub fn resolve(required: &[u8]) -> Option<u32> {
         api!(__aeabi_fcmpge),
         api!(__aeabi_fcmpgt),
         api!(__aeabi_fcmpun),
+
         // Floating-point to integer conversions.
         // RTABI chapter 4.1.2, Table 6
         api!(__aeabi_d2iz),
@@ -158,9 +170,11 @@ pub fn resolve(required: &[u8]) -> Option<u32> {
         api!(__aeabi_f2uiz),
         api!(__aeabi_f2lz),
         api!(__aeabi_f2ulz),
+
         // Conversions between floating types.
         // RTABI chapter 4.1.2, Table 7
         api!(__aeabi_f2d),
+
         // Integer to floating-point conversions.
         // RTABI chapter 4.1.2, Table 8
         api!(__aeabi_i2d),
@@ -171,12 +185,14 @@ pub fn resolve(required: &[u8]) -> Option<u32> {
         api!(__aeabi_ui2f),
         api!(__aeabi_l2f),
         api!(__aeabi_ul2f),
+
         // Long long helper functions
         // RTABI chapter 4.2, Table 9
         api!(__aeabi_lmul),
         api!(__aeabi_llsl),
         api!(__aeabi_llsr),
         api!(__aeabi_lasr),
+
         // Integer division functions
         // RTABI chapter 4.3.1
         api!(__aeabi_idiv),
@@ -184,6 +200,7 @@ pub fn resolve(required: &[u8]) -> Option<u32> {
         api!(__aeabi_idivmod),
         api!(__aeabi_uidiv),
         api!(__aeabi_uldivmod),
+
         // 4.3.4 Memory copying, clearing, and setting
         api!(__aeabi_memcpy8),
         api!(__aeabi_memcpy4),
@@ -199,10 +216,30 @@ pub fn resolve(required: &[u8]) -> Option<u32> {
         api!(__aeabi_memclr),
 
         // libc
-        api!(memcpy, extern { fn memcpy(dest: *mut u8, src: *const u8, n: usize) -> *mut u8; }),
+        api!(
-        api!(memmove, extern { fn memmove(dest: *mut u8, src: *const u8, n: usize) -> *mut u8; }),
+            memcpy,
-        api!(memset, extern { fn memset(s: *mut u8, c: i32, n: usize) -> *mut u8; }),
+            extern "C" {
-        api!(memcmp, extern { fn memcmp(s1: *const u8, s2: *const u8, n: usize) -> i32; }),
+                fn memcpy(dest: *mut u8, src: *const u8, n: usize) -> *mut u8;
+            }
+        ),
+        api!(
+            memmove,
+            extern "C" {
+                fn memmove(dest: *mut u8, src: *const u8, n: usize) -> *mut u8;
+            }
+        ),
+        api!(
+            memset,
+            extern "C" {
+                fn memset(s: *mut u8, c: i32, n: usize) -> *mut u8;
+            }
+        ),
+        api!(
+            memcmp,
+            extern "C" {
+                fn memcmp(s1: *const u8, s2: *const u8, n: usize) -> i32;
+            }
+        ),
 
         // exceptions
         api!(_Unwind_Resume = unwind::_Unwind_Resume),
@@ -210,6 +247,7 @@ pub fn resolve(required: &[u8]) -> Option<u32> {
         api!(__nac3_raise = eh_artiq::raise),
         api!(__nac3_resume = eh_artiq::resume),
         api!(__nac3_end_catch = eh_artiq::end_catch),
+
         // legacy exception symbols
         api!(__artiq_personality = eh_artiq::artiq_personality),
         api!(__artiq_raise = eh_artiq::raise),
@@ -241,7 +279,7 @@ pub fn resolve(required: &[u8]) -> Option<u32> {
         api_libm_f64f64!(fabs),
         api_libm_f64f64!(floor),
         {
-            extern fn fma(x: f64, y: f64, z: f64) -> f64 {
+            extern "C" fn fma(x: f64, y: f64, z: f64) -> f64 {
                 libm::fma(x, y, z)
             }
             api!(fma = fma)
@@ -253,7 +291,7 @@ pub fn resolve(required: &[u8]) -> Option<u32> {
         api_libm_f64f64!(j0),
         api_libm_f64f64!(j1),
         {
-            extern fn jn(n: i32, x: f64) -> f64 {
+            extern "C" fn jn(n: i32, x: f64) -> f64 {
                 libm::jn(n, x)
             }
             api!(jn = jn)
@@ -275,13 +313,13 @@ pub fn resolve(required: &[u8]) -> Option<u32> {
         api_libm_f64f64!(y0),
         api_libm_f64f64!(y1),
         {
-            extern fn yn(n: i32, x: f64) -> f64 {
+            extern "C" fn yn(n: i32, x: f64) -> f64 {
                 libm::yn(n, x)
             }
             api!(yn = yn)
         },
     ];
     api.iter()
-       .find(|&&(exported, _)| exported.as_bytes() == required)
+        .find(|&&(exported, _)| exported.as_bytes() == required)
-       .map(|&(_, ptr)| ptr as u32)
+        .map(|&(_, ptr)| ptr as u32)
 }

src/libksupport/src/kernel/cache.rs

@@ -1,12 +1,17 @@
-use alloc::{string::String, boxed::Box};
+use alloc::{boxed::Box, string::String};
-use cslice::{CSlice, AsCSlice};
+use core::mem::{forget, transmute};
-use core::mem::{transmute, forget};
-use super::{KERNEL_CHANNEL_0TO1, KERNEL_CHANNEL_1TO0, Message};
 
-pub extern fn get(key: CSlice<u8>) -> &CSlice<'static, i32> {
+use cslice::{AsCSlice, CSlice};
+
+use super::{Message, KERNEL_CHANNEL_0TO1, KERNEL_CHANNEL_1TO0};
+
+pub extern "C" fn get(key: CSlice<u8>) -> &CSlice<'static, i32> {
     let key = String::from_utf8(key.as_ref().to_vec()).unwrap();
     unsafe {
-        KERNEL_CHANNEL_1TO0.as_mut().unwrap().send(Message::CacheGetRequest(key));
+        KERNEL_CHANNEL_1TO0
+            .as_mut()
+            .unwrap()
+            .send(Message::CacheGetRequest(key));
         let msg = KERNEL_CHANNEL_0TO1.as_mut().unwrap().recv();
         if let Message::CacheGetReply(v) = msg {
             let leaked = Box::new(v.as_c_slice());
@@ -20,11 +25,13 @@ pub extern fn get(key: CSlice<u8>) -> &CSlice<'static, i32> {
     }
 }
 
-pub extern fn put(key: CSlice<u8>, list: &CSlice<i32>) {
+pub extern "C" fn put(key: CSlice<u8>, list: &CSlice<i32>) {
     let key = String::from_utf8(key.as_ref().to_vec()).unwrap();
     let value = list.as_ref().to_vec();
     unsafe {
-        KERNEL_CHANNEL_1TO0.as_mut().unwrap().send(Message::CachePutRequest(key, value));
+        KERNEL_CHANNEL_1TO0
+            .as_mut()
+            .unwrap()
+            .send(Message::CachePutRequest(key, value));
     }
 }
-

src/libksupport/src/kernel/control.rs

@@ -1,11 +1,11 @@
-use libcortex_a9::sync_channel::{Sender, Receiver};
+use core::mem::{forget, replace};
+
+use libcortex_a9::sync_channel::{Receiver, Sender};
 use libsupport_zynq::boot::Core1;
 
-use super::{CHANNEL_0TO1, CHANNEL_1TO0, CHANNEL_SEM, INIT_LOCK, Message};
+use super::{Message, CHANNEL_0TO1, CHANNEL_1TO0, CHANNEL_SEM, INIT_LOCK};
 use crate::irq::restart_core1;
 
-use core::mem::{forget, replace};
-
 pub struct Control {
     pub tx: Sender<'static, Message>,
     pub rx: Receiver<'static, Message>,
@@ -53,4 +53,3 @@ impl Control {
         forget(replace(&mut self.rx, core0_rx));
     }
 }
-

src/libksupport/src/kernel/core1.rs

@@ -1,31 +1,20 @@
 //! Kernel prologue/epilogue that runs on the 2nd CPU core
 
-use core::{mem, ptr, cell::UnsafeCell};
 use alloc::borrow::ToOwned;
-use log::{debug, info, error};
+use core::{cell::UnsafeCell, mem, ptr};
-use cslice::CSlice;
 
-use libcortex_a9::{
+use cslice::CSlice;
-    enable_fpu,
+use dyld::{self, elf::EXIDX_Entry, Library};
-    cache::{dcci_slice, iciallu, bpiall},
+use libboard_zynq::{gic, mpcore};
-    asm::{dsb, isb},
+use libcortex_a9::{asm::{dsb, isb},
-    sync_channel,
+                   cache::{bpiall, dcci_slice, iciallu},
-};
+                   enable_fpu, sync_channel};
-use libboard_zynq::{mpcore, gic};
 use libsupport_zynq::ram;
-use dyld::{self, Library, elf::EXIDX_Entry};
+use log::{debug, error, info};
+
+use super::{api::resolve, dma, rpc::rpc_send_async, Message, CHANNEL_0TO1, CHANNEL_1TO0, CHANNEL_SEM, INIT_LOCK,
+            KERNEL_CHANNEL_0TO1, KERNEL_CHANNEL_1TO0, KERNEL_IMAGE};
 use crate::{eh_artiq, get_async_errors, rtio};
-use super::{
-    api::resolve,
-    rpc::rpc_send_async,
-    INIT_LOCK,
-    CHANNEL_0TO1, CHANNEL_1TO0,
-    CHANNEL_SEM,
-    KERNEL_CHANNEL_0TO1, KERNEL_CHANNEL_1TO0,
-    KERNEL_IMAGE,
-    Message,
-    dma,
-};
 
 // linker symbols
 extern "C" {
@@ -38,13 +27,13 @@ extern "C" {
 unsafe fn attribute_writeback(typeinfo: *const ()) {
     struct Attr {
         offset: usize,
-        tag:    CSlice<'static, u8>,
+        tag: CSlice<'static, u8>,
-        name:   CSlice<'static, u8>
+        name: CSlice<'static, u8>,
     }
 
     struct Type {
         attributes: *const *const Attr,
-        objects:    *const *const ()
+        objects: *const *const (),
     }
 
     let mut tys = typeinfo as *const *const Type;
@@ -63,11 +52,16 @@ unsafe fn attribute_writeback(typeinfo: *const ()) {
                 attributes = attributes.offset(1);
 
                 if (*attribute).tag.len() > 0 {
-                    rpc_send_async(0, &(*attribute).tag, [
+                    rpc_send_async(
-                        &object as *const _ as *const (),
+                        0,
-                        &(*attribute).name as *const _ as *const (),
+                        &(*attribute).tag,
-                        (object as usize + (*attribute).offset) as *const ()
+                        [
-                    ].as_ptr());
+                            &object as *const _ as *const (),
+                            &(*attribute).name as *const _ as *const (),
+                            (object as usize + (*attribute).offset) as *const (),
+                        ]
+                        .as_ptr(),
+                    );
                 }
             }
         }
@@ -82,7 +76,8 @@ pub struct KernelImage {
 
 impl KernelImage {
     pub fn new(library: Library) -> Result<Self, dyld::Error> {
-        let __modinit__ = library.lookup(b"__modinit__")
+        let __modinit__ = library
+            .lookup(b"__modinit__")
             .ok_or(dyld::Error::Lookup("__modinit__".to_owned()))?;
         let typeinfo = library.lookup(b"typeinfo");
 
@@ -90,8 +85,7 @@ impl KernelImage {
         let bss_start = library.lookup(b"__bss_start");
         let end = library.lookup(b"_end");
         if let Some(bss_start) = bss_start {
-            let end = end
+            let end = end.ok_or(dyld::Error::Lookup("_end".to_owned()))?;
-                .ok_or(dyld::Error::Lookup("_end".to_owned()))?;
             unsafe {
                 ptr::write_bytes(bss_start as *mut u8, 0, (end - bss_start) as usize);
             }
@@ -126,9 +120,7 @@ impl KernelImage {
     }
 
     pub fn get_load_addr(&self) -> usize {
-        unsafe {
+        unsafe { self.library.get().as_ref().unwrap().image.as_ptr() as usize }
-            self.library.get().as_ref().unwrap().image.as_ptr() as usize
-        }
     }
 }
 
@@ -164,20 +156,19 @@ pub extern "C" fn main_core1() {
         let message = core1_rx.recv();
         match message {
             Message::LoadRequest(data) => {
-                let result = dyld::load(&data, &resolve)
+                let result = dyld::load(&data, &resolve).and_then(KernelImage::new);
-                    .and_then(KernelImage::new);
                 match result {
                     Ok(kernel) => {
                         loaded_kernel = Some(kernel);
                         debug!("kernel loaded");
                         core1_tx.send(Message::LoadCompleted);
-                    },
+                    }
                     Err(error) => {
                         error!("failed to load shared library: {}", error);
                         core1_tx.send(Message::LoadFailed);
                     }
                 }
-            },
+            }
             Message::StartRequest => {
                 info!("kernel starting");
                 if let Some(kernel) = loaded_kernel.take() {
@@ -202,9 +193,11 @@ pub extern "C" fn main_core1() {
 }
 
 /// Called by eh_artiq
-pub fn terminate(exceptions: &'static [Option<eh_artiq::Exception<'static>>],
+pub fn terminate(
-                 stack_pointers: &'static [eh_artiq::StackPointerBacktrace],
+    exceptions: &'static [Option<eh_artiq::Exception<'static>>],
-                 backtrace: &'static mut [(usize, usize)]) -> ! {
+    stack_pointers: &'static [eh_artiq::StackPointerBacktrace],
+    backtrace: &'static mut [(usize, usize)],
+) -> ! {
     {
         let core1_tx = unsafe { KERNEL_CHANNEL_1TO0.as_mut().unwrap() };
         let errors = unsafe { get_async_errors() };
@@ -215,26 +208,34 @@ pub fn terminate(exceptions: &'static [Option<eh_artiq::Exception<'static>>],
 
 /// Called by llvm_libunwind
 #[no_mangle]
-extern fn dl_unwind_find_exidx(pc: *const u32, len_ptr: *mut u32) -> *const u32 {
+extern "C" fn dl_unwind_find_exidx(pc: *const u32, len_ptr: *mut u32) -> *const u32 {
     let length;
     let start: *const EXIDX_Entry;
     unsafe {
         if &__text_start as *const u32 <= pc && pc < &__text_end as *const u32 {
             length = (&__exidx_end as *const EXIDX_Entry).offset_from(&__exidx_start) as u32;
             start = &__exidx_start;
-        } else {
+        } else if KERNEL_IMAGE != ptr::null() {
-            let exidx = KERNEL_IMAGE.as_ref()
+            let exidx = KERNEL_IMAGE
+                .as_ref()
                 .expect("dl_unwind_find_exidx kernel image")
-                .library.get().as_ref().unwrap().exidx();
+                .library
+                .get()
+                .as_ref()
+                .unwrap()
+                .exidx();
             length = exidx.len() as u32;
             start = exidx.as_ptr();
+        } else {
+            length = 0;
+            start = ptr::null();
         }
         *len_ptr = length;
     }
     start as *const u32
 }
 
-pub extern fn rtio_get_destination_status(destination: i32) -> bool {
+pub extern "C" fn rtio_get_destination_status(destination: i32) -> bool {
     #[cfg(has_drtio)]
     if destination > 0 && destination < 255 {
         let reply = unsafe {
@@ -245,7 +246,7 @@ pub extern fn rtio_get_destination_status(destination: i32) -> bool {
         };
         return match reply {
             Message::UpDestinationsReply(x) => x,
-            _ => panic!("received unexpected reply to UpDestinationsRequest: {:?}", reply)
+            _ => panic!("received unexpected reply to UpDestinationsRequest: {:?}", reply),
         };
     }
 

src/libksupport/src/kernel/dma.rs

@@ -0,0 +1,255 @@
+use alloc::{string::String, vec::Vec};
+use core::mem;
+
+use cslice::CSlice;
+
+use super::{Message, KERNEL_CHANNEL_0TO1, KERNEL_CHANNEL_1TO0, KERNEL_IMAGE};
+use crate::{artiq_raise, pl::csr, rtio};
+
+#[repr(C)]
+pub struct DmaTrace {
+    duration: i64,
+    address: i32,
+    uses_ddma: bool,
+}
+
+#[derive(Clone, Debug)]
+pub struct DmaRecorder {
+    pub name: String,
+    pub buffer: Vec<u8>,
+    pub duration: i64,
+    pub enable_ddma: bool,
+}
+
+static mut RECORDER: Option<DmaRecorder> = None;
+
+pub unsafe fn init_dma_recorder() {
+    // as static would remain after restart, we have to reset it,
+    // without running its destructor.
+    mem::forget(mem::replace(&mut RECORDER, None));
+}
+
+pub extern "C" fn dma_record_start(name: CSlice<u8>) {
+    let name = String::from_utf8(name.as_ref().to_vec()).unwrap();
+    unsafe {
+        KERNEL_CHANNEL_1TO0
+            .as_mut()
+            .unwrap()
+            .send(Message::DmaEraseRequest(name.clone()));
+    }
+    unsafe {
+        if RECORDER.is_some() {
+            artiq_raise!("DMAError", "DMA is already recording")
+        }
+
+        let library = KERNEL_IMAGE.as_ref().unwrap();
+        library.rebind(b"rtio_output", dma_record_output as *const ()).unwrap();
+        library
+            .rebind(b"rtio_output_wide", dma_record_output_wide as *const ())
+            .unwrap();
+
+        RECORDER = Some(DmaRecorder {
+            name,
+            buffer: Vec::new(),
+            duration: 0,
+            enable_ddma: false,
+        });
+    }
+}
+
+pub extern "C" fn dma_record_stop(duration: i64, enable_ddma: bool) {
+    unsafe {
+        if RECORDER.is_none() {
+            artiq_raise!("DMAError", "DMA is not recording")
+        }
+
+        let library = KERNEL_IMAGE.as_ref().unwrap();
+        library.rebind(b"rtio_output", rtio::output as *const ()).unwrap();
+        library
+            .rebind(b"rtio_output_wide", rtio::output_wide as *const ())
+            .unwrap();
+
+        let mut recorder = RECORDER.take().unwrap();
+        recorder.duration = duration;
+        recorder.enable_ddma = enable_ddma;
+        KERNEL_CHANNEL_1TO0
+            .as_mut()
+            .unwrap()
+            .send(Message::DmaPutRequest(recorder));
+    }
+}
+
+#[inline(always)]
+unsafe fn dma_record_output_prepare(timestamp: i64, target: i32, words: usize) {
+    // See gateware/rtio/dma.py.
+    const HEADER_LENGTH: usize = /*length*/1 + /*channel*/3 + /*timestamp*/8 + /*address*/1;
+    let length = HEADER_LENGTH + /*data*/words * 4;
+
+    let buffer = &mut RECORDER.as_mut().unwrap().buffer;
+    buffer.reserve(length);
+    buffer.extend_from_slice(&[
+        (length >> 0) as u8,
+        (target >> 8) as u8,
+        (target >> 16) as u8,
+        (target >> 24) as u8,
+        (timestamp >> 0) as u8,
+        (timestamp >> 8) as u8,
+        (timestamp >> 16) as u8,
+        (timestamp >> 24) as u8,
+        (timestamp >> 32) as u8,
+        (timestamp >> 40) as u8,
+        (timestamp >> 48) as u8,
+        (timestamp >> 56) as u8,
+        (target >> 0) as u8,
+    ]);
+}
+
+pub extern "C" fn dma_record_output(target: i32, word: i32) {
+    unsafe {
+        let timestamp = rtio::now_mu();
+        dma_record_output_prepare(timestamp, target, 1);
+        RECORDER.as_mut().unwrap().buffer.extend_from_slice(&[
+            (word >> 0) as u8,
+            (word >> 8) as u8,
+            (word >> 16) as u8,
+            (word >> 24) as u8,
+        ]);
+    }
+}
+
+pub extern "C" fn dma_record_output_wide(target: i32, words: &CSlice<i32>) {
+    assert!(words.len() <= 16); // enforce the hardware limit
+
+    unsafe {
+        let timestamp = rtio::now_mu();
+        dma_record_output_prepare(timestamp, target, words.len());
+        let buffer = &mut RECORDER.as_mut().unwrap().buffer;
+        for word in words.as_ref().iter() {
+            buffer.extend_from_slice(&[
+                (word >> 0) as u8,
+                (word >> 8) as u8,
+                (word >> 16) as u8,
+                (word >> 24) as u8,
+            ]);
+        }
+    }
+}
+
+pub extern "C" fn dma_erase(name: CSlice<u8>) {
+    let name = String::from_utf8(name.as_ref().to_vec()).unwrap();
+    unsafe {
+        KERNEL_CHANNEL_1TO0
+            .as_mut()
+            .unwrap()
+            .send(Message::DmaEraseRequest(name));
+    }
+}
+
+pub extern "C" fn dma_retrieve(name: CSlice<u8>) -> DmaTrace {
+    let name = String::from_utf8(name.as_ref().to_vec()).unwrap();
+    unsafe {
+        KERNEL_CHANNEL_1TO0.as_mut().unwrap().send(Message::DmaGetRequest(name));
+    }
+    match unsafe { KERNEL_CHANNEL_0TO1.as_mut().unwrap() }.recv() {
+        Message::DmaGetReply(None) => (),
+        Message::DmaGetReply(Some((address, duration, uses_ddma))) => {
+            return DmaTrace {
+                address,
+                duration,
+                uses_ddma,
+            };
+        }
+        _ => panic!("Expected DmaGetReply after DmaGetRequest!"),
+    }
+    // we have to defer raising error as we have to drop the message first...
+    artiq_raise!("DMAError", "DMA trace not found");
+}
+
+pub extern "C" fn dma_playback(timestamp: i64, ptr: i32, _uses_ddma: bool) {
+    unsafe {
+        csr::rtio_dma::base_address_write(ptr as u32);
+        csr::rtio_dma::time_offset_write(timestamp as u64);
+
+        let old_cri_master = csr::cri_con::selected_read();
+        csr::cri_con::selected_write(1);
+        csr::rtio_dma::enable_write(1);
+        #[cfg(has_drtio)]
+        if _uses_ddma {
+            KERNEL_CHANNEL_1TO0
+                .as_mut()
+                .unwrap()
+                .send(Message::DmaStartRemoteRequest {
+                    id: ptr,
+                    timestamp: timestamp,
+                });
+        }
+        while csr::rtio_dma::enable_read() != 0 {}
+        csr::cri_con::selected_write(old_cri_master);
+
+        let error = csr::rtio_dma::error_read();
+        if error != 0 {
+            let timestamp = csr::rtio_dma::error_timestamp_read();
+            let channel = csr::rtio_dma::error_channel_read();
+            csr::rtio_dma::error_write(1);
+            if error & 1 != 0 {
+                artiq_raise!(
+                    "RTIOUnderflow",
+                    "RTIO underflow at {1} mu, channel {rtio_channel_info:0}",
+                    channel as i64,
+                    timestamp as i64,
+                    0
+                );
+            }
+            if error & 2 != 0 {
+                artiq_raise!(
+                    "RTIODestinationUnreachable",
+                    "RTIO destination unreachable, output, at {1} mu, channel {rtio_channel_info:0}",
+                    channel as i64,
+                    timestamp as i64,
+                    0
+                );
+            }
+        }
+        #[cfg(has_drtio)]
+        if _uses_ddma {
+            KERNEL_CHANNEL_1TO0
+                .as_mut()
+                .unwrap()
+                .send(Message::DmaAwaitRemoteRequest(ptr));
+            match KERNEL_CHANNEL_0TO1.as_mut().unwrap().recv() {
+                Message::DmaAwaitRemoteReply {
+                    timeout,
+                    error,
+                    channel,
+                    timestamp,
+                } => {
+                    if timeout {
+                        artiq_raise!(
+                            "DMAError",
+                            "Error running DMA on satellite device, timed out waiting for results"
+                        );
+                    }
+                    if error & 1 != 0 {
+                        artiq_raise!(
+                            "RTIOUnderflow",
+                            "RTIO underflow at {1} mu, channel {rtio_channel_info:0}",
+                            channel as i64,
+                            timestamp as i64,
+                            0
+                        );
+                    }
+                    if error & 2 != 0 {
+                        artiq_raise!(
+                            "RTIODestinationUnreachable",
+                            "RTIO destination unreachable, output, at {1} mu, channel {rtio_channel_info:0}",
+                            channel as i64,
+                            timestamp as i64,
+                            0
+                        );
+                    }
+                }
+                _ => panic!("Expected DmaAwaitRemoteReply after DmaAwaitRemoteRequest!"),
+            }
+        }
+    }
+}

src/libksupport/src/kernel/mod.rs

@@ -0,0 +1,126 @@
+use alloc::{string::String, vec::Vec};
+use core::ptr;
+
+use libcortex_a9::{mutex::Mutex, semaphore::Semaphore, sync_channel};
+
+use crate::{eh_artiq, RPCException};
+
+mod control;
+pub use control::Control;
+mod api;
+pub mod core1;
+mod dma;
+mod rpc;
+pub use dma::DmaRecorder;
+mod cache;
+#[cfg(has_drtio)]
+mod subkernel;
+
+#[cfg(has_drtio)]
+#[derive(Debug, Clone)]
+pub enum SubkernelStatus {
+    NoError,
+    Timeout,
+    IncorrectState,
+    CommLost,
+    OtherError,
+}
+
+#[derive(Debug, Clone)]
+pub enum Message {
+    LoadRequest(Vec<u8>),
+    LoadCompleted,
+    LoadFailed,
+    StartRequest,
+    KernelFinished(u8),
+    KernelException(
+        &'static [Option<eh_artiq::Exception<'static>>],
+        &'static [eh_artiq::StackPointerBacktrace],
+        &'static [(usize, usize)],
+        u8,
+    ),
+    RpcSend {
+        is_async: bool,
+        data: Vec<u8>,
+    },
+    RpcRecvRequest(*mut ()),
+    RpcRecvReply(Result<usize, RPCException>),
+
+    CacheGetRequest(String),
+    CacheGetReply(Vec<i32>),
+    CachePutRequest(String, Vec<i32>),
+
+    DmaPutRequest(DmaRecorder),
+    DmaEraseRequest(String),
+    DmaGetRequest(String),
+    DmaGetReply(Option<(i32, i64, bool)>),
+    #[cfg(has_drtio)]
+    DmaStartRemoteRequest {
+        id: i32,
+        timestamp: i64,
+    },
+    #[cfg(has_drtio)]
+    DmaAwaitRemoteRequest(i32),
+    #[cfg(has_drtio)]
+    DmaAwaitRemoteReply {
+        timeout: bool,
+        error: u8,
+        channel: u32,
+        timestamp: u64,
+    },
+
+    #[cfg(has_drtio)]
+    UpDestinationsRequest(i32),
+    #[cfg(has_drtio)]
+    UpDestinationsReply(bool),
+
+    #[cfg(has_drtio)]
+    SubkernelLoadRunRequest {
+        id: u32,
+        destination: u8,
+        run: bool,
+    },
+    #[cfg(has_drtio)]
+    SubkernelLoadRunReply {
+        succeeded: bool,
+    },
+    #[cfg(has_drtio)]
+    SubkernelAwaitFinishRequest {
+        id: u32,
+        timeout: i64,
+    },
+    #[cfg(has_drtio)]
+    SubkernelAwaitFinishReply {
+        status: SubkernelStatus,
+    },
+    #[cfg(has_drtio)]
+    SubkernelMsgSend {
+        id: u32,
+        destination: Option<u8>,
+        data: Vec<u8>,
+    },
+    #[cfg(has_drtio)]
+    SubkernelMsgSent,
+    #[cfg(has_drtio)]
+    SubkernelMsgRecvRequest {
+        id: i32,
+        timeout: i64,
+        tags: Vec<u8>,
+    },
+    #[cfg(has_drtio)]
+    SubkernelMsgRecvReply {
+        status: SubkernelStatus,
+        count: u8,
+    },
+}
+
+static CHANNEL_0TO1: Mutex<Option<sync_channel::Sender<'static, Message>>> = Mutex::new(None);
+static CHANNEL_1TO0: Mutex<Option<sync_channel::Receiver<'static, Message>>> = Mutex::new(None);
+static CHANNEL_SEM: Semaphore = Semaphore::new(0, 1);
+
+static mut KERNEL_CHANNEL_0TO1: Option<sync_channel::Receiver<'static, Message>> = None;
+static mut KERNEL_CHANNEL_1TO0: Option<sync_channel::Sender<'static, Message>> = None;
+
+pub static mut KERNEL_IMAGE: *const core1::KernelImage = ptr::null();
+
+static INIT_LOCK: Mutex<()> = Mutex::new(());

src/libksupport/src/kernel/rpc.rs

@@ -1,31 +1,28 @@
 //! Kernel-side RPC API
 
 use alloc::vec::Vec;
+
 use cslice::CSlice;
 
-use crate::eh_artiq;
+use super::{Message, KERNEL_CHANNEL_0TO1, KERNEL_CHANNEL_1TO0};
-use crate::rpc::send_args;
+use crate::{eh_artiq, rpc::send_args};
-use super::{
-    KERNEL_CHANNEL_0TO1, KERNEL_CHANNEL_1TO0,
-    Message,
-};
 
 fn rpc_send_common(is_async: bool, service: u32, tag: &CSlice<u8>, data: *const *const ()) {
     let core1_tx = unsafe { KERNEL_CHANNEL_1TO0.as_mut().unwrap() };
     let mut buffer = Vec::<u8>::new();
-    send_args(&mut buffer, service, tag.as_ref(), data).expect("RPC encoding failed");
+    send_args(&mut buffer, service, tag.as_ref(), data, true).expect("RPC encoding failed");
     core1_tx.send(Message::RpcSend { is_async, data: buffer });
 }
 
-pub extern fn rpc_send(service: u32, tag: &CSlice<u8>, data: *const *const ()) {
+pub extern "C" fn rpc_send(service: u32, tag: &CSlice<u8>, data: *const *const ()) {
     rpc_send_common(false, service, tag, data);
 }
 
-pub extern fn rpc_send_async(service: u32, tag: &CSlice<u8>, data: *const *const ()) {
+pub extern "C" fn rpc_send_async(service: u32, tag: &CSlice<u8>, data: *const *const ()) {
     rpc_send_common(true, service, tag, data);
 }
 
-pub extern fn rpc_recv(slot: *mut ()) -> usize {
+pub extern "C" fn rpc_recv(slot: *mut ()) -> usize {
     let reply = unsafe {
         let core1_rx = KERNEL_CHANNEL_0TO1.as_mut().unwrap();
         let core1_tx = KERNEL_CHANNEL_1TO0.as_mut().unwrap();
@@ -36,15 +33,15 @@ pub extern fn rpc_recv(slot: *mut ()) -> usize {
         Message::RpcRecvReply(Ok(alloc_size)) => alloc_size,
         Message::RpcRecvReply(Err(exception)) => unsafe {
             eh_artiq::raise(&eh_artiq::Exception {
-                id:       exception.id,
+                id: exception.id,
-                file:     CSlice::new(exception.file as *const u8, usize::MAX),
+                file: CSlice::new(exception.file as *const u8, usize::MAX),
-                line:     exception.line as u32,
+                line: exception.line as u32,
-                column:   exception.column as u32,
+                column: exception.column as u32,
                 function: CSlice::new(exception.function as *const u8, usize::MAX),
-                message:  CSlice::new(exception.message as *const u8, usize::MAX),
+                message: CSlice::new(exception.message as *const u8, usize::MAX),
-                param:    exception.param
+                param: exception.param,
             })
         },
-        _ => panic!("received unexpected reply to RpcRecvRequest: {:?}", reply)
+        _ => panic!("received unexpected reply to RpcRecvRequest: {:?}", reply),
     }
 }

src/libksupport/src/kernel/subkernel.rs

@@ -0,0 +1,122 @@
+use alloc::vec::Vec;
+
+use cslice::CSlice;
+
+use super::{Message, SubkernelStatus, KERNEL_CHANNEL_0TO1, KERNEL_CHANNEL_1TO0};
+use crate::{artiq_raise, rpc::send_args};
+
+pub extern "C" fn load_run(id: u32, destination: u8, run: bool) {
+    unsafe {
+        KERNEL_CHANNEL_1TO0
+            .as_mut()
+            .unwrap()
+            .send(Message::SubkernelLoadRunRequest {
+                id: id,
+                destination: destination,
+                run: run,
+            });
+    }
+    match unsafe { KERNEL_CHANNEL_0TO1.as_mut().unwrap() }.recv() {
+        Message::SubkernelLoadRunReply { succeeded: true } => (),
+        Message::SubkernelLoadRunReply { succeeded: false } => {
+            artiq_raise!("SubkernelError", "Error loading or running the subkernel")
+        }
+        _ => panic!("Expected SubkernelLoadRunReply after SubkernelLoadRunRequest!"),
+    }
+}
+
+pub extern "C" fn await_finish(id: u32, timeout: i64) {
+    unsafe {
+        KERNEL_CHANNEL_1TO0
+            .as_mut()
+            .unwrap()
+            .send(Message::SubkernelAwaitFinishRequest {
+                id: id,
+                timeout: timeout,
+            });
+    }
+    match unsafe { KERNEL_CHANNEL_0TO1.as_mut().unwrap() }.recv() {
+        Message::SubkernelAwaitFinishReply {
+            status: SubkernelStatus::NoError,
+        } => (),
+        Message::SubkernelAwaitFinishReply {
+            status: SubkernelStatus::IncorrectState,
+        } => artiq_raise!("SubkernelError", "Subkernel not running"),
+        Message::SubkernelAwaitFinishReply {
+            status: SubkernelStatus::Timeout,
+        } => artiq_raise!("SubkernelError", "Subkernel timed out"),
+        Message::SubkernelAwaitFinishReply {
+            status: SubkernelStatus::CommLost,
+        } => artiq_raise!("SubkernelError", "Lost communication with satellite"),
+        Message::SubkernelAwaitFinishReply {
+            status: SubkernelStatus::OtherError,
+        } => artiq_raise!("SubkernelError", "An error occurred during subkernel operation"),
+        _ => panic!("expected SubkernelAwaitFinishReply after SubkernelAwaitFinishRequest"),
+    }
+}
+
+pub extern "C" fn send_message(
+    id: u32,
+    is_return: bool,
+    destination: u8,
+    count: u8,
+    tag: &CSlice<u8>,
+    data: *const *const (),
+) {
+    let mut buffer = Vec::<u8>::new();
+    send_args(&mut buffer, 0, tag.as_ref(), data, false).expect("RPC encoding failed");
+    // overwrite service tag, include how many tags are in the message
+    buffer[3] = count;
+    unsafe {
+        KERNEL_CHANNEL_1TO0.as_mut().unwrap().send(Message::SubkernelMsgSend {
+            id: id,
+            destination: if is_return { None } else { Some(destination) },
+            data: buffer[3..].to_vec(),
+        });
+    }
+    match unsafe { KERNEL_CHANNEL_0TO1.as_mut().unwrap() }.recv() {
+        Message::SubkernelMsgSent => (),
+        _ => panic!("expected SubkernelMsgSent after SubkernelMsgSend"),
+    }
+}
+
+pub extern "C" fn await_message(id: i32, timeout: i64, tags: &CSlice<u8>, min: u8, max: u8) {
+    unsafe {
+        KERNEL_CHANNEL_1TO0
+            .as_mut()
+            .unwrap()
+            .send(Message::SubkernelMsgRecvRequest {
+                id: id,
+                timeout: timeout,
+                tags: tags.as_ref().to_vec(),
+            });
+    }
+    match unsafe { KERNEL_CHANNEL_0TO1.as_mut().unwrap() }.recv() {
+        Message::SubkernelMsgRecvReply {
+            status: SubkernelStatus::NoError,
+            count,
+        } => {
+            if min > count || count > max {
+                artiq_raise!("SubkernelError", "Received more or less arguments than required")
+            }
+        }
+        Message::SubkernelMsgRecvReply {
+            status: SubkernelStatus::IncorrectState,
+            ..
+        } => artiq_raise!("SubkernelError", "Subkernel not running"),
+        Message::SubkernelMsgRecvReply {
+            status: SubkernelStatus::Timeout,
+            ..
+        } => artiq_raise!("SubkernelError", "Subkernel timed out"),
+        Message::SubkernelMsgRecvReply {
+            status: SubkernelStatus::CommLost,
+            ..
+        } => artiq_raise!("SubkernelError", "Lost communication with satellite"),
+        Message::SubkernelMsgRecvReply {
+            status: SubkernelStatus::OtherError,
+            ..
+        } => artiq_raise!("SubkernelError", "An error occurred during subkernel operation"),
+        _ => panic!("expected SubkernelMsgRecvReply after SubkernelMsgRecvRequest"),
+    }
+    // RpcRecvRequest should be called after this to receive message data
+}

src/libksupport/src/lib.rs

@@ -0,0 +1,163 @@
+#![no_std]
+#![feature(c_variadic)]
+#![feature(const_btree_new)]
+#![feature(const_in_array_repeat_expressions)]
+#![feature(naked_functions)]
+#![feature(asm)]
+
+#[macro_use]
+extern crate alloc;
+
+use alloc::{collections::BTreeMap, string::String};
+
+use io::{Cursor, ProtoRead};
+use libasync::block_async;
+use libconfig::Config;
+use log::{error, warn};
+#[cfg(has_drtiosat)]
+pub use pl::csr::drtiosat as rtio_core;
+#[cfg(has_rtio_core)]
+pub use pl::csr::rtio_core;
+use void::Void;
+
+pub mod eh_artiq;
+pub mod i2c;
+pub mod irq;
+pub mod kernel;
+pub mod rpc;
+#[cfg(ki_impl = "csr")]
+#[path = "rtio_csr.rs"]
+pub mod rtio;
+#[cfg(ki_impl = "acp")]
+#[path = "rtio_acp.rs"]
+pub mod rtio;
+#[rustfmt::skip]
+#[path = "../../../build/pl.rs"]
+pub mod pl;
+
+
+#[derive(Debug, Clone)]
+pub struct RPCException {
+    pub id: u32,
+    pub message: u32,
+    pub param: [i64; 3],
+    pub file: u32,
+    pub line: i32,
+    pub column: i32,
+    pub function: u32,
+}
+
+pub static mut SEEN_ASYNC_ERRORS: u8 = 0;
+
+pub const ASYNC_ERROR_COLLISION: u8 = 1 << 0;
+pub const ASYNC_ERROR_BUSY: u8 = 1 << 1;
+pub const ASYNC_ERROR_SEQUENCE_ERROR: u8 = 1 << 2;
+
+pub unsafe fn get_async_errors() -> u8 {
+    let errors = SEEN_ASYNC_ERRORS;
+    SEEN_ASYNC_ERRORS = 0;
+    errors
+}
+
+fn wait_for_async_rtio_error() -> nb::Result<(), Void> {
+    unsafe {
+        #[cfg(has_rtio_core)]
+        let errors = rtio_core::async_error_read();
+        #[cfg(has_drtiosat)]
+        let errors = rtio_core::protocol_error_read();
+        if errors != 0 {
+            Ok(())
+        } else {
+            Err(nb::Error::WouldBlock)
+        }
+    }
+}
+
+pub async fn report_async_rtio_errors() {
+    loop {
+        let _ = block_async!(wait_for_async_rtio_error()).await;
+        unsafe {
+            #[cfg(has_rtio_core)]
+            let errors = rtio_core::async_error_read();
+            #[cfg(has_drtiosat)]
+            let errors = rtio_core::protocol_error_read();
+            if errors & ASYNC_ERROR_COLLISION != 0 {
+                let channel = rtio_core::collision_channel_read();
+                error!(
+                    "RTIO collision involving channel 0x{:04x}:{}",
+                    channel,
+                    resolve_channel_name(channel as u32)
+                );
+            }
+            if errors & ASYNC_ERROR_BUSY != 0 {
+                let channel = rtio_core::busy_channel_read();
+                error!(
+                    "RTIO busy error involving channel 0x{:04x}:{}",
+                    channel,
+                    resolve_channel_name(channel as u32)
+                );
+            }
+            if errors & ASYNC_ERROR_SEQUENCE_ERROR != 0 {
+                let channel = rtio_core::sequence_error_channel_read();
+                error!(
+                    "RTIO sequence error involving channel 0x{:04x}:{}",
+                    channel,
+                    resolve_channel_name(channel as u32)
+                );
+            }
+            SEEN_ASYNC_ERRORS = errors;
+            #[cfg(has_rtio_core)]
+            rtio_core::async_error_write(errors);
+            #[cfg(has_drtiosat)]
+            rtio_core::protocol_error_write(errors);
+        }
+    }
+}
+
+static mut RTIO_DEVICE_MAP: BTreeMap<u32, String> = BTreeMap::new();
+
+fn read_device_map(cfg: &Config) -> BTreeMap<u32, String> {
+    let mut device_map: BTreeMap<u32, String> = BTreeMap::new();
+    let _ = cfg
+        .read("device_map")
+        .and_then(|raw_bytes| {
+            let mut bytes_cr = Cursor::new(raw_bytes);
+            let size = bytes_cr.read_u32().unwrap();
+            for _ in 0..size {
+                let channel = bytes_cr.read_u32().unwrap();
+                let device_name = bytes_cr.read_string().unwrap();
+                if let Some(old_entry) = device_map.insert(channel, device_name.clone()) {
+                    warn!(
+                        "conflicting device map entries for RTIO channel {}: '{}' and '{}'",
+                        channel, old_entry, device_name
+                    );
+                }
+            }
+            Ok(())
+        })
+        .or_else(|err| {
+            warn!(
+                "error reading device map ({}), device names will not be available in RTIO error messages",
+                err
+            );
+            Err(err)
+        });
+    device_map
+}
+
+fn _resolve_channel_name(channel: u32, device_map: &BTreeMap<u32, String>) -> String {
+    match device_map.get(&channel) {
+        Some(val) => val.clone(),
+        None => String::from("unknown"),
+    }
+}
+
+pub fn resolve_channel_name(channel: u32) -> String {
+    _resolve_channel_name(channel, unsafe { &RTIO_DEVICE_MAP })
+}
+
+pub fn setup_device_map(cfg: &Config) {
+    unsafe {
+        RTIO_DEVICE_MAP = read_device_map(cfg);
+    }
+}

src/libksupport/src/rpc.rs

@@ -1,135 +1,116 @@
 use core::str;
-use core::future::Future;
+
-use cslice::{CSlice, CMutSlice};
+use byteorder::{ByteOrder, NativeEndian};
+use core_io::{Error, Read, Write};
+use cslice::{CMutSlice, CSlice};
+use io::{ProtoRead, ProtoWrite};
 use log::trace;
-use byteorder::{NativeEndian, ByteOrder};
 
-use core_io::{Write, Error};
+use self::tag::{split_tag, Tag, TagIterator};
-use libboard_zynq::smoltcp;
-use libasync::smoltcp::TcpStream;
-use alloc::boxed::Box;
-use async_recursion::async_recursion;
-
-use io::proto::ProtoWrite;
-use crate::proto_async;
-use self::tag::{Tag, TagIterator, split_tag};
 
 #[inline]
-fn round_up(val: usize, power_of_two: usize) -> usize {
+pub fn round_up(val: usize, power_of_two: usize) -> usize {
     assert!(power_of_two.is_power_of_two());
     let max_rem = power_of_two - 1;
     (val + max_rem) & (!max_rem)
 }
 
 #[inline]
-unsafe fn round_up_mut<T>(ptr: *mut T, power_of_two: usize) -> *mut T {
+pub unsafe fn round_up_mut<T>(ptr: *mut T, power_of_two: usize) -> *mut T {
     round_up(ptr as usize, power_of_two) as *mut T
 }
 
 #[inline]
-unsafe fn round_up_const<T>(ptr: *const T, power_of_two: usize) -> *const T {
+pub unsafe fn round_up_const<T>(ptr: *const T, power_of_two: usize) -> *const T {
     round_up(ptr as usize, power_of_two) as *const T
 }
 
 #[inline]
-unsafe fn align_ptr<T>(ptr: *const ()) -> *const T {
+pub unsafe fn align_ptr<T>(ptr: *const ()) -> *const T {
     round_up_const(ptr, core::mem::align_of::<T>()) as *const T
 }
 
 #[inline]
-unsafe fn align_ptr_mut<T>(ptr: *mut ()) -> *mut T {
+pub unsafe fn align_ptr_mut<T>(ptr: *mut ()) -> *mut T {
     round_up_mut(ptr, core::mem::align_of::<T>()) as *mut T
 }
 
-/// Reads (deserializes) `length` array or list elements of type `tag` from `stream`,
+// versions for reader rather than TcpStream
-/// writing them into the buffer given by `storage`.
+// they will be made into sync for satellite subkernels later
-///
+unsafe fn recv_elements<F, R>(
-/// `alloc` is used for nested allocations (if elements themselves contain
+    reader: &mut R,
-/// lists/arrays), see [recv_value].
+    elt_tag: Tag,
-#[async_recursion(?Send)]
-async unsafe fn recv_elements<F>(
-    stream: &TcpStream,
-    elt_tag: Tag<'async_recursion>,
     length: usize,
     storage: *mut (),
-    alloc: &(impl Fn(usize) -> F + 'async_recursion)
+    alloc: &mut F,
-) -> Result<(), smoltcp::Error>
+) -> Result<(), Error>
 where
-    F: Future<Output=*mut ()>,
+    F: FnMut(usize) -> *mut (),
+    R: Read + ?Sized,
 {
-    // List of simple types are special-cased in the protocol for performance.
     match elt_tag {
         Tag::Bool => {
             let dest = core::slice::from_raw_parts_mut(storage as *mut u8, length);
-            proto_async::read_chunk(stream, dest).await?;
+            reader.read_exact(dest)?;
-        },
+        }
         Tag::Int32 => {
             let ptr = storage as *mut u32;
             let dest = core::slice::from_raw_parts_mut(ptr as *mut u8, length * 4);
-            proto_async::read_chunk(stream, dest).await?;
+            reader.read_exact(dest)?;
             drop(dest);
             let dest = core::slice::from_raw_parts_mut(ptr, length);
             NativeEndian::from_slice_u32(dest);
-        },
+        }
         Tag::Int64 | Tag::Float64 => {
             let ptr = storage as *mut u64;
             let dest = core::slice::from_raw_parts_mut(ptr as *mut u8, length * 8);
-            proto_async::read_chunk(stream, dest).await?;
+            reader.read_exact(dest)?;
             drop(dest);
             let dest = core::slice::from_raw_parts_mut(ptr, length);
             NativeEndian::from_slice_u64(dest);
-        },
+        }
         _ => {
             let mut data = storage;
             for _ in 0..length {
-                recv_value(stream, elt_tag, &mut data, alloc).await?
+                recv_value(reader, elt_tag, &mut data, alloc)?
             }
         }
     }
     Ok(())
 }
 
-/// Reads (deserializes) a value of type `tag` from `stream`, writing the results to
+unsafe fn recv_value<F, R>(reader: &mut R, tag: Tag, data: &mut *mut (), alloc: &mut F) -> Result<(), Error>
-/// the kernel-side buffer `data` (the passed pointer to which is incremented to point
+where
-/// past the just-received data). For nested allocations (lists/arrays), `alloc` is
+    F: FnMut(usize) -> *mut (),
-/// invoked any number of times with the size of the required allocation as a parameter
+    R: Read + ?Sized,
-/// (which is assumed to be correctly aligned for all payload types).
-#[async_recursion(?Send)]
-async unsafe fn recv_value<F>(stream: &TcpStream, tag: Tag<'async_recursion>, data: &mut *mut (),
-                              alloc: &(impl Fn(usize) -> F + 'async_recursion))
-                             -> Result<(), smoltcp::Error>
-                          where F: Future<Output=*mut ()>
 {
     macro_rules! consume_value {
-        ($ty:ty, |$ptr:ident| $map:expr) => ({
+        ($ty:ty, | $ptr:ident | $map:expr) => {{
             let $ptr = align_ptr_mut::<$ty>(*data);
             *data = $ptr.offset(1) as *mut ();
             $map
-        })
+        }};
     }
 
     match tag {
         Tag::None => Ok(()),
-        Tag::Bool =>
+        Tag::Bool => consume_value!(i8, |ptr| {
-            consume_value!(i8, |ptr| {
+            *ptr = reader.read_u8()? as i8;
-                *ptr = proto_async::read_i8(stream).await?;
+            Ok(())
-                Ok(())
+        }),
-            }),
+        Tag::Int32 => consume_value!(i32, |ptr| {
-        Tag::Int32 =>
+            *ptr = reader.read_u32()? as i32;
-            consume_value!(i32, |ptr| {
+            Ok(())
-                *ptr = proto_async::read_i32(stream).await?;
+        }),
-                Ok(())
+        Tag::Int64 | Tag::Float64 => consume_value!(i64, |ptr| {
-            }),
+            *ptr = reader.read_u64()? as i64;
-        Tag::Int64 | Tag::Float64 =>
+            Ok(())
-            consume_value!(i64, |ptr| {
+        }),
-                *ptr = proto_async::read_i64(stream).await?;
-                Ok(())
-            }),
         Tag::String | Tag::Bytes | Tag::ByteArray => {
             consume_value!(CMutSlice<u8>, |ptr| {
-                let length = proto_async::read_i32(stream).await? as usize;
+                let length = reader.read_u32()? as usize;
-                *ptr = CMutSlice::new(alloc(length).await as *mut u8, length);
+                *ptr = CMutSlice::new(alloc(length) as *mut u8, length);
-                proto_async::read_chunk(stream, (*ptr).as_mut()).await?;
+                reader.read_exact((*ptr).as_mut())?;
                 Ok(())
             })
         }
@@ -139,169 +120,173 @@ async unsafe fn recv_value<F>(stream: &TcpStream, tag: Tag<'async_recursion>, da
             let mut it = it.clone();
             for _ in 0..arity {
                 let tag = it.next().expect("truncated tag");
-                recv_value(stream, tag, data, alloc).await?
+                recv_value(reader, tag, data, alloc)?
             }
-            // Take into account any tail padding (if element(s) with largest alignment
-            // are not at the end).
             *data = round_up_mut(*data, alignment);
             Ok(())
         }
         Tag::List(it) => {
             #[repr(C)]
-            struct List { elements: *mut (), length: usize }
+            struct List {
+                elements: *mut (),
+                length: usize,
+            }
             consume_value!(*mut List, |ptr_to_list| {
                 let tag = it.clone().next().expect("truncated tag");
-                let length = proto_async::read_i32(stream).await? as usize;
+                let length = reader.read_u32()? as usize;
 
-                // To avoid multiple kernel CPU roundtrips, use a single allocation for
-                // both the pointer/length List (slice) and the backing storage for the
-                // elements. We can assume that alloc() is aligned suitably, so just
-                // need to take into account any extra padding required.
-                // (Note: At the time of writing, there will never actually be any types
-                // with alignment larger than 8 bytes, so storage_offset == 0 always.)
                 let list_size = 4 + 4;
                 let storage_offset = round_up(list_size, tag.alignment());
                 let storage_size = tag.size() * length;
 
-                let allocation = alloc(storage_offset + storage_size).await as *mut u8;
+                let allocation = alloc(storage_offset + storage_size) as *mut u8;
                 *ptr_to_list = allocation as *mut List;
                 let storage = allocation.offset(storage_offset as isize) as *mut ();
 
                 (**ptr_to_list).length = length;
                 (**ptr_to_list).elements = storage;
-                recv_elements(stream, tag, length, storage, alloc).await
+                recv_elements(reader, tag, length, storage, alloc)
             })
         }
         Tag::Array(it, num_dims) => {
             consume_value!(*mut (), |buffer| {
-                // Deserialize length along each dimension and compute total number of
-                // elements.
                 let mut total_len: usize = 1;
                 for _ in 0..num_dims {
-                    let len = proto_async::read_i32(stream).await? as usize;
+                    let len = reader.read_u32()? as usize;
                     total_len *= len;
-                    consume_value!(usize, |ptr| *ptr = len )
+                    consume_value!(usize, |ptr| *ptr = len)
                 }
 
-                // Allocate backing storage for elements; deserialize them.
                 let elt_tag = it.clone().next().expect("truncated tag");
-                *buffer = alloc(elt_tag.size() * total_len).await;
+                *buffer = alloc(elt_tag.size() * total_len);
-                recv_elements(stream, elt_tag, total_len, *buffer, alloc).await
+                recv_elements(reader, elt_tag, total_len, *buffer, alloc)
             })
         }
         Tag::Range(it) => {
             *data = round_up_mut(*data, tag.alignment());
             let tag = it.clone().next().expect("truncated tag");
-            recv_value(stream, tag, data, alloc).await?;
+            recv_value(reader, tag, data, alloc)?;
-            recv_value(stream, tag, data, alloc).await?;
+            recv_value(reader, tag, data, alloc)?;
-            recv_value(stream, tag, data, alloc).await?;
+            recv_value(reader, tag, data, alloc)?;
             Ok(())
         }
         Tag::Keyword(_) => unreachable!(),
-        Tag::Object => unreachable!()
+        Tag::Object => unreachable!(),
     }
 }
 
-pub async fn recv_return<F>(stream: &TcpStream, tag_bytes: &[u8], data: *mut (),
+pub fn recv_return<'a, F, R>(
-                            alloc: &impl Fn(usize) -> F)
+    reader: &mut R,
-                           -> Result<(), smoltcp::Error>
+    tag_bytes: &'a [u8],
-                        where F: Future<Output=*mut ()>
+    data: *mut (),
+    alloc: &mut F,
+) -> Result<&'a [u8], Error>
+where
+    F: FnMut(usize) -> *mut (),
+    R: Read + ?Sized,
 {
     let mut it = TagIterator::new(tag_bytes);
     trace!("recv ...->{}", it);
 
     let tag = it.next().expect("truncated tag");
     let mut data = data;
-    unsafe { recv_value(stream, tag, &mut data, alloc).await? };
+    unsafe { recv_value(reader, tag, &mut data, alloc)? };
 
-    Ok(())
+    Ok(it.data)
 }
 
-unsafe fn send_elements<W>(writer: &mut W, elt_tag: Tag, length: usize, data: *const ())
+unsafe fn send_elements<W>(
-                          -> Result<(), Error>
+    writer: &mut W,
-    where W: Write + ?Sized
+    elt_tag: Tag,
+    length: usize,
+    data: *const (),
+    write_tags: bool,
+) -> Result<(), Error>
+where
+    W: Write + ?Sized,
 {
-    writer.write_u8(elt_tag.as_u8())?;
+    if write_tags {
+        writer.write_u8(elt_tag.as_u8())?;
+    }
     match elt_tag {
         // we cannot use NativeEndian::from_slice_i32 as the data is not mutable,
         // and that is not needed as the data is already in native endian
         Tag::Bool => {
             let slice = core::slice::from_raw_parts(data as *const u8, length);
             writer.write_all(slice)?;
-        },
+        }
         Tag::Int32 => {
             let slice = core::slice::from_raw_parts(data as *const u8, length * 4);
             writer.write_all(slice)?;
-        },
+        }
         Tag::Int64 | Tag::Float64 => {
             let slice = core::slice::from_raw_parts(data as *const u8, length * 8);
             writer.write_all(slice)?;
-        },
+        }
         _ => {
             let mut data = data;
             for _ in 0..length {
-                send_value(writer, elt_tag, &mut data)?;
+                send_value(writer, elt_tag, &mut data, write_tags)?;
             }
         }
     }
     Ok(())
 }
 
-unsafe fn send_value<W>(writer: &mut W, tag: Tag, data: &mut *const ())
+unsafe fn send_value<W>(writer: &mut W, tag: Tag, data: &mut *const (), write_tags: bool) -> Result<(), Error>
-                       -> Result<(), Error>
+where W: Write + ?Sized {
-    where W: Write + ?Sized
-{
     macro_rules! consume_value {
-        ($ty:ty, |$ptr:ident| $map:expr) => ({
+        ($ty:ty, | $ptr:ident | $map:expr) => {{
             let $ptr = align_ptr::<$ty>(*data);
             *data = $ptr.offset(1) as *const ();
             $map
-        })
+        }};
     }
 
-    writer.write_u8(tag.as_u8())?;
+    if write_tags {
+        writer.write_u8(tag.as_u8())?;
+    }
     match tag {
         Tag::None => Ok(()),
-        Tag::Bool =>
+        Tag::Bool => consume_value!(u8, |ptr| writer.write_u8(*ptr)),
-            consume_value!(u8, |ptr|
+        Tag::Int32 => consume_value!(u32, |ptr| writer.write_u32(*ptr)),
-                writer.write_u8(*ptr)),
+        Tag::Int64 | Tag::Float64 => consume_value!(u64, |ptr| writer.write_u64(*ptr)),
-        Tag::Int32 =>
+        Tag::String => consume_value!(CSlice<u8>, |ptr| {
-            consume_value!(u32, |ptr|
+            writer.write_string(str::from_utf8((*ptr).as_ref()).unwrap())
-                writer.write_u32(*ptr)),
+        }),
-        Tag::Int64 | Tag::Float64 =>
+        Tag::Bytes | Tag::ByteArray => consume_value!(CSlice<u8>, |ptr| writer.write_bytes((*ptr).as_ref())),
-            consume_value!(u64, |ptr|
-                writer.write_u64(*ptr)),
-        Tag::String =>
-            consume_value!(CSlice<u8>, |ptr|
-                writer.write_string(str::from_utf8((*ptr).as_ref()).unwrap())),
-        Tag::Bytes | Tag::ByteArray =>
-            consume_value!(CSlice<u8>, |ptr|
-                writer.write_bytes((*ptr).as_ref())),
         Tag::Tuple(it, arity) => {
             let mut it = it.clone();
-            writer.write_u8(arity)?;
+            if write_tags {
+                writer.write_u8(arity)?;
+            }
             let mut max_alignment = 0;
             for _ in 0..arity {
                 let tag = it.next().expect("truncated tag");
                 max_alignment = core::cmp::max(max_alignment, tag.alignment());
-                send_value(writer, tag, data)?
+                send_value(writer, tag, data, write_tags)?
             }
             *data = round_up_const(*data, max_alignment);
             Ok(())
         }
         Tag::List(it) => {
             #[repr(C)]
-            struct List { elements: *const (), length: u32 }
+            struct List {
+                elements: *const (),
+                length: u32,
+            }
             consume_value!(&List, |ptr| {
                 let length = (**ptr).length as usize;
                 writer.write_u32((*ptr).length)?;
                 let tag = it.clone().next().expect("truncated tag");
-                send_elements(writer, tag, length, (**ptr).elements)
+                send_elements(writer, tag, length, (**ptr).elements, write_tags)
             })
         }
         Tag::Array(it, num_dims) => {
-            writer.write_u8(num_dims)?;
+            if write_tags {
-            consume_value!(*const(), |buffer| {
+                writer.write_u8(num_dims)?;
+            }
+            consume_value!(*const (), |buffer| {
                 let elt_tag = it.clone().next().expect("truncated tag");
 
                 let mut total_len = 1;
@@ -312,40 +297,49 @@ unsafe fn send_value<W>(writer: &mut W, tag: Tag, data: &mut *const ())
                     })
                 }
                 let length = total_len as usize;
-                send_elements(writer, elt_tag, length, *buffer)
+                send_elements(writer, elt_tag, length, *buffer, write_tags)
             })
         }
         Tag::Range(it) => {
             let tag = it.clone().next().expect("truncated tag");
-            send_value(writer, tag, data)?;
+            send_value(writer, tag, data, write_tags)?;
-            send_value(writer, tag, data)?;
+            send_value(writer, tag, data, write_tags)?;
-            send_value(writer, tag, data)?;
+            send_value(writer, tag, data, write_tags)?;
             Ok(())
         }
         Tag::Keyword(it) => {
             #[repr(C)]
-            struct Keyword<'a> { name: CSlice<'a, u8> }
+            struct Keyword<'a> {
+                name: CSlice<'a, u8>,
+            }
             consume_value!(Keyword, |ptr| {
                 writer.write_string(str::from_utf8((*ptr).name.as_ref()).unwrap())?;
                 let tag = it.clone().next().expect("truncated tag");
                 let mut data = ptr.offset(1) as *const ();
-                send_value(writer, tag, &mut data)
+                send_value(writer, tag, &mut data, write_tags)
             })
             // Tag::Keyword never appears in composite types, so we don't have
             // to accurately advance data.
         }
         Tag::Object => {
             #[repr(C)]
-            struct Object { id: u32 }
+            struct Object {
-            consume_value!(*const Object, |ptr|
+                id: u32,
-                writer.write_u32((**ptr).id))
+            }
+            consume_value!(*const Object, |ptr| writer.write_u32((**ptr).id))
         }
     }
 }
 
-pub fn send_args<W>(writer: &mut W, service: u32, tag_bytes: &[u8], data: *const *const ())
+pub fn send_args<W>(
-                   -> Result<(), Error>
+    writer: &mut W,
-    where W: Write + ?Sized
+    service: u32,
+    tag_bytes: &[u8],
+    data: *const *const (),
+    write_tags: bool,
+) -> Result<(), Error>
+where
+    W: Write + ?Sized,
 {
     let (arg_tags_bytes, return_tag_bytes) = split_tag(tag_bytes);
 
@@ -357,9 +351,9 @@ pub fn send_args<W>(writer: &mut W, service: u32, tag_bytes: &[u8], data: *const
     for index in 0.. {
         if let Some(arg_tag) = args_it.next() {
             let mut data = unsafe { *data.offset(index) };
-            unsafe { send_value(writer, arg_tag, &mut data)? };
+            unsafe { send_value(writer, arg_tag, &mut data, write_tags)? };
         } else {
-            break
+            break;
         }
     }
     writer.write_u8(0)?;
@@ -368,14 +362,14 @@ pub fn send_args<W>(writer: &mut W, service: u32, tag_bytes: &[u8], data: *const
     Ok(())
 }
 
-mod tag {
+pub mod tag {
     use core::fmt;
 
     pub fn split_tag(tag_bytes: &[u8]) -> (&[u8], &[u8]) {
-        let tag_separator =
+        let tag_separator = tag_bytes
-            tag_bytes.iter()
+            .iter()
-                     .position(|&b| b == b':')
+            .position(|&b| b == b':')
-                     .expect("tag without a return separator");
+            .expect("tag without a return separator");
         let (arg_tags_bytes, rest) = tag_bytes.split_at(tag_separator);
         let return_tag_bytes = &rest[1..];
         (arg_tags_bytes, return_tag_bytes)
@@ -396,7 +390,7 @@ mod tag {
         Array(TagIterator<'a>, u8),
         Range(TagIterator<'a>),
         Keyword(TagIterator<'a>),
-        Object
+        Object,
     }
 
     impl<'a> Tag<'a> {
@@ -431,14 +425,15 @@ mod tag {
                 Tag::Tuple(it, arity) => {
                     let it = it.clone();
                     it.take(arity.into()).map(|t| t.alignment()).max().unwrap()
-                },
+                }
                 Tag::Range(it) => {
                     let it = it.clone();
                     it.take(3).map(|t| t.alignment()).max().unwrap()
                 }
                 // the ptr/length(s) pair is basically CSlice
-                Tag::Bytes | Tag::String | Tag::ByteArray | Tag::List(_) | Tag::Array(_, _) =>
+                Tag::Bytes | Tag::String | Tag::ByteArray | Tag::List(_) | Tag::Array(_, _) => {
-                    core::mem::align_of::<CSlice<()>>(),
+                    core::mem::align_of::<CSlice<()>>()
+                }
                 Tag::Keyword(_) => unreachable!("Tag::Keyword should not appear in composite types"),
                 Tag::Object => core::mem::align_of::<u32>(),
             }
@@ -484,7 +479,7 @@ mod tag {
 
     #[derive(Debug, Clone, Copy)]
     pub struct TagIterator<'a> {
-        data: &'a [u8]
+        pub data: &'a [u8],
     }
 
     impl<'a> TagIterator<'a> {
@@ -492,13 +487,14 @@ mod tag {
             TagIterator { data }
         }
 
-
         fn sub(&mut self, count: u8) -> TagIterator<'a> {
             let data = self.data;
             for _ in 0..count {
                 self.next().expect("truncated tag");
             }
-            TagIterator { data: &data[..(data.len() - self.data.len())] }
+            TagIterator {
+                data: &data[..(data.len() - self.data.len())],
+            }
         }
     }
 
@@ -507,7 +503,7 @@ mod tag {
 
         fn next(&mut self) -> Option<Tag<'a>> {
             if self.data.len() == 0 {
-                return None
+                return None;
             }
 
             let tag_byte = self.data[0];
@@ -535,7 +531,7 @@ mod tag {
                 b'r' => Tag::Range(self.sub(1)),
                 b'k' => Tag::Keyword(self.sub(1)),
                 b'O' => Tag::Object,
-                _    => unreachable!()
+                _ => unreachable!(),
             })
         }
     }
@@ -552,22 +548,14 @@ mod tag {
                 }
 
                 match tag {
-                    Tag::None =>
+                    Tag::None => write!(f, "None")?,
-                        write!(f, "None")?,
+                    Tag::Bool => write!(f, "Bool")?,
-                    Tag::Bool =>
+                    Tag::Int32 => write!(f, "Int32")?,
-                        write!(f, "Bool")?,
+                    Tag::Int64 => write!(f, "Int64")?,
-                    Tag::Int32 =>
+                    Tag::Float64 => write!(f, "Float64")?,
-                        write!(f, "Int32")?,
+                    Tag::String => write!(f, "String")?,
-                    Tag::Int64 =>
+                    Tag::Bytes => write!(f, "Bytes")?,
-                        write!(f, "Int64")?,
+                    Tag::ByteArray => write!(f, "ByteArray")?,
-                    Tag::Float64 =>
-                        write!(f, "Float64")?,
-                    Tag::String =>
-                        write!(f, "String")?,
-                    Tag::Bytes =>
-                        write!(f, "Bytes")?,
-                    Tag::ByteArray =>
-                        write!(f, "ByteArray")?,
                     Tag::Tuple(it, _) => {
                         write!(f, "Tuple(")?;
                         it.fmt(f)?;
@@ -593,8 +581,7 @@ mod tag {
                         it.fmt(f)?;
                         write!(f, ")")?;
                     }
-                    Tag::Object =>
+                    Tag::Object => write!(f, "Object")?,
-                        write!(f, "Object")?,
                 }
             }
 

src/libksupport/src/rtio_acp.rs

@@ -1,19 +1,19 @@
-use cslice::CSlice;
-use vcell::VolatileCell;
-use libcortex_a9::asm;
-use crate::artiq_raise;
 use core::sync::atomic::{fence, Ordering};
 
-use crate::pl::csr;
+use cslice::CSlice;
+use libcortex_a9::asm;
+use vcell::VolatileCell;
 
-pub const RTIO_O_STATUS_WAIT:                      i32 = 1;
+use crate::{artiq_raise, pl::csr, resolve_channel_name, rtio_core};
-pub const RTIO_O_STATUS_UNDERFLOW:                 i32 = 2;
+
-pub const RTIO_O_STATUS_DESTINATION_UNREACHABLE:   i32 = 4;
+pub const RTIO_O_STATUS_WAIT: i32 = 1;
-pub const RTIO_I_STATUS_WAIT_EVENT:                i32 = 1;
+pub const RTIO_O_STATUS_UNDERFLOW: i32 = 2;
-pub const RTIO_I_STATUS_OVERFLOW:                  i32 = 2;
+pub const RTIO_O_STATUS_DESTINATION_UNREACHABLE: i32 = 4;
+pub const RTIO_I_STATUS_WAIT_EVENT: i32 = 1;
+pub const RTIO_I_STATUS_OVERFLOW: i32 = 2;
 #[allow(unused)]
-pub const RTIO_I_STATUS_WAIT_STATUS:               i32 = 4; // TODO
+pub const RTIO_I_STATUS_WAIT_STATUS: i32 = 4; // TODO
-pub const RTIO_I_STATUS_DESTINATION_UNREACHABLE:   i32 = 8;
+pub const RTIO_I_STATUS_DESTINATION_UNREACHABLE: i32 = 8;
 
 #[repr(C)]
 pub struct TimestampedData {
@@ -47,18 +47,18 @@ static mut TRANSACTION_BUFFER: Transaction = Transaction {
     reply_timestamp: VolatileCell::new(0),
     padding0: [0; 2],
     padding1: [0; 2],
-    padding2: [0; 2]
+    padding2: [0; 2],
 };
 
-pub extern fn init() {
+pub extern "C" fn init() {
     unsafe {
-        csr::rtio_core::reset_write(1);
+        rtio_core::reset_write(1);
         csr::rtio::engine_addr_base_write(&TRANSACTION_BUFFER as *const Transaction as u32);
         csr::rtio::enable_write(1);
     }
 }
 
-pub extern fn get_counter() -> i64 {
+pub extern "C" fn get_counter() -> i64 {
     unsafe {
         csr::rtio::counter_update_write(1);
         csr::rtio::counter_read() as i64
@@ -67,15 +67,15 @@ pub extern fn get_counter() -> i64 {
 
 static mut NOW: i64 = 0;
 
-pub extern fn now_mu() -> i64 {
+pub extern "C" fn now_mu() -> i64 {
     unsafe { NOW }
 }
 
-pub extern fn at_mu(t: i64) {
+pub extern "C" fn at_mu(t: i64) {
     unsafe { NOW = t }
 }
 
-pub extern fn delay_mu(dt: i64) {
+pub extern "C" fn delay_mu(dt: i64) {
     unsafe { NOW += dt }
 }
 
@@ -87,18 +87,34 @@ unsafe fn process_exceptional_status(channel: i32, status: i32) {
         while csr::rtio::o_status_read() as i32 & RTIO_O_STATUS_WAIT != 0 {}
     }
     if status & RTIO_O_STATUS_UNDERFLOW != 0 {
-        artiq_raise!("RTIOUnderflow",
+        artiq_raise!(
-            "RTIO underflow at {0} mu, channel {1}, slack {2} mu",
+            "RTIOUnderflow",
-            timestamp, channel as i64, timestamp - get_counter());
+            format!(
+                "RTIO underflow at {{1}} mu, channel 0x{:04x}:{}, slack {{2}} mu",
+                channel,
+                resolve_channel_name(channel as u32)
+            ),
+            channel as i64,
+            timestamp,
+            timestamp - get_counter()
+        );
     }
     if status & RTIO_O_STATUS_DESTINATION_UNREACHABLE != 0 {
-        artiq_raise!("RTIODestinationUnreachable",
+        artiq_raise!(
-            "RTIO destination unreachable, output, at {0} mu, channel {1}",
+            "RTIODestinationUnreachable",
-            timestamp, channel as i64, 0);
+            format!(
+                "RTIO destination unreachable, output, at {{0}} mu, channel 0x{:04x}:{}",
+                channel,
+                resolve_channel_name(channel as u32)
+            ),
+            timestamp,
+            channel as i64,
+            0
+        );
     }
 }
 
-pub extern fn output(target: i32, data: i32) {
+pub extern "C" fn output(target: i32, data: i32) {
     unsafe {
         // Clear status so we can observe response
         TRANSACTION_BUFFER.reply_status.set(0);
@@ -126,7 +142,7 @@ pub extern fn output(target: i32, data: i32) {
     }
 }
 
-pub extern fn output_wide(target: i32, data: CSlice<i32>) {
+pub extern "C" fn output_wide(target: i32, data: CSlice<i32>) {
     unsafe {
         // Clear status so we can observe response
         TRANSACTION_BUFFER.reply_status.set(0);
@@ -143,7 +159,7 @@ pub extern fn output_wide(target: i32, data: CSlice<i32>) {
         loop {
             status = TRANSACTION_BUFFER.reply_status.get();
             if status != 0 {
-                break
+                break;
             }
         }
 
@@ -154,8 +170,8 @@ pub extern fn output_wide(target: i32, data: CSlice<i32>) {
     }
 }
 
-pub extern fn input_timestamp(timeout: i64, channel: i32) -> i64 {
+pub extern "C" fn input_timestamp(timeout: i64, channel: i32) -> i64 {
-   unsafe {
+    unsafe {
         // Clear status so we can observe response
         TRANSACTION_BUFFER.reply_status.set(0);
 
@@ -171,29 +187,45 @@ pub extern fn input_timestamp(timeout: i64, channel: i32) -> i64 {
         loop {
             status = TRANSACTION_BUFFER.reply_status.get();
             if status != 0 {
-                break
+                break;
             }
         }
 
         if status & RTIO_I_STATUS_OVERFLOW != 0 {
-            artiq_raise!("RTIOOverflow",
+            artiq_raise!(
-                         "RTIO input overflow on channel {0}",
+                "RTIOOverflow",
-                         channel as i64, 0, 0);
+                format!(
+                    "RTIO input overflow on channel 0x{:04x}:{}",
+                    channel,
+                    resolve_channel_name(channel as u32)
+                ),
+                channel as i64,
+                0,
+                0
+            );
         }
         if status & RTIO_I_STATUS_WAIT_EVENT != 0 {
-            return -1
+            return -1;
         }
         if status & RTIO_I_STATUS_DESTINATION_UNREACHABLE != 0 {
-            artiq_raise!("RTIODestinationUnreachable",
+            artiq_raise!(
-                         "RTIO destination unreachable, input, on channel {0}",
+                "RTIODestinationUnreachable",
-                         channel as i64, 0, 0);
+                format!(
+                    "RTIO destination unreachable, input, on channel 0x{:04x}:{}",
+                    channel,
+                    resolve_channel_name(channel as u32)
+                ),
+                channel as i64,
+                0,
+                0
+            );
         }
 
         TRANSACTION_BUFFER.reply_timestamp.get()
     }
 }
 
-pub extern fn input_data(channel: i32) -> i32 {
+pub extern "C" fn input_data(channel: i32) -> i32 {
     unsafe {
         TRANSACTION_BUFFER.reply_status.set(0);
 
@@ -209,26 +241,42 @@ pub extern fn input_data(channel: i32) -> i32 {
         loop {
             status = TRANSACTION_BUFFER.reply_status.get();
             if status != 0 {
-                break
+                break;
             }
         }
 
         if status & RTIO_I_STATUS_OVERFLOW != 0 {
-            artiq_raise!("RTIOOverflow",
+            artiq_raise!(
-                         "RTIO input overflow on channel {0}",
+                "RTIOOverflow",
-                         channel as i64, 0, 0);
+                format!(
+                    "RTIO input overflow on channel 0x{:04x}:{}",
+                    channel,
+                    resolve_channel_name(channel as u32)
+                ),
+                channel as i64,
+                0,
+                0
+            );
         }
         if status & RTIO_I_STATUS_DESTINATION_UNREACHABLE != 0 {
-            artiq_raise!("RTIODestinationUnreachable",
+            artiq_raise!(
-                         "RTIO destination unreachable, input, on channel {0}",
+                "RTIODestinationUnreachable",
-                         channel as i64, 0, 0);
+                format!(
+                    "RTIO destination unreachable, input, on channel 0x{:04x}:{}",
+                    channel,
+                    resolve_channel_name(channel as u32)
+                ),
+                channel as i64,
+                0,
+                0
+            );
         }
 
         TRANSACTION_BUFFER.reply_data.get()
     }
 }
 
-pub extern fn input_timestamped_data(timeout: i64, channel: i32) -> TimestampedData {
+pub extern "C" fn input_timestamped_data(timeout: i64, channel: i32) -> TimestampedData {
     unsafe {
         TRANSACTION_BUFFER.reply_status.set(0);
 
@@ -244,19 +292,35 @@ pub extern fn input_timestamped_data(timeout: i64, channel: i32) -> TimestampedD
         loop {
             status = TRANSACTION_BUFFER.reply_status.get();
             if status != 0 {
-                break
+                break;
             }
         }
 
         if status & RTIO_I_STATUS_OVERFLOW != 0 {
-            artiq_raise!("RTIOOverflow",
+            artiq_raise!(
-                         "RTIO input overflow on channel {0}",
+                "RTIOOverflow",
-                         channel as i64, 0, 0);
+                format!(
+                    "RTIO input overflow on channel 0x{:04x}:{}",
+                    channel,
+                    resolve_channel_name(channel as u32)
+                ),
+                channel as i64,
+                0,
+                0
+            );
         }
         if status & RTIO_I_STATUS_DESTINATION_UNREACHABLE != 0 {
-            artiq_raise!("RTIODestinationUnreachable",
+            artiq_raise!(
-                         "RTIO destination unreachable, input, on channel {0}",
+                "RTIODestinationUnreachable",
-                         channel as i64, 0, 0);
+                format!(
+                    "RTIO destination unreachable, input, on channel 0x{:04x}:{}",
+                    channel,
+                    resolve_channel_name(channel as u32)
+                ),
+                channel as i64,
+                0,
+                0
+            );
         }
 
         TimestampedData {

src/libksupport/src/rtio_csr.rs

@@ -0,0 +1,274 @@
+use core::ptr::{read_volatile, write_volatile};
+
+use cslice::CSlice;
+
+use crate::{artiq_raise, pl::csr, resolve_channel_name, rtio_core};
+
+pub const RTIO_O_STATUS_WAIT: u8 = 1;
+pub const RTIO_O_STATUS_UNDERFLOW: u8 = 2;
+pub const RTIO_O_STATUS_DESTINATION_UNREACHABLE: u8 = 4;
+pub const RTIO_I_STATUS_WAIT_EVENT: u8 = 1;
+pub const RTIO_I_STATUS_OVERFLOW: u8 = 2;
+pub const RTIO_I_STATUS_WAIT_STATUS: u8 = 4;
+pub const RTIO_I_STATUS_DESTINATION_UNREACHABLE: u8 = 8;
+
+#[repr(C)]
+pub struct TimestampedData {
+    timestamp: i64,
+    data: i32,
+}
+
+pub extern "C" fn init() {
+    unsafe {
+        rtio_core::reset_write(1);
+    }
+}
+
+pub extern "C" fn get_counter() -> i64 {
+    unsafe {
+        csr::rtio::counter_update_write(1);
+        csr::rtio::counter_read() as i64
+    }
+}
+
+pub extern "C" fn now_mu() -> i64 {
+    unsafe { csr::rtio::now_read() as i64 }
+}
+
+pub extern "C" fn at_mu(t: i64) {
+    unsafe {
+        csr::rtio::now_write(t as u64);
+    }
+}
+
+pub extern "C" fn delay_mu(dt: i64) {
+    unsafe {
+        csr::rtio::now_write(csr::rtio::now_read() + dt as u64);
+    }
+}
+
+// writing the LSB of o_data (offset=0) triggers the RTIO write
+#[inline(always)]
+pub unsafe fn rtio_o_data_write(offset: usize, data: u32) {
+    write_volatile(
+        csr::rtio::O_DATA_ADDR.offset((csr::rtio::O_DATA_SIZE - 1 - offset) as isize),
+        data,
+    );
+}
+
+#[inline(always)]
+pub unsafe fn rtio_i_data_read(offset: usize) -> u32 {
+    read_volatile(csr::rtio::I_DATA_ADDR.offset((csr::rtio::I_DATA_SIZE - 1 - offset) as isize))
+}
+
+#[inline(never)]
+unsafe fn process_exceptional_status(channel: i32, status: u8) {
+    let timestamp = csr::rtio::now_read() as i64;
+    if status & RTIO_O_STATUS_WAIT != 0 {
+        while csr::rtio::o_status_read() & RTIO_O_STATUS_WAIT != 0 {}
+    }
+    if status & RTIO_O_STATUS_UNDERFLOW != 0 {
+        artiq_raise!(
+            "RTIOUnderflow",
+            format!(
+                "RTIO underflow at {{1}} mu, channel 0x{:04x}:{}, slack {{2}} mu",
+                channel,
+                resolve_channel_name(channel as u32)
+            ),
+            channel as i64,
+            timestamp,
+            timestamp - get_counter()
+        );
+    }
+    if status & RTIO_O_STATUS_DESTINATION_UNREACHABLE != 0 {
+        artiq_raise!(
+            "RTIODestinationUnreachable",
+            format!(
+                "RTIO destination unreachable, output, at {{0}} mu, channel 0x{:04x}:{}",
+                channel,
+                resolve_channel_name(channel as u32)
+            ),
+            timestamp,
+            channel as i64,
+            0
+        );
+    }
+}
+
+pub extern "C" fn output(target: i32, data: i32) {
+    unsafe {
+        csr::rtio::target_write(target as u32);
+        // writing target clears o_data
+        rtio_o_data_write(0, data as _);
+        let status = csr::rtio::o_status_read();
+        if status != 0 {
+            process_exceptional_status(target >> 8, status);
+        }
+    }
+}
+
+pub extern "C" fn output_wide(target: i32, data: &CSlice<i32>) {
+    unsafe {
+        csr::rtio::target_write(target as u32);
+        // writing target clears o_data
+        for i in (0..data.len()).rev() {
+            rtio_o_data_write(i, data[i] as _)
+        }
+        let status = csr::rtio::o_status_read();
+        if status != 0 {
+            process_exceptional_status(target >> 8, status);
+        }
+    }
+}
+
+pub extern "C" fn input_timestamp(timeout: i64, channel: i32) -> i64 {
+    unsafe {
+        csr::rtio::target_write((channel as u32) << 8);
+        csr::rtio::i_timeout_write(timeout as u64);
+
+        let mut status = RTIO_I_STATUS_WAIT_STATUS;
+        while status & RTIO_I_STATUS_WAIT_STATUS != 0 {
+            status = csr::rtio::i_status_read();
+        }
+
+        if status & RTIO_I_STATUS_OVERFLOW != 0 {
+            artiq_raise!(
+                "RTIOOverflow",
+                format!(
+                    "RTIO input overflow on channel 0x{:04x}:{}",
+                    channel,
+                    resolve_channel_name(channel as u32)
+                ),
+                channel as i64,
+                0,
+                0
+            );
+        }
+        if status & RTIO_I_STATUS_WAIT_EVENT != 0 {
+            return -1;
+        }
+        if status & RTIO_I_STATUS_DESTINATION_UNREACHABLE != 0 {
+            artiq_raise!(
+                "RTIODestinationUnreachable",
+                format!(
+                    "RTIO destination unreachable, input, on channel 0x{:04x}:{}",
+                    channel,
+                    resolve_channel_name(channel as u32)
+                ),
+                channel as i64,
+                0,
+                0
+            );
+        }
+
+        csr::rtio::i_timestamp_read() as i64
+    }
+}
+
+pub extern "C" fn input_data(channel: i32) -> i32 {
+    unsafe {
+        csr::rtio::target_write((channel as u32) << 8);
+        csr::rtio::i_timeout_write(0xffffffff_ffffffff);
+
+        let mut status = RTIO_I_STATUS_WAIT_STATUS;
+        while status & RTIO_I_STATUS_WAIT_STATUS != 0 {
+            status = csr::rtio::i_status_read();
+        }
+
+        if status & RTIO_I_STATUS_OVERFLOW != 0 {
+            artiq_raise!(
+                "RTIOOverflow",
+                format!(
+                    "RTIO input overflow on channel 0x{:04x}:{}",
+                    channel,
+                    resolve_channel_name(channel as u32)
+                ),
+                channel as i64,
+                0,
+                0
+            );
+        }
+        if status & RTIO_I_STATUS_DESTINATION_UNREACHABLE != 0 {
+            artiq_raise!(
+                "RTIODestinationUnreachable",
+                format!(
+                    "RTIO destination unreachable, input, on channel 0x{:04x}:{}",
+                    channel,
+                    resolve_channel_name(channel as u32)
+                ),
+                channel as i64,
+                0,
+                0
+            );
+        }
+
+        rtio_i_data_read(0) as i32
+    }
+}
+
+pub extern "C" fn input_timestamped_data(timeout: i64, channel: i32) -> TimestampedData {
+    unsafe {
+        csr::rtio::target_write((channel as u32) << 8);
+        csr::rtio::i_timeout_write(timeout as u64);
+
+        let mut status = RTIO_I_STATUS_WAIT_STATUS;
+        while status & RTIO_I_STATUS_WAIT_STATUS != 0 {
+            status = csr::rtio::i_status_read();
+        }
+
+        if status & RTIO_I_STATUS_OVERFLOW != 0 {
+            artiq_raise!(
+                "RTIOOverflow",
+                format!(
+                    "RTIO input overflow on channel 0x{:04x}:{}",
+                    channel,
+                    resolve_channel_name(channel as u32)
+                ),
+                channel as i64,
+                0,
+                0
+            );
+        }
+        if status & RTIO_I_STATUS_WAIT_EVENT != 0 {
+            return TimestampedData { timestamp: -1, data: 0 };
+        }
+        if status & RTIO_I_STATUS_DESTINATION_UNREACHABLE != 0 {
+            artiq_raise!(
+                "RTIODestinationUnreachable",
+                format!(
+                    "RTIO destination unreachable, input, on channel 0x{:04x}:{}",
+                    channel,
+                    resolve_channel_name(channel as u32)
+                ),
+                channel as i64,
+                0,
+                0
+            );
+        }
+
+        TimestampedData {
+            timestamp: csr::rtio::i_timestamp_read() as i64,
+            data: rtio_i_data_read(0) as i32,
+        }
+    }
+}
+
+pub fn write_log(data: &[i8]) {
+    unsafe {
+        csr::rtio::target_write(csr::CONFIG_RTIO_LOG_CHANNEL << 8);
+
+        let mut word: u32 = 0;
+        for i in 0..data.len() {
+            word <<= 8;
+            word |= data[i] as u32;
+            if i % 4 == 3 {
+                rtio_o_data_write(0, word);
+                word = 0;
+            }
+        }
+
+        if word != 0 {
+            rtio_o_data_write(0, word);
+        }
+    }
+}

src/libunwind/backtrace.rs

@@ -1,27 +1,25 @@
-use libc::{c_void, c_int};
+use libc::{c_int, c_void};
+
 use crate::libunwind as uw;
 
 const UW_REG_SP: c_int = 13;
 
 pub fn backtrace<F>(f: F) -> Result<(), uw::_Unwind_Reason_Code>
-    where F: FnMut(usize) -> ()
+where F: FnMut(usize) -> () {
-{
     struct TraceContext<F> {
         step_fn: F,
-        prev_sp: uw::_Unwind_Word
+        prev_sp: uw::_Unwind_Word,
     }
 
-    extern fn trace<F>(context: *mut uw::_Unwind_Context, arg: *mut c_void)
+    extern "C" fn trace<F>(context: *mut uw::_Unwind_Context, arg: *mut c_void) -> uw::_Unwind_Reason_Code
-                      -> uw::_Unwind_Reason_Code
+    where F: FnMut(usize) -> () {
-        where F: FnMut(usize) -> ()
-    {
         unsafe {
             let trace_context = &mut *(arg as *mut TraceContext<F>);
 
             // Detect the root of a libfringe thread
             let cur_sp = uw::_Unwind_GetGR(context, UW_REG_SP);
             if cur_sp == trace_context.prev_sp {
-                return uw::_URC_END_OF_STACK
+                return uw::_URC_END_OF_STACK;
             } else {
                 trace_context.prev_sp = cur_sp;
             }
@@ -35,7 +33,7 @@ pub fn backtrace<F>(f: F) -> Result<(), uw::_Unwind_Reason_Code>
         let mut trace_context = TraceContext { step_fn: f, prev_sp: 0 };
         match uw::_Unwind_Backtrace(trace::<F>, &mut trace_context as *mut _ as *mut c_void) {
             uw::_URC_NO_REASON => Ok(()),
-            err => Err(err)
+            err => Err(err),
         }
     }
 }

src/libunwind/build.rs

@@ -5,8 +5,7 @@ fn main() {
 }
 
 mod llvm_libunwind {
-    use std::path::Path;
+    use std::{env, path::Path};
-    use std::env;
 
     fn setup_options(cfg: &mut cc::Build) {
         cfg.no_default_flags(true);
@@ -82,11 +81,7 @@ mod llvm_libunwind {
         cfg.flag("-fvisibility=hidden");
         cfg.flag_if_supported("-fvisibility-global-new-delete-hidden");
 
-        let unwind_sources = vec![
+        let unwind_sources = vec!["Unwind-EHABI.cpp", "Unwind-seh.cpp", "libunwind.cpp"];
-            "Unwind-EHABI.cpp",
-            "Unwind-seh.cpp",
-            "libunwind.cpp"
-        ];
 
         let root = Path::new("../llvm_libunwind");
         cfg.include(root.join("include"));

src/libunwind/libunwind.rs

@@ -21,8 +21,7 @@ pub use _Unwind_Reason_Code::*;
 pub type _Unwind_Exception_Class = u64;
 pub type _Unwind_Word = uintptr_t;
 pub type _Unwind_Ptr = uintptr_t;
-pub type _Unwind_Trace_Fn =
+pub type _Unwind_Trace_Fn = extern "C" fn(ctx: *mut _Unwind_Context, arg: *mut c_void) -> _Unwind_Reason_Code;
-    extern "C" fn(ctx: *mut _Unwind_Context, arg: *mut c_void) -> _Unwind_Reason_Code;
 
 #[cfg(target_arch = "x86")]
 pub const unwinder_private_data_size: usize = 5;
@@ -279,7 +278,6 @@ if #[cfg(all(windows, target_arch = "x86_64", target_env = "gnu"))] {
 } // cfg_if!
 
 #[no_mangle]
-extern fn abort() {
+extern "C" fn abort() {
     panic!("Abort!");
 }
-

src/runtime/Cargo.toml.tpl

@@ -18,7 +18,6 @@ num-traits = { version = "0.2", default-features = false }
 num-derive = "0.3"
 cslice = "0.3"
 log = "0.4"
-nb = "0.1"
 embedded-hal = "0.2"
 core_io = { version = "0.1", features = ["collections"] }
 byteorder = { version = "1.3", default-features = false }
@@ -26,19 +25,23 @@ void = { version = "1", default-features = false }
 futures = { version = "0.3", default-features = false, features = ["async-await"] }
 async-recursion = "0.3"
 log_buffer = { version = "1.2" }
-libm = { version = "0.2", features = ["unstable"] }
 vcell = "0.1"
 
-libboard_zynq = { git = "https://git.m-labs.hk/M-Labs/zynq-rs.git", features = ["ipv6"]}
+libboard_zynq = { path = "@@ZYNQ_RS@@/libboard_zynq", features = ["ipv6"]}
-libsupport_zynq = { default-features = false, features = ["alloc_core"], git = "https://git.m-labs.hk/M-Labs/zynq-rs.git" }
+libsupport_zynq = { path = "@@ZYNQ_RS@@/libsupport_zynq", default-features = false, features = ["alloc_core"] }
-libcortex_a9 = { git = "https://git.m-labs.hk/M-Labs/zynq-rs.git" }
+libcortex_a9 = { path = "@@ZYNQ_RS@@/libcortex_a9" }
-libasync = { git = "https://git.m-labs.hk/M-Labs/zynq-rs.git" }
+libasync = { path = "@@ZYNQ_RS@@/libasync" }
-libregister = { git = "https://git.m-labs.hk/M-Labs/zynq-rs.git" }
+libregister = { path = "@@ZYNQ_RS@@/libregister" }
-libconfig = { git = "https://git.m-labs.hk/M-Labs/zynq-rs.git", features = ["fat_lfn", "ipv6"] }
+libconfig = { path = "@@ZYNQ_RS@@/libconfig", features = ["fat_lfn", "ipv6"] }
 
 dyld = { path = "../libdyld" }
 dwarf = { path = "../libdwarf" }
 unwind = { path = "../libunwind" }
 libc = { path = "../libc" }
-io = { path = "../libio" }
+io = { path = "../libio", features = ["alloc"] }
+ksupport = { path = "../libksupport" }
 libboard_artiq = { path = "../libboard_artiq" }
+
+[dependencies.tar-no-std]
+git = "https://git.m-labs.hk/M-Labs/tar-no-std"
+rev = "2ab6dc5"

src/runtime/src/analyzer.rs

@@ -1,10 +1,15 @@
+use alloc::rc::Rc;
+#[cfg(has_drtio)]
+use alloc::vec::Vec;
+use core::cell::RefCell;
+
 use libasync::{smoltcp::TcpStream, task};
-use libboard_zynq::smoltcp::Error;
+use libboard_artiq::drtio_routing;
-use libcortex_a9::cache;
+use libboard_zynq::{smoltcp::Error, timer::GlobalTimer};
+use libcortex_a9::{cache, mutex::Mutex};
 use log::{debug, info, warn};
 
-use crate::proto_async::*;
+use crate::{pl, proto_async::*};
-use crate::pl;
 
 const BUFFER_SIZE: usize = 512 * 1024;
 
@@ -13,9 +18,7 @@ struct Buffer {
     data: [u8; BUFFER_SIZE],
 }
 
-static mut BUFFER: Buffer = Buffer {
+static mut BUFFER: Buffer = Buffer { data: [0; BUFFER_SIZE] };
-    data: [0; BUFFER_SIZE]
-};
 
 fn arm() {
     debug!("arming RTIO analyzer");
@@ -40,13 +43,57 @@ fn disarm() {
     debug!("RTIO analyzer disarmed");
 }
 
+#[cfg(has_drtio)]
+pub mod remote_analyzer {
+    use super::*;
+    use crate::rtio_mgt::drtio;
+
+    pub struct RemoteBuffer {
+        pub total_byte_count: u64,
+        pub sent_bytes: u32,
+        pub error: bool,
+        pub data: Vec<u8>,
+    }
+
+    pub async fn get_data(
+        aux_mutex: &Rc<Mutex<bool>>,
+        routing_table: &drtio_routing::RoutingTable,
+        up_destinations: &Rc<RefCell<[bool; drtio_routing::DEST_COUNT]>>,
+        timer: GlobalTimer,
+    ) -> Result<RemoteBuffer, drtio::Error> {
+        // gets data from satellites and returns consolidated data
+        let mut remote_data: Vec<u8> = Vec::new();
+        let mut remote_error = false;
+        let mut remote_sent_bytes = 0;
+        let mut remote_total_bytes = 0;
+
+        let data_vec = match drtio::analyzer_query(aux_mutex, routing_table, up_destinations, timer).await {
+            Ok(data_vec) => data_vec,
+            Err(e) => return Err(e),
+        };
+        for data in data_vec {
+            remote_total_bytes += data.total_byte_count;
+            remote_sent_bytes += data.sent_bytes;
+            remote_error |= data.error;
+            remote_data.extend(data.data);
+        }
+
+        Ok(RemoteBuffer {
+            total_byte_count: remote_total_bytes,
+            sent_bytes: remote_sent_bytes,
+            error: remote_error,
+            data: remote_data,
+        })
+    }
+}
+
 #[derive(Debug)]
 struct Header {
     sent_bytes: u32,
     total_byte_count: u64,
     error_occurred: bool,
     log_channel: u8,
-    dds_onehot_sel: bool
+    dds_onehot_sel: bool,
 }
 
 async fn write_header(stream: &mut TcpStream, header: &Header) -> Result<(), Error> {
@@ -59,7 +106,13 @@ async fn write_header(stream: &mut TcpStream, header: &Header) -> Result<(), Err
     Ok(())
 }
 
-async fn handle_connection(stream: &mut TcpStream) -> Result<(), Error> {
+async fn handle_connection(
+    stream: &mut TcpStream,
+    _aux_mutex: &Rc<Mutex<bool>>,
+    _routing_table: &drtio_routing::RoutingTable,
+    _up_destinations: &Rc<RefCell<[bool; drtio_routing::DEST_COUNT]>>,
+    _timer: GlobalTimer,
+) -> Result<(), Error> {
     info!("received connection");
 
     let data = unsafe { &BUFFER.data[..] };
@@ -68,6 +121,11 @@ async fn handle_connection(stream: &mut TcpStream) -> Result<(), Error> {
     let total_byte_count = unsafe { pl::csr::rtio_analyzer::dma_byte_count_read() as u64 };
     let pointer = (total_byte_count % BUFFER_SIZE as u64) as usize;
     let wraparound = total_byte_count >= BUFFER_SIZE as u64;
+    let sent_bytes = if wraparound {
+        BUFFER_SIZE as u32
+    } else {
+        total_byte_count as u32
+    };
 
     if overflow_occurred {
         warn!("overflow occured");
@@ -76,12 +134,42 @@ async fn handle_connection(stream: &mut TcpStream) -> Result<(), Error> {
         warn!("bus error occured");
     }
 
+    #[cfg(has_drtio)]
+    let remote = remote_analyzer::get_data(_aux_mutex, _routing_table, _up_destinations, _timer).await;
+    #[cfg(has_drtio)]
+    let (header, remote_data) = match remote {
+        Ok(remote) => (
+            Header {
+                total_byte_count: total_byte_count + remote.total_byte_count,
+                sent_bytes: sent_bytes + remote.sent_bytes,
+                error_occurred: overflow_occurred | bus_error_occurred | remote.error,
+                log_channel: pl::csr::CONFIG_RTIO_LOG_CHANNEL as u8,
+                dds_onehot_sel: true,
+            },
+            remote.data,
+        ),
+        Err(e) => {
+            warn!("Error getting remote analyzer data: {}", e);
+            (
+                Header {
+                    total_byte_count: total_byte_count,
+                    sent_bytes: sent_bytes,
+                    error_occurred: true,
+                    log_channel: pl::csr::CONFIG_RTIO_LOG_CHANNEL as u8,
+                    dds_onehot_sel: true,
+                },
+                Vec::new(),
+            )
+        }
+    };
+
+    #[cfg(not(has_drtio))]
     let header = Header {
         total_byte_count: total_byte_count,
-        sent_bytes: if wraparound { BUFFER_SIZE as u32 } else { total_byte_count as u32 },
+        sent_bytes: sent_bytes,
         error_occurred: overflow_occurred | bus_error_occurred,
         log_channel: pl::csr::CONFIG_RTIO_LOG_CHANNEL as u8,
-        dds_onehot_sel: true  // kept for backward compatibility of analyzer dumps
+        dds_onehot_sel: true, // kept for backward compatibility of analyzer dumps
     };
     debug!("{:?}", header);
 
@@ -92,17 +180,28 @@ async fn handle_connection(stream: &mut TcpStream) -> Result<(), Error> {
     } else {
         stream.send(data[..pointer].iter().copied()).await?;
     }
+    #[cfg(has_drtio)]
+    stream.send(remote_data.iter().copied()).await?;
 
     Ok(())
 }
 
-pub fn start() {
+pub fn start(
+    aux_mutex: &Rc<Mutex<bool>>,
+    routing_table: &Rc<RefCell<drtio_routing::RoutingTable>>,
+    up_destinations: &Rc<RefCell<[bool; drtio_routing::DEST_COUNT]>>,
+    timer: GlobalTimer,
+) {
+    let aux_mutex = aux_mutex.clone();
+    let routing_table = routing_table.clone();
+    let up_destinations = up_destinations.clone();
     task::spawn(async move {
         loop {
             arm();
             let mut stream = TcpStream::accept(1382, 2048, 2048).await.unwrap();
             disarm();
-            let _ = handle_connection(&mut stream)
+            let routing_table = routing_table.borrow();
+            let _ = handle_connection(&mut stream, &aux_mutex, &routing_table, &up_destinations, timer)
                 .await
                 .map_err(|e| warn!("connection terminated: {:?}", e));
             let _ = stream.flush().await;

src/runtime/src/comms.rs

@@ -1,44 +1,57 @@
-use core::fmt;
+use alloc::{collections::BTreeMap, rc::Rc, string::String, vec, vec::Vec};
-use core::cell::RefCell;
+use core::{cell::RefCell, fmt, slice, str};
-use alloc::{vec, vec::Vec, string::String, collections::BTreeMap, rc::Rc};
-use log::{info, warn, error};
-use cslice::CSlice;
 
+use core_io::Error as IoError;
+use cslice::CSlice;
+use dyld::elf;
+use futures::{future::FutureExt, select_biased};
+#[cfg(has_drtio)]
+use io::Cursor;
+#[cfg(has_drtio)]
+use ksupport::rpc;
+use ksupport::{kernel, resolve_channel_name};
+#[cfg(has_drtio)]
+use libasync::delay;
+use libasync::{smoltcp::{Sockets, TcpStream},
+               task};
+use libboard_artiq::drtio_routing;
+#[cfg(feature = "target_kasli_soc")]
+use libboard_zynq::error_led::ErrorLED;
+#[cfg(has_drtio)]
+use libboard_zynq::time::Milliseconds;
+use libboard_zynq::{self as zynq,
+                    smoltcp::{self,
+                              iface::{EthernetInterfaceBuilder, NeighborCache},
+                              time::Instant,
+                              wire::IpCidr},
+                    timer::GlobalTimer};
+use libconfig::{net_settings, Config};
+use libcortex_a9::{mutex::Mutex,
+                   semaphore::Semaphore,
+                   sync_channel::{Receiver, Sender}};
+use log::{error, info, warn};
 use num_derive::{FromPrimitive, ToPrimitive};
 use num_traits::{FromPrimitive, ToPrimitive};
+#[cfg(has_drtio)]
+use tar_no_std::TarArchiveRef;
 
-use libboard_zynq::{
-    self as zynq,
-    smoltcp::{
-        self,
-        wire::IpCidr,
-        iface::{NeighborCache, EthernetInterfaceBuilder},
-        time::Instant,
-    },
-    timer::GlobalTimer,
-};
-use libcortex_a9::{semaphore::Semaphore, mutex::Mutex, sync_channel::{Sender, Receiver}};
-use futures::{select_biased, future::FutureExt};
-use libasync::{smoltcp::{Sockets, TcpStream}, task};
-use libconfig::{Config, net_settings};
-use libboard_artiq::drtio_routing;
-
-use crate::proto_async::*;
-use crate::kernel;
-use crate::rpc;
-use crate::moninj;
-use crate::mgmt;
-use crate::analyzer;
-use crate::rtio_mgt;
 #[cfg(has_drtio)]
 use crate::pl;
+use crate::{analyzer, mgmt, moninj, proto_async::*, rpc_async, rtio_dma, rtio_mgt};
+#[cfg(has_drtio)]
+use crate::{subkernel, subkernel::Error as SubkernelError};
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub enum Error {
     NetworkError(smoltcp::Error),
+    IoError,
     UnexpectedPattern,
     UnrecognizedPacket,
     BufferExhausted,
+    #[cfg(has_drtio)]
+    SubkernelError(subkernel::Error),
+    #[cfg(has_drtio)]
+    DestinationDown,
 }
 
 pub type Result<T> = core::result::Result<T, Error>;
@@ -47,9 +60,14 @@ impl fmt::Display for Error {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         match self {
             Error::NetworkError(error) => write!(f, "network error: {}", error),
-            Error::UnexpectedPattern   => write!(f, "unexpected pattern"),
+            Error::IoError => write!(f, "io error"),
-            Error::UnrecognizedPacket  => write!(f, "unrecognized packet"),
+            Error::UnexpectedPattern => write!(f, "unexpected pattern"),
-            Error::BufferExhausted     => write!(f, "buffer exhausted"),
+            Error::UnrecognizedPacket => write!(f, "unrecognized packet"),
+            Error::BufferExhausted => write!(f, "buffer exhausted"),
+            #[cfg(has_drtio)]
+            Error::SubkernelError(error) => write!(f, "subkernel error: {:?}", error),
+            #[cfg(has_drtio)]
+            Error::DestinationDown => write!(f, "subkernel destination down"),
         }
     }
 }
@@ -60,6 +78,19 @@ impl From<smoltcp::Error> for Error {
     }
 }
 
+impl From<IoError> for Error {
+    fn from(_error: IoError) -> Self {
+        Error::IoError
+    }
+}
+
+#[cfg(has_drtio)]
+impl From<subkernel::Error> for Error {
+    fn from(error: subkernel::Error) -> Self {
+        Error::SubkernelError(error)
+    }
+}
+
 #[derive(Debug, FromPrimitive, ToPrimitive)]
 enum Request {
     SystemInfo = 3,
@@ -67,6 +98,7 @@ enum Request {
     RunKernel = 6,
     RPCReply = 7,
     RPCException = 8,
+    UploadSubkernel = 9,
 }
 
 #[derive(Debug, FromPrimitive, ToPrimitive)]
@@ -83,18 +115,18 @@ enum Reply {
 }
 
 static CACHE_STORE: Mutex<BTreeMap<String, Vec<i32>>> = Mutex::new(BTreeMap::new());
-static DMA_RECORD_STORE: Mutex<BTreeMap<String, (Vec<u8>, i64)>> = Mutex::new(BTreeMap::new());
 
 async fn write_header(stream: &TcpStream, reply: Reply) -> Result<()> {
-    stream.send_slice(&[0x5a, 0x5a, 0x5a, 0x5a, reply.to_u8().unwrap()]).await?;
+    stream
+        .send_slice(&[0x5a, 0x5a, 0x5a, 0x5a, reply.to_u8().unwrap()])
+        .await?;
     Ok(())
 }
 
 async fn read_request(stream: &TcpStream, allow_close: bool) -> Result<Option<Request>> {
     match expect(stream, &[0x5a, 0x5a, 0x5a, 0x5a]).await {
         Ok(true) => {}
-        Ok(false) =>
+        Ok(false) => return Err(Error::UnexpectedPattern),
-            return Err(Error::UnexpectedPattern),
         Err(smoltcp::Error::Finished) => {
             if allow_close {
                 info!("peer closed connection");
@@ -103,11 +135,12 @@ async fn read_request(stream: &TcpStream, allow_close: bool) -> Result<Option<Re
                 error!("peer unexpectedly closed connection");
                 return Err(smoltcp::Error::Finished)?;
             }
-        },
+        }
-        Err(e) =>
+        Err(e) => return Err(e)?,
-            return Err(e)?,
     }
-    Ok(Some(FromPrimitive::from_i8(read_i8(&stream).await?).ok_or(Error::UnrecognizedPacket)?))
+    Ok(Some(
+        FromPrimitive::from_i8(read_i8(&stream).await?).ok_or(Error::UnrecognizedPacket)?,
+    ))
 }
 
 async fn read_bytes(stream: &TcpStream, max_length: usize) -> Result<Vec<u8>> {
@@ -126,9 +159,7 @@ async fn fast_send(sender: &mut Sender<'_, kernel::Message>, content: kernel::Me
     let mut content = content;
     for _ in 0..RETRY_LIMIT {
         match sender.try_send(content) {
-            Ok(()) => {
+            Ok(()) => return,
-                return
-            },
             Err(v) => {
                 content = v;
             }
@@ -140,10 +171,8 @@ async fn fast_send(sender: &mut Sender<'_, kernel::Message>, content: kernel::Me
 async fn fast_recv(receiver: &mut Receiver<'_, kernel::Message>) -> kernel::Message {
     for _ in 0..RETRY_LIMIT {
         match receiver.try_recv() {
-            Ok(v) => {
+            Ok(v) => return v,
-                return v;
+            Err(()) => (),
-            },
-            Err(()) => ()
         }
     }
     receiver.async_recv().await
@@ -159,7 +188,14 @@ async fn write_exception_string(stream: &TcpStream, s: CSlice<'static, u8>) -> R
     Ok(())
 }
 
-async fn handle_run_kernel(stream: Option<&TcpStream>, control: &Rc<RefCell<kernel::Control>>, _up_destinations: &Rc<RefCell<[bool; drtio_routing::DEST_COUNT]>>) -> Result<()> {
+async fn handle_run_kernel(
+    stream: Option<&TcpStream>,
+    control: &Rc<RefCell<kernel::Control>>,
+    _up_destinations: &Rc<RefCell<[bool; drtio_routing::DEST_COUNT]>>,
+    aux_mutex: &Rc<Mutex<bool>>,
+    routing_table: &drtio_routing::RoutingTable,
+    timer: GlobalTimer,
+) -> Result<()> {
     control.borrow_mut().tx.async_send(kernel::Message::StartRequest).await;
     loop {
         let reply = control.borrow_mut().rx.async_recv().await;
@@ -167,7 +203,7 @@ async fn handle_run_kernel(stream: Option<&TcpStream>, control: &Rc<RefCell<kern
             kernel::Message::RpcSend { is_async, data } => {
                 if stream.is_none() {
                     error!("Unexpected RPC from startup/idle kernel!");
-                    break
+                    break;
                 }
                 let stream = stream.unwrap();
                 write_header(stream, Reply::RPCRequest).await?;
@@ -182,7 +218,7 @@ async fn handle_run_kernel(stream: Option<&TcpStream>, control: &Rc<RefCell<kern
                                 kernel::Message::RpcRecvRequest(slot) => slot,
                                 other => panic!("expected root value slot from core1, not {:?}", other),
                             };
-                            rpc::recv_return(stream, &tag, slot, &|size| {
+                            rpc_async::recv_return(stream, &tag, slot, &|size| {
                                 let control = control.clone();
                                 async move {
                                     if size == 0 {
@@ -194,46 +230,64 @@ async fn handle_run_kernel(stream: Option<&TcpStream>, control: &Rc<RefCell<kern
                                         fast_send(&mut control.tx, kernel::Message::RpcRecvReply(Ok(size))).await;
                                         match fast_recv(&mut control.rx).await {
                                             kernel::Message::RpcRecvRequest(slot) => slot,
-                                            other => panic!("expected nested value slot from kernel CPU, not {:?}", other),
+                                            other => {
+                                                panic!("expected nested value slot from kernel CPU, not {:?}", other)
+                                            }
                                         }
                                     }
                                 }
-                            }).await?;
+                            })
-                            control.borrow_mut().tx.async_send(kernel::Message::RpcRecvReply(Ok(0))).await;
+                            .await?;
-                        },
+                            control
+                                .borrow_mut()
+                                .tx
+                                .async_send(kernel::Message::RpcRecvReply(Ok(0)))
+                                .await;
+                        }
                         Request::RPCException => {
                             let mut control = control.borrow_mut();
                             match control.rx.async_recv().await {
                                 kernel::Message::RpcRecvRequest(_) => (),
                                 other => panic!("expected (ignored) root value slot from kernel CPU, not {:?}", other),
                             }
-                            let id =       read_i32(stream).await? as u32;
+                            let id = read_i32(stream).await? as u32;
-                            let message =  read_i32(stream).await? as u32;
+                            let message = read_i32(stream).await? as u32;
-                            let param =    [read_i64(stream).await?,
+                            let param = [
-                                            read_i64(stream).await?,
+                                read_i64(stream).await?,
-                                            read_i64(stream).await?];
+                                read_i64(stream).await?,
-                            let file =     read_i32(stream).await? as u32;
+                                read_i64(stream).await?,
-                            let line =     read_i32(stream).await?;
+                            ];
-                            let column =   read_i32(stream).await?;
+                            let file = read_i32(stream).await? as u32;
+                            let line = read_i32(stream).await?;
+                            let column = read_i32(stream).await?;
                             let function = read_i32(stream).await? as u32;
-                            control.tx.async_send(kernel::Message::RpcRecvReply(Err(kernel::RPCException {
+                            control
-                                id, message, param, file, line, column, function
+                                .tx
-                            }))).await;
+                                .async_send(kernel::Message::RpcRecvReply(Err(ksupport::RPCException {
-                        },
+                                    id,
+                                    message,
+                                    param,
+                                    file,
+                                    line,
+                                    column,
+                                    function,
+                                })))
+                                .await;
+                        }
                         _ => {
                             error!("unexpected RPC request from host: {:?}", host_request);
-                            return Err(Error::UnrecognizedPacket)
+                            return Err(Error::UnrecognizedPacket);
                         }
                     }
                 }
-            },
+            }
             kernel::Message::KernelFinished(async_errors) => {
                 if let Some(stream) = stream {
                     write_header(stream, Reply::KernelFinished).await?;
                     write_i8(stream, async_errors as i8).await?;
                 }
                 break;
-            },
+            }
             kernel::Message::KernelException(exceptions, stack_pointers, backtrace, async_errors) => {
                 match stream {
                     Some(stream) => {
@@ -244,7 +298,26 @@ async fn handle_run_kernel(stream: Option<&TcpStream>, control: &Rc<RefCell<kern
                         for exception in exceptions.iter() {
                             let exception = exception.as_ref().unwrap();
                             write_i32(stream, exception.id as i32).await?;
-                            write_exception_string(stream, exception.message).await?;
+
+                            if exception.message.len() == usize::MAX {
+                                // exception with host string
+                                write_exception_string(stream, exception.message).await?;
+                            } else {
+                                let msg = str::from_utf8(unsafe {
+                                    slice::from_raw_parts(exception.message.as_ptr(), exception.message.len())
+                                })
+                                .unwrap()
+                                .replace(
+                                    "{rtio_channel_info:0}",
+                                    &format!(
+                                        "0x{:04x}:{}",
+                                        exception.param[0],
+                                        resolve_channel_name(exception.param[0] as u32)
+                                    ),
+                                );
+                                write_exception_string(stream, unsafe { CSlice::new(msg.as_ptr(), msg.len()) }).await?;
+                            }
+
                             write_i64(stream, exception.param[0] as i64).await?;
                             write_i64(stream, exception.param[1] as i64).await?;
                             write_i64(stream, exception.param[2] as i64).await?;
@@ -264,7 +337,7 @@ async fn handle_run_kernel(stream: Option<&TcpStream>, control: &Rc<RefCell<kern
                             write_i32(stream, sp as i32).await?;
                         }
                         write_i8(stream, async_errors as i8).await?;
-                    },
+                    }
                     None => {
                         error!("Uncaught kernel exceptions: {:?}", exceptions);
                     }
@@ -273,27 +346,205 @@ async fn handle_run_kernel(stream: Option<&TcpStream>, control: &Rc<RefCell<kern
             }
             kernel::Message::CachePutRequest(key, value) => {
                 CACHE_STORE.lock().insert(key, value);
-            },
+            }
             kernel::Message::CacheGetRequest(key) => {
                 const DEFAULT: Vec<i32> = Vec::new();
                 let value = CACHE_STORE.lock().get(&key).unwrap_or(&DEFAULT).clone();
-                control.borrow_mut().tx.async_send(kernel::Message::CacheGetReply(value)).await;
+                control
-            },
+                    .borrow_mut()
+                    .tx
+                    .async_send(kernel::Message::CacheGetReply(value))
+                    .await;
+            }
             kernel::Message::DmaPutRequest(recorder) => {
-                DMA_RECORD_STORE.lock().insert(recorder.name, (recorder.buffer, recorder.duration));
+                let _id = rtio_dma::put_record(aux_mutex, routing_table, timer, recorder).await;
-            },
+                #[cfg(has_drtio)]
+                rtio_dma::remote_dma::upload_traces(aux_mutex, routing_table, timer, _id).await;
+            }
             kernel::Message::DmaEraseRequest(name) => {
                 // prevent possible OOM when we have large DMA record replacement.
-                DMA_RECORD_STORE.lock().remove(&name);
+                rtio_dma::erase(name, aux_mutex, routing_table, timer).await;
-            },
+            }
             kernel::Message::DmaGetRequest(name) => {
-                let result = DMA_RECORD_STORE.lock().get(&name).map(|v| v.clone());
+                let result = rtio_dma::retrieve(name).await;
-                control.borrow_mut().tx.async_send(kernel::Message::DmaGetReply(result)).await;
+                control
-            },
+                    .borrow_mut()
+                    .tx
+                    .async_send(kernel::Message::DmaGetReply(result))
+                    .await;
+            }
+            #[cfg(has_drtio)]
+            kernel::Message::DmaStartRemoteRequest { id, timestamp } => {
+                rtio_dma::remote_dma::playback(aux_mutex, routing_table, timer, id as u32, timestamp as u64).await;
+            }
+            #[cfg(has_drtio)]
+            kernel::Message::DmaAwaitRemoteRequest(id) => {
+                let result = rtio_dma::remote_dma::await_done(id as u32, Some(10_000), timer).await;
+                let reply = match result {
+                    Ok(rtio_dma::remote_dma::RemoteState::PlaybackEnded {
+                        error,
+                        channel,
+                        timestamp,
+                    }) => kernel::Message::DmaAwaitRemoteReply {
+                        timeout: false,
+                        error: error,
+                        channel: channel,
+                        timestamp: timestamp,
+                    },
+                    _ => kernel::Message::DmaAwaitRemoteReply {
+                        timeout: true,
+                        error: 0,
+                        channel: 0,
+                        timestamp: 0,
+                    },
+                };
+                control.borrow_mut().tx.async_send(reply).await;
+            }
+            #[cfg(has_drtio)]
+            kernel::Message::SubkernelLoadRunRequest {
+                id,
+                destination: _,
+                run,
+            } => {
+                let succeeded = match subkernel::load(aux_mutex, routing_table, timer, id, run).await {
+                    Ok(()) => true,
+                    Err(e) => {
+                        error!("Error loading subkernel: {:?}", e);
+                        false
+                    }
+                };
+                control
+                    .borrow_mut()
+                    .tx
+                    .async_send(kernel::Message::SubkernelLoadRunReply { succeeded: succeeded })
+                    .await;
+            }
+            #[cfg(has_drtio)]
+            kernel::Message::SubkernelAwaitFinishRequest { id, timeout } => {
+                let res = subkernel::await_finish(aux_mutex, routing_table, timer, id, timeout).await;
+                let status = match res {
+                    Ok(ref res) => {
+                        if res.status == subkernel::FinishStatus::CommLost {
+                            kernel::SubkernelStatus::CommLost
+                        } else if let Some(exception) = &res.exception {
+                            error!("Exception in subkernel");
+                            match stream {
+                                None => (),
+                                Some(stream) => {
+                                    write_chunk(stream, exception).await?;
+                                }
+                            }
+                            // will not be called after exception is served
+                            kernel::SubkernelStatus::OtherError
+                        } else {
+                            kernel::SubkernelStatus::NoError
+                        }
+                    }
+                    Err(SubkernelError::Timeout) => kernel::SubkernelStatus::Timeout,
+                    Err(SubkernelError::IncorrectState) => kernel::SubkernelStatus::IncorrectState,
+                    Err(_) => kernel::SubkernelStatus::OtherError,
+                };
+                control
+                    .borrow_mut()
+                    .tx
+                    .async_send(kernel::Message::SubkernelAwaitFinishReply { status: status })
+                    .await;
+            }
+            #[cfg(has_drtio)]
+            kernel::Message::SubkernelMsgSend { id, destination, data } => {
+                let res =
+                    subkernel::message_send(aux_mutex, routing_table, timer, id, destination.unwrap(), data).await;
+                match res {
+                    Ok(_) => (),
+                    Err(e) => {
+                        error!("error sending subkernel message: {:?}", e)
+                    }
+                };
+                control
+                    .borrow_mut()
+                    .tx
+                    .async_send(kernel::Message::SubkernelMsgSent)
+                    .await;
+            }
+            #[cfg(has_drtio)]
+            kernel::Message::SubkernelMsgRecvRequest { id, timeout, tags } => {
+                let message_received = subkernel::message_await(id as u32, timeout, timer).await;
+                let (status, count) = match message_received {
+                    Ok(ref message) => (kernel::SubkernelStatus::NoError, message.count),
+                    Err(SubkernelError::Timeout) => (kernel::SubkernelStatus::Timeout, 0),
+                    Err(SubkernelError::IncorrectState) => (kernel::SubkernelStatus::IncorrectState, 0),
+                    Err(SubkernelError::CommLost) => (kernel::SubkernelStatus::CommLost, 0),
+                    Err(SubkernelError::SubkernelException) => {
+                        error!("Exception in subkernel");
+                        // just retrieve the exception
+                        let status = subkernel::await_finish(aux_mutex, routing_table, timer, id as u32, timeout)
+                            .await
+                            .unwrap();
+                        match stream {
+                            None => (),
+                            Some(stream) => {
+                                write_chunk(stream, &status.exception.unwrap()).await?;
+                            }
+                        }
+                        (kernel::SubkernelStatus::OtherError, 0)
+                    }
+                    Err(_) => (kernel::SubkernelStatus::OtherError, 0),
+                };
+                control
+                    .borrow_mut()
+                    .tx
+                    .async_send(kernel::Message::SubkernelMsgRecvReply {
+                        status: status,
+                        count: count,
+                    })
+                    .await;
+                if let Ok(message) = message_received {
+                    // receive code almost identical to RPC recv, except we are not reading from a stream
+                    let mut reader = Cursor::new(message.data);
+                    let mut current_tags: &[u8] = &tags;
+                    let mut i = 0;
+                    loop {
+                        // kernel has to consume all arguments in the whole message
+                        let slot = match fast_recv(&mut control.borrow_mut().rx).await {
+                            kernel::Message::RpcRecvRequest(slot) => slot,
+                            other => panic!("expected root value slot from core1, not {:?}", other),
+                        };
+                        let remaining_tags = rpc::recv_return(&mut reader, &current_tags, slot, &mut |size| {
+                            if size == 0 {
+                                0 as *mut ()
+                            } else {
+                                let mut control = control.borrow_mut();
+                                control.tx.send(kernel::Message::RpcRecvReply(Ok(size)));
+                                match control.rx.recv() {
+                                    kernel::Message::RpcRecvRequest(slot) => slot,
+                                    other => {
+                                        panic!("expected nested value slot from kernel CPU, not {:?}", other)
+                                    }
+                                }
+                            }
+                        })?;
+                        control
+                            .borrow_mut()
+                            .tx
+                            .async_send(kernel::Message::RpcRecvReply(Ok(0)))
+                            .await;
+                        i += 1;
+                        if i < count {
+                            current_tags = remaining_tags;
+                        } else {
+                            break;
+                        }
+                    }
+                }
+            }
             #[cfg(has_drtio)]
             kernel::Message::UpDestinationsRequest(destination) => {
                 let result = _up_destinations.borrow()[destination as usize];
-                control.borrow_mut().tx.async_send(kernel::Message::UpDestinationsReply(result)).await;
+                control
+                    .borrow_mut()
+                    .tx
+                    .async_send(kernel::Message::UpDestinationsReply(result))
+                    .await;
             }
             _ => {
                 panic!("unexpected message from core1 while kernel was running: {:?}", reply);
@@ -303,11 +554,67 @@ async fn handle_run_kernel(stream: Option<&TcpStream>, control: &Rc<RefCell<kern
     Ok(())
 }
 
+async fn handle_flash_kernel(
+    buffer: &Vec<u8>,
+    control: &Rc<RefCell<kernel::Control>>,
+    _up_destinations: &Rc<RefCell<[bool; drtio_routing::DEST_COUNT]>>,
+    _aux_mutex: &Rc<Mutex<bool>>,
+    _routing_table: &drtio_routing::RoutingTable,
+    _timer: GlobalTimer,
+) -> Result<()> {
+    if buffer[0] == elf::ELFMAG0 && buffer[1] == elf::ELFMAG1 && buffer[2] == elf::ELFMAG2 && buffer[3] == elf::ELFMAG3
+    {
+        // assume ELF file, proceed as before
+        load_kernel(buffer, control, None).await
+    } else {
+        #[cfg(has_drtio)]
+        {
+            let archive = TarArchiveRef::new(buffer.as_ref());
+            let entries = archive.entries();
+            let mut main_lib: Vec<u8> = Vec::new();
+            for entry in entries {
+                if entry.filename().as_str() == "main.elf" {
+                    main_lib = entry.data().to_vec();
+                } else {
+                    // subkernel filename must be in format:
+                    // "<subkernel id> <destination>.elf"
+                    let filename = entry.filename();
+                    let mut iter = filename.as_str().split_whitespace();
+                    let sid: u32 = iter.next().unwrap().parse().unwrap();
+                    let dest: u8 = iter.next().unwrap().strip_suffix(".elf").unwrap().parse().unwrap();
+                    let up = _up_destinations.borrow()[dest as usize];
+                    if up {
+                        let subkernel_lib = entry.data().to_vec();
+                        subkernel::add_subkernel(sid, dest, subkernel_lib).await;
+                        match subkernel::upload(_aux_mutex, _routing_table, _timer, sid).await {
+                            Ok(_) => (),
+                            Err(_) => return Err(Error::UnexpectedPattern),
+                        }
+                    } else {
+                        return Err(Error::DestinationDown);
+                    }
+                }
+            }
+            load_kernel(&main_lib, control, None).await
+        }
+        #[cfg(not(has_drtio))]
+        {
+            panic!("multi-kernel libraries are not supported in standalone systems");
+        }
+    }
+}
 
-async fn load_kernel(buffer: &Vec<u8>, control: &Rc<RefCell<kernel::Control>>, stream: Option<&TcpStream>) -> Result<()> {
+async fn load_kernel(
+    buffer: &Vec<u8>,
+    control: &Rc<RefCell<kernel::Control>>,
+    stream: Option<&TcpStream>,
+) -> Result<()> {
     let mut control = control.borrow_mut();
     control.restart();
-    control.tx.async_send(kernel::Message::LoadRequest(buffer.to_vec())).await;
+    control
+        .tx
+        .async_send(kernel::Message::LoadRequest(buffer.to_vec()))
+        .await;
     let reply = control.rx.async_recv().await;
     match reply {
         kernel::Message::LoadCompleted => {
@@ -315,7 +622,7 @@ async fn load_kernel(buffer: &Vec<u8>, control: &Rc<RefCell<kernel::Control>>, s
                 write_header(stream, Reply::LoadCompleted).await?;
             }
             Ok(())
-        },
+        }
         kernel::Message::LoadFailed => {
             if let Some(stream) = stream {
                 write_header(stream, Reply::LoadFailed).await?;
@@ -324,7 +631,7 @@ async fn load_kernel(buffer: &Vec<u8>, control: &Rc<RefCell<kernel::Control>>, s
                 error!("Kernel load failed");
             }
             Err(Error::UnexpectedPattern)
-        },
+        }
         _ => {
             error!("unexpected message from core1: {:?}", reply);
             if let Some(stream) = stream {
@@ -336,16 +643,27 @@ async fn load_kernel(buffer: &Vec<u8>, control: &Rc<RefCell<kernel::Control>>, s
     }
 }
 
-async fn handle_connection(stream: &mut TcpStream, control: Rc<RefCell<kernel::Control>>, up_destinations: &Rc<RefCell<[bool; drtio_routing::DEST_COUNT]>>) -> Result<()> {
+async fn handle_connection(
+    stream: &mut TcpStream,
+    control: Rc<RefCell<kernel::Control>>,
+    up_destinations: &Rc<RefCell<[bool; drtio_routing::DEST_COUNT]>>,
+    aux_mutex: &Rc<Mutex<bool>>,
+    routing_table: &drtio_routing::RoutingTable,
+    timer: GlobalTimer,
+) -> Result<()> {
     stream.set_ack_delay(None);
 
     if !expect(stream, b"ARTIQ coredev\n").await? {
         return Err(Error::UnexpectedPattern);
     }
     stream.send_slice("e".as_bytes()).await?;
+    #[cfg(has_drtio)]
+    subkernel::clear_subkernels().await;
     loop {
         let request = read_request(stream, true).await?;
         if request.is_none() {
+            #[cfg(has_drtio)]
+            subkernel::clear_subkernels().await;
             return Ok(());
         }
         let request = request.unwrap();
@@ -353,17 +671,48 @@ async fn handle_connection(stream: &mut TcpStream, control: Rc<RefCell<kernel::C
             Request::SystemInfo => {
                 write_header(stream, Reply::SystemInfo).await?;
                 stream.send_slice("ARZQ".as_bytes()).await?;
-            },
+            }
             Request::LoadKernel => {
-                let buffer = read_bytes(stream, 1024*1024).await?;
+                let buffer = read_bytes(stream, 1024 * 1024).await?;
                 load_kernel(&buffer, &control, Some(stream)).await?;
-            },
+            }
             Request::RunKernel => {
-                handle_run_kernel(Some(stream), &control, &up_destinations).await?;
+                handle_run_kernel(
-            },
+                    Some(stream),
+                    &control,
+                    &up_destinations,
+                    aux_mutex,
+                    routing_table,
+                    timer,
+                )
+                .await?;
+            }
+            Request::UploadSubkernel => {
+                #[cfg(has_drtio)]
+                {
+                    let id = read_i32(stream).await? as u32;
+                    let destination = read_i8(stream).await? as u8;
+                    let buffer = read_bytes(stream, 1024 * 1024).await?;
+                    subkernel::add_subkernel(id, destination, buffer).await;
+                    match subkernel::upload(aux_mutex, routing_table, timer, id).await {
+                        Ok(_) => write_header(stream, Reply::LoadCompleted).await?,
+                        Err(_) => {
+                            write_header(stream, Reply::LoadFailed).await?;
+                            write_chunk(stream, b"subkernel failed to load").await?;
+                            return Err(Error::UnexpectedPattern);
+                        }
+                    }
+                }
+                #[cfg(not(has_drtio))]
+                {
+                    write_header(stream, Reply::LoadFailed).await?;
+                    write_chunk(stream, b"No DRTIO on this system, subkernels are not supported").await?;
+                    return Err(Error::UnexpectedPattern);
+                }
+            }
             _ => {
                 error!("unexpected request from host: {:?}", request);
-                return Err(Error::UnrecognizedPacket)
+                return Err(Error::UnrecognizedPacket);
             }
         }
     }
@@ -387,34 +736,35 @@ pub fn main(timer: GlobalTimer, cfg: Config) {
             let ip_addrs = [
                 IpCidr::new(net_addresses.ipv4_addr, 0),
                 IpCidr::new(net_addresses.ipv6_ll_addr, 0),
-                IpCidr::new(addr, 0)
+                IpCidr::new(addr, 0),
             ];
             EthernetInterfaceBuilder::new(&mut eth)
-                       .ethernet_addr(net_addresses.hardware_addr)
+                .ethernet_addr(net_addresses.hardware_addr)
-                       .ip_addrs(ip_addrs)
+                .ip_addrs(ip_addrs)
-                       .neighbor_cache(neighbor_cache)
+                .neighbor_cache(neighbor_cache)
-                       .finalize()
+                .finalize()
         }
         None => {
             let ip_addrs = [
                 IpCidr::new(net_addresses.ipv4_addr, 0),
-                IpCidr::new(net_addresses.ipv6_ll_addr, 0)
+                IpCidr::new(net_addresses.ipv6_ll_addr, 0),
             ];
             EthernetInterfaceBuilder::new(&mut eth)
-                       .ethernet_addr(net_addresses.hardware_addr)
+                .ethernet_addr(net_addresses.hardware_addr)
-                       .ip_addrs(ip_addrs)
+                .ip_addrs(ip_addrs)
-                       .neighbor_cache(neighbor_cache)
+                .neighbor_cache(neighbor_cache)
-                       .finalize()
+                .finalize()
         }
     };
 
     Sockets::init(32);
 
-    // before, mutex was on io, but now that io isn't used...?
     let aux_mutex: Rc<Mutex<bool>> = Rc::new(Mutex::new(false));
     #[cfg(has_drtio)]
-    let drtio_routing_table = Rc::new(RefCell::new(
+    let drtio_routing_table = Rc::new(RefCell::new(drtio_routing::config_routing_table(
-        drtio_routing::config_routing_table(pl::csr::DRTIO.len(), &cfg)));
+        pl::csr::DRTIO.len(),
+        &cfg,
+    )));
     #[cfg(not(has_drtio))]
     let drtio_routing_table = Rc::new(RefCell::new(drtio_routing::RoutingTable::default_empty()));
     let up_destinations = Rc::new(RefCell::new([false; drtio_routing::DEST_COUNT]));
@@ -422,17 +772,33 @@ pub fn main(timer: GlobalTimer, cfg: Config) {
     drtio_routing::interconnect_disable_all();
 
     rtio_mgt::startup(&aux_mutex, &drtio_routing_table, &up_destinations, timer);
+    ksupport::setup_device_map(&cfg);
 
-    analyzer::start();
+    analyzer::start(&aux_mutex, &drtio_routing_table, &up_destinations, timer);
-    moninj::start(timer, aux_mutex, drtio_routing_table);
+    moninj::start(timer, &aux_mutex, &drtio_routing_table);
 
     let control: Rc<RefCell<kernel::Control>> = Rc::new(RefCell::new(kernel::Control::start()));
     let idle_kernel = Rc::new(cfg.read("idle_kernel").ok());
     if let Ok(buffer) = cfg.read("startup_kernel") {
         info!("Loading startup kernel...");
-        if let Ok(()) = task::block_on(load_kernel(&buffer, &control, None)) {
+        let routing_table = drtio_routing_table.borrow();
+        if let Ok(()) = task::block_on(handle_flash_kernel(
+            &buffer,
+            &control,
+            &up_destinations,
+            &aux_mutex,
+            &routing_table,
+            timer,
+        )) {
             info!("Starting startup kernel...");
-            let _ = task::block_on(handle_run_kernel(None, &control, &up_destinations));
+            let _ = task::block_on(handle_run_kernel(
+                None,
+                &control,
+                &up_destinations,
+                &aux_mutex,
+                &routing_table,
+                timer,
+            ));
             info!("Startup kernel finished!");
         } else {
             error!("Error loading startup kernel!");
@@ -458,21 +824,33 @@ pub fn main(timer: GlobalTimer, cfg: Config) {
             let connection = connection.clone();
             let terminate = terminate.clone();
             let up_destinations = up_destinations.clone();
+            let aux_mutex = aux_mutex.clone();
+            let routing_table = drtio_routing_table.clone();
 
             // we make sure the value of terminate is 0 before we start
             let _ = terminate.try_wait();
             task::spawn(async move {
+                let routing_table = routing_table.borrow();
                 select_biased! {
                     _ = (async {
-                        let _ = handle_connection(&mut stream, control.clone(), &up_destinations)
+                        let _ = handle_connection(&mut stream, control.clone(), &up_destinations, &aux_mutex, &routing_table, timer)
                             .await
                             .map_err(|e| warn!("connection terminated: {}", e));
                         if let Some(buffer) = &*idle_kernel {
                             info!("Loading idle kernel");
-                            let _ = load_kernel(&buffer, &control, None)
+                            let res = handle_flash_kernel(&buffer, &control, &up_destinations,  &aux_mutex, &routing_table, timer)
-                                .await.map_err(|_| warn!("error loading idle kernel"));
+                                .await;
+                            match res {
+                                #[cfg(has_drtio)]
+                                Err(Error::DestinationDown) => {
+                                    let mut countdown = timer.countdown();
+                                    delay(&mut countdown, Milliseconds(500)).await;
+                                }
+                                Err(_) => warn!("error loading idle kernel"),
+                                _ => (),
+                            }
                             info!("Running idle kernel");
-                            let _ = handle_run_kernel(None, &control, &up_destinations)
+                            let _ = handle_run_kernel(None, &control, &up_destinations, &aux_mutex, &routing_table, timer)
                                 .await.map_err(|_| warn!("error running idle kernel"));
                             info!("Idle kernel terminated");
                         }
@@ -486,7 +864,59 @@ pub fn main(timer: GlobalTimer, cfg: Config) {
         }
     });
 
-    Sockets::run(&mut iface, || {
+    Sockets::run(&mut iface, || Instant::from_millis(timer.get_time().0 as i32));
-        Instant::from_millis(timer.get_time().0 as i32)
+}
-    });
+
+pub fn soft_panic_main(timer: GlobalTimer, cfg: Config) -> ! {
+    let net_addresses = net_settings::get_addresses(&cfg);
+    info!("network addresses: {}", net_addresses);
+
+    let eth = zynq::eth::Eth::eth0(net_addresses.hardware_addr.0.clone());
+    const RX_LEN: usize = 64;
+    // Number of transmission buffers (minimum is two because with
+    // one, duplicate packet transmission occurs)
+    const TX_LEN: usize = 64;
+    let eth = eth.start_rx(RX_LEN);
+    let mut eth = eth.start_tx(TX_LEN);
+
+    let neighbor_cache = NeighborCache::new(alloc::collections::BTreeMap::new());
+    let mut iface = match net_addresses.ipv6_addr {
+        Some(addr) => {
+            let ip_addrs = [
+                IpCidr::new(net_addresses.ipv4_addr, 0),
+                IpCidr::new(net_addresses.ipv6_ll_addr, 0),
+                IpCidr::new(addr, 0),
+            ];
+            EthernetInterfaceBuilder::new(&mut eth)
+                .ethernet_addr(net_addresses.hardware_addr)
+                .ip_addrs(ip_addrs)
+                .neighbor_cache(neighbor_cache)
+                .finalize()
+        }
+        None => {
+            let ip_addrs = [
+                IpCidr::new(net_addresses.ipv4_addr, 0),
+                IpCidr::new(net_addresses.ipv6_ll_addr, 0),
+            ];
+            EthernetInterfaceBuilder::new(&mut eth)
+                .ethernet_addr(net_addresses.hardware_addr)
+                .ip_addrs(ip_addrs)
+                .neighbor_cache(neighbor_cache)
+                .finalize()
+        }
+    };
+
+    Sockets::init(32);
+
+    mgmt::start(cfg);
+
+    // getting eth settings disables the LED as it resets GPIO
+    // need to re-enable it here
+    #[cfg(feature = "target_kasli_soc")]
+    {
+        let mut err_led = ErrorLED::error_led();
+        err_led.toggle(true);
+    }
+
+    Sockets::run(&mut iface, || Instant::from_millis(timer.get_time().0 as i32));
 }

src/runtime/src/kernel/dma.rs

@@ -1,212 +0,0 @@
-use crate::{
-    pl::csr,
-    artiq_raise,
-    rtio,
-};
-use alloc::{vec::Vec, string::String, boxed::Box};
-use cslice::CSlice;
-use super::{KERNEL_IMAGE, KERNEL_CHANNEL_0TO1, KERNEL_CHANNEL_1TO0, Message};
-use core::mem;
-
-use libcortex_a9::cache::dcci_slice;
-
-const ALIGNMENT: usize = 16 * 8;
-
-#[repr(C)]
-pub struct DmaTrace {
-    duration: i64,
-    address:  i32,
-}
-
-#[derive(Clone, Debug)]
-pub struct DmaRecorder {
-    pub name: String,
-    pub buffer: Vec<u8>,
-    pub duration: i64,
-}
-
-static mut RECORDER: Option<DmaRecorder> = None;
-
-pub unsafe fn init_dma_recorder() {
-    // as static would remain after restart, we have to reset it,
-    // without running its destructor.
-    mem::forget(mem::replace(&mut RECORDER, None));
-}
-
-pub extern fn dma_record_start(name: CSlice<u8>) {
-    let name = String::from_utf8(name.as_ref().to_vec()).unwrap();
-    unsafe {
-        KERNEL_CHANNEL_1TO0.as_mut().unwrap().send(Message::DmaEraseRequest(name.clone()));
-    }
-    unsafe {
-        if RECORDER.is_some() {
-            artiq_raise!("DMAError", "DMA is already recording")
-        }
-
-        let library = KERNEL_IMAGE.as_ref().unwrap();
-        library.rebind(b"rtio_output",
-                       dma_record_output as *const ()).unwrap();
-        library.rebind(b"rtio_output_wide",
-                       dma_record_output_wide as *const ()).unwrap();
-
-        RECORDER = Some(DmaRecorder {
-            name,
-            buffer: Vec::new(),
-            duration: 0,
-        });
-    }
-}
-
-pub extern fn dma_record_stop(duration: i64) {
-    unsafe {
-        if RECORDER.is_none() {
-            artiq_raise!("DMAError", "DMA is not recording")
-        }
-
-        let library = KERNEL_IMAGE.as_ref().unwrap();
-        library.rebind(b"rtio_output",
-                       rtio::output as *const ()).unwrap();
-        library.rebind(b"rtio_output_wide",
-                       rtio::output_wide as *const ()).unwrap();
-
-        let mut recorder = RECORDER.take().unwrap();
-        recorder.duration = duration;
-        KERNEL_CHANNEL_1TO0.as_mut().unwrap().send(
-            Message::DmaPutRequest(recorder)
-        );
-    }
-}
-
-#[inline(always)]
-unsafe fn dma_record_output_prepare(timestamp: i64, target: i32,
-                                    words: usize) {
-    // See gateware/rtio/dma.py.
-    const HEADER_LENGTH: usize = /*length*/1 + /*channel*/3 + /*timestamp*/8 + /*address*/1;
-    let length = HEADER_LENGTH + /*data*/words * 4;
-
-    let buffer = &mut RECORDER.as_mut().unwrap().buffer;
-    buffer.reserve(length);
-    buffer.extend_from_slice(&[
-        (length    >>  0) as u8,
-        (target    >>  8) as u8,
-        (target    >>  16) as u8,
-        (target    >>  24) as u8,
-        (timestamp >>  0) as u8,
-        (timestamp >>  8) as u8,
-        (timestamp >> 16) as u8,
-        (timestamp >> 24) as u8,
-        (timestamp >> 32) as u8,
-        (timestamp >> 40) as u8,
-        (timestamp >> 48) as u8,
-        (timestamp >> 56) as u8,
-        (target    >>  0) as u8,
-    ]);
-}
-
-pub extern fn dma_record_output(target: i32, word: i32) {
-    unsafe {
-        let timestamp = rtio::now_mu();
-        dma_record_output_prepare(timestamp, target, 1);
-        RECORDER.as_mut().unwrap().buffer.extend_from_slice(&[
-            (word >>  0) as u8,
-            (word >>  8) as u8,
-            (word >> 16) as u8,
-            (word >> 24) as u8,
-        ]);
-    }
-}
-
-pub extern fn dma_record_output_wide(target: i32, words: &CSlice<i32>) {
-    assert!(words.len() <= 16); // enforce the hardware limit
-
-    unsafe {
-        let timestamp = rtio::now_mu();
-        dma_record_output_prepare(timestamp, target, words.len());
-        let buffer = &mut RECORDER.as_mut().unwrap().buffer;
-        for word in words.as_ref().iter() {
-            buffer.extend_from_slice(&[
-                (word >>  0) as u8,
-                (word >>  8) as u8,
-                (word >> 16) as u8,
-                (word >> 24) as u8,
-            ]);
-        }
-    }
-}
-
-pub extern fn dma_erase(name: CSlice<u8>) {
-    let name = String::from_utf8(name.as_ref().to_vec()).unwrap();
-    unsafe {
-        KERNEL_CHANNEL_1TO0.as_mut().unwrap().send(Message::DmaEraseRequest(name));
-    }
-}
-
-pub extern fn dma_retrieve(name: CSlice<u8>) -> DmaTrace {
-    let name = String::from_utf8(name.as_ref().to_vec()).unwrap();
-    unsafe {
-        KERNEL_CHANNEL_1TO0.as_mut().unwrap().send(Message::DmaGetRequest(name));
-    }
-    match unsafe {KERNEL_CHANNEL_0TO1.as_mut().unwrap()}.recv() {
-        Message::DmaGetReply(None) => (),
-        Message::DmaGetReply(Some((mut v, duration))) => {
-            v.reserve(ALIGNMENT - 1);
-            let original_length = v.len();
-            let padding = ALIGNMENT - v.as_ptr() as usize % ALIGNMENT;
-            let padding = if padding == ALIGNMENT { 0 } else { padding };
-            for _ in 0..padding {
-                v.push(0);
-            }
-            // trailing zero to indicate end of buffer
-            v.push(0);
-            v.copy_within(0..original_length, padding);
-            dcci_slice(&v);
-            let v = Box::new(v);
-            let address = Box::into_raw(v) as *mut Vec<u8> as i32;
-            return DmaTrace {
-                address,
-                duration,
-            };
-        },
-        _ => panic!("Expected DmaGetReply after DmaGetRequest!"),
-    }
-    // we have to defer raising error as we have to drop the message first...
-    artiq_raise!("DMAError", "DMA trace not found");
-}
-
-pub extern fn dma_playback(timestamp: i64, ptr: i32) {
-    unsafe {
-        let v = Box::from_raw(ptr as *mut Vec<u8>);
-        let padding = ALIGNMENT - v.as_ptr() as usize % ALIGNMENT;
-        let padding = if padding == ALIGNMENT { 0 } else { padding };
-        let ptr = v.as_ptr().add(padding) as i32;
-
-        csr::rtio_dma::base_address_write(ptr as u32);
-        csr::rtio_dma::time_offset_write(timestamp as u64);
-
-        csr::cri_con::selected_write(1);
-        csr::rtio_dma::enable_write(1);
-        while csr::rtio_dma::enable_read() != 0 {}
-        csr::cri_con::selected_write(0);
-
-        // leave the handle as we may try to do playback for another time.
-        mem::forget(v);
-
-        let error = csr::rtio_dma::error_read();
-        if error != 0 {
-            let timestamp = csr::rtio_dma::error_timestamp_read();
-            let channel = csr::rtio_dma::error_channel_read();
-            csr::rtio_dma::error_write(1);
-            if error & 1 != 0 {
-                artiq_raise!("RTIOUnderflow",
-                    "RTIO underflow at {0} mu, channel {1}",
-                    timestamp as i64, channel as i64, 0);
-            }
-            if error & 2 != 0 {
-                artiq_raise!("RTIODestinationUnreachable",
-                    "RTIO destination unreachable, output, at {0} mu, channel {1}",
-                    timestamp as i64, channel as i64, 0);
-            }
-        }
-    }
-}
-

src/runtime/src/kernel/mod.rs

@@ -1,67 +0,0 @@
-use core::ptr;
-use alloc::{vec::Vec, string::String};
-
-use libcortex_a9::{mutex::Mutex, sync_channel, semaphore::Semaphore};
-use crate::eh_artiq;
-
-mod control;
-pub use control::Control;
-pub mod core1;
-mod api;
-mod rpc;
-mod dma;
-pub use dma::DmaRecorder;
-mod cache;
-
-#[derive(Debug, Clone)]
-pub struct RPCException {
-    pub id: u32,
-    pub message: u32,
-    pub param: [i64; 3],
-    pub file: u32,
-    pub line: i32,
-    pub column: i32,
-    pub function: u32
-}
-
-#[derive(Debug, Clone)]
-pub enum Message {
-    LoadRequest(Vec<u8>),
-    LoadCompleted,
-    LoadFailed,
-    StartRequest,
-    KernelFinished(u8),
-    KernelException(&'static [Option<eh_artiq::Exception<'static>>],
-                    &'static [eh_artiq::StackPointerBacktrace],
-                    &'static [(usize, usize)],
-                    u8),
-    RpcSend { is_async: bool, data: Vec<u8> },
-    RpcRecvRequest(*mut ()),
-    RpcRecvReply(Result<usize, RPCException>),
-
-    CacheGetRequest(String),
-    CacheGetReply(Vec<i32>),
-    CachePutRequest(String, Vec<i32>),
-
-    DmaPutRequest(DmaRecorder),
-    DmaEraseRequest(String),
-    DmaGetRequest(String),
-    DmaGetReply(Option<(Vec<u8>, i64)>),
-
-    #[cfg(has_drtio)]
-    UpDestinationsRequest(i32),
-    #[cfg(has_drtio)]
-    UpDestinationsReply(bool),
-}
-
-static CHANNEL_0TO1: Mutex<Option<sync_channel::Sender<'static, Message>>> = Mutex::new(None);
-static CHANNEL_1TO0: Mutex<Option<sync_channel::Receiver<'static, Message>>> = Mutex::new(None);
-static CHANNEL_SEM: Semaphore = Semaphore::new(0, 1);
-
-static mut KERNEL_CHANNEL_0TO1: Option<sync_channel::Receiver<'static, Message>> = None;
-static mut KERNEL_CHANNEL_1TO0: Option<sync_channel::Sender<'static, Message>> = None;
-
-pub static mut KERNEL_IMAGE: *const core1::KernelImage = ptr::null();
-
-static INIT_LOCK: Mutex<()> = Mutex::new(());
-

src/runtime/src/main.rs

@@ -1,99 +1,49 @@
 #![no_std]
 #![no_main]
-#![recursion_limit="1024"]  // for futures_util::select!
+#![recursion_limit = "1024"] // for futures_util::select!
 #![feature(alloc_error_handler)]
-#![feature(panic_info_message)]
-#![feature(c_variadic)]
 #![feature(const_btree_new)]
-#![feature(const_in_array_repeat_expressions)]
+#![feature(panic_info_message)]
-#![feature(naked_functions)]
-#![feature(asm)]
 
+#[macro_use]
 extern crate alloc;
 
-#[cfg(all(feature = "target_kasli_soc", has_drtio))]
+#[cfg(all(feature = "target_kasli_soc", has_virtual_leds))]
 use core::cell::RefCell;
-use log::{info, warn, error};
 
-use libboard_zynq::{timer::GlobalTimer, mpcore, gic};
+use ksupport;
-use libasync::{task, block_async};
+use libasync::task;
-use libsupport_zynq::ram;
+#[cfg(has_drtio_eem)]
-use nb;
+use libboard_artiq::drtio_eem;
-use void::Void;
+#[cfg(feature = "target_kasli_soc")]
+use libboard_artiq::io_expander;
+use libboard_artiq::{identifier_read, logger, pl};
+use libboard_zynq::{gic, mpcore, timer::GlobalTimer};
 use libconfig::Config;
 use libcortex_a9::l2c::enable_l2_cache;
-use libboard_artiq::{logger, identifier_read, init_gateware, pl};
+use libsupport_zynq::{exception_vectors, ram};
-#[cfg(all(feature = "target_kasli_soc", has_drtio))]
+use log::{info, warn};
-use libboard_artiq::io_expander;
 
-const ASYNC_ERROR_COLLISION: u8 = 1 << 0;
-const ASYNC_ERROR_BUSY: u8 = 1 << 1;
-const ASYNC_ERROR_SEQUENCE_ERROR: u8 = 1 << 2;
-
-mod proto_async;
-mod comms;
-mod rpc;
-#[cfg(ki_impl = "csr")]
-#[path = "rtio_csr.rs"]
-mod rtio;
-#[cfg(ki_impl = "acp")]
-#[path = "rtio_acp.rs"]
-mod rtio;
-mod rtio_mgt;
-mod rtio_clocking;
-mod kernel;
-mod moninj;
-mod eh_artiq;
-mod panic;
-mod mgmt;
 mod analyzer;
-mod irq;
+mod comms;
-mod i2c;
 
-static mut SEEN_ASYNC_ERRORS: u8 = 0;
+mod mgmt;
+mod moninj;
+mod panic;
+mod proto_async;
+mod rpc_async;
+mod rtio_clocking;
+mod rtio_dma;
+mod rtio_mgt;
+#[cfg(has_drtio)]
+mod subkernel;
 
-pub unsafe fn get_async_errors() -> u8 {
+// linker symbols
-    let errors = SEEN_ASYNC_ERRORS;
+extern "C" {
-    SEEN_ASYNC_ERRORS = 0;
+    static __exceptions_start: u32;
-    errors
 }
 
-fn wait_for_async_rtio_error() -> nb::Result<(), Void> {
+#[cfg(all(feature = "target_kasli_soc", has_virtual_leds))]
-    unsafe {
-        if pl::csr::rtio_core::async_error_read() != 0 {
-            Ok(())
-        } else {
-            Err(nb::Error::WouldBlock)
-        }
-    }
-}
-
-async fn report_async_rtio_errors() {
-    loop {
-        let _ = block_async!(wait_for_async_rtio_error()).await;
-        unsafe {
-            let errors = pl::csr::rtio_core::async_error_read();
-            if errors & ASYNC_ERROR_COLLISION != 0 {
-                error!("RTIO collision involving channel {}",
-                       pl::csr::rtio_core::collision_channel_read());
-            }
-            if errors & ASYNC_ERROR_BUSY != 0 {
-                error!("RTIO busy error involving channel {}",
-                       pl::csr::rtio_core::busy_channel_read());
-            }
-            if errors & ASYNC_ERROR_SEQUENCE_ERROR != 0 {
-                error!("RTIO sequence error involving channel {}",
-                       pl::csr::rtio_core::sequence_error_channel_read());
-            }
-            SEEN_ASYNC_ERRORS = errors;
-            pl::csr::rtio_core::async_error_write(errors);
-        }
-    }
-}
-
-
-
-#[cfg(all(feature = "target_kasli_soc", has_drtio))]
 async fn io_expanders_service(
     i2c_bus: RefCell<&mut libboard_zynq::i2c::I2c>,
     io_expander0: RefCell<io_expander::IoExpander>,
@@ -111,11 +61,13 @@ async fn io_expanders_service(
             .expect("I2C I/O expander #1 service failed");
     }
 }
+
 #[cfg(has_grabber)]
 mod grabber {
     use libasync::delay;
     use libboard_artiq::grabber;
     use libboard_zynq::time::Milliseconds;
+
     use crate::GlobalTimer;
     pub async fn grabber_thread(timer: GlobalTimer) {
         let mut countdown = timer.countdown();
@@ -126,16 +78,17 @@ mod grabber {
     }
 }
 
-static mut LOG_BUFFER: [u8; 1<<17] = [0; 1<<17];
+static mut LOG_BUFFER: [u8; 1 << 17] = [0; 1 << 17];
 
 #[no_mangle]
 pub fn main_core0() {
+    unsafe {
+        exception_vectors::set_vector_table(&__exceptions_start as *const u32 as u32);
+    }
     enable_l2_cache(0x8);
     let mut timer = GlobalTimer::start();
 
-    let buffer_logger = unsafe {
+    let buffer_logger = unsafe { logger::BufferLogger::new(&mut LOG_BUFFER[..]) };
-        logger::BufferLogger::new(&mut LOG_BUFFER[..])
-    };
     buffer_logger.set_uart_log_level(log::LevelFilter::Info);
     buffer_logger.register();
     log::set_max_level(log::LevelFilter::Info);
@@ -145,13 +98,12 @@ pub fn main_core0() {
     ram::init_alloc_core0();
     gic::InterruptController::gic(mpcore::RegisterBlock::mpcore()).enable_interrupts();
 
-    init_gateware();
     info!("gateware ident: {}", identifier_read(&mut [0; 64]));
 
-    i2c::init();
+    ksupport::i2c::init();
-    #[cfg(all(feature = "target_kasli_soc", has_drtio))]
+    #[cfg(feature = "target_kasli_soc")]
     {
-        let i2c_bus = unsafe { (i2c::I2C_BUS).as_mut().unwrap() };
+        let i2c_bus = unsafe { (ksupport::i2c::I2C_BUS).as_mut().unwrap() };
         let mut io_expander0 = io_expander::IoExpander::new(i2c_bus, 0).unwrap();
         let mut io_expander1 = io_expander::IoExpander::new(i2c_bus, 1).unwrap();
         io_expander0
@@ -160,6 +112,11 @@ pub fn main_core0() {
         io_expander1
             .init(i2c_bus)
             .expect("I2C I/O expander #1 initialization failed");
+
+        // Drive CLK_SEL to true
+        #[cfg(has_si549)]
+        io_expander0.set(1, 7, true);
+
         // Drive TX_DISABLE to false on SFP0..3
         io_expander0.set(0, 1, false);
         io_expander1.set(0, 1, false);
@@ -167,6 +124,7 @@ pub fn main_core0() {
         io_expander1.set(1, 1, false);
         io_expander0.service(i2c_bus).unwrap();
         io_expander1.service(i2c_bus).unwrap();
+        #[cfg(has_virtual_leds)]
         task::spawn(io_expanders_service(
             RefCell::new(i2c_bus),
             RefCell::new(io_expander0),
@@ -184,10 +142,13 @@ pub fn main_core0() {
 
     rtio_clocking::init(&mut timer, &cfg);
 
-    task::spawn(report_async_rtio_errors());
+    #[cfg(has_drtio_eem)]
+    drtio_eem::init(&mut timer, &cfg);
 
     #[cfg(has_grabber)]
     task::spawn(grabber::grabber_thread(timer));
 
+    task::spawn(ksupport::report_async_rtio_errors());
+
     comms::main(timer, cfg);
 }

src/runtime/src/mgmt.rs

@@ -1,16 +1,17 @@
+use alloc::{rc::Rc, string::String, vec::Vec};
+use core::cell::RefCell;
+
 use futures::{future::poll_fn, task::Poll};
 use libasync::{smoltcp::TcpStream, task};
-use libboard_zynq::smoltcp;
-use libconfig::Config;
-use core::cell::RefCell;
-use alloc::{rc::Rc, vec::Vec, string::String};
-use log::{self, info, debug, warn, error, LevelFilter};
-
 use libboard_artiq::logger::{BufferLogger, LogBufferRef};
-use crate::proto_async::*;
+use libboard_zynq::{slcr, smoltcp};
+use libconfig::Config;
+use log::{self, debug, error, info, warn, LevelFilter};
 use num_derive::FromPrimitive;
 use num_traits::FromPrimitive;
 
+use crate::proto_async::*;
+
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub enum Error {
     NetworkError(smoltcp::Error),
@@ -24,10 +25,10 @@ type Result<T> = core::result::Result<T, Error>;
 impl core::fmt::Display for Error {
     fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result {
         match self {
-            &Error::NetworkError(error)  => write!(f, "network error: {}", error),
+            &Error::NetworkError(error) => write!(f, "network error: {}", error),
             &Error::UnknownLogLevel(lvl) => write!(f, "unknown log level {}", lvl),
-            &Error::UnexpectedPattern    => write!(f, "unexpected pattern"),
+            &Error::UnexpectedPattern => write!(f, "unexpected pattern"),
-            &Error::UnrecognizedPacket   => write!(f, "unrecognized packet"),
+            &Error::UnrecognizedPacket => write!(f, "unrecognized packet"),
         }
     }
 }
@@ -44,6 +45,7 @@ pub enum Request {
     ClearLog = 2,
     PullLog = 7,
     SetLogFilter = 3,
+    Reboot = 5,
     SetUartLogFilter = 6,
 
     ConfigRead = 12,
@@ -55,6 +57,7 @@ pub enum Request {
 pub enum Reply {
     Success = 1,
     LogContent = 2,
+    RebootImminent = 3,
     Error = 6,
     ConfigData = 7,
 }
@@ -72,9 +75,7 @@ async fn read_log_level_filter(stream: &mut TcpStream) -> Result<log::LevelFilte
 }
 
 async fn get_logger_buffer_pred<F>(f: F) -> LogBufferRef<'static>
-where
+where F: Fn(&LogBufferRef) -> bool {
-    F: Fn(&LogBufferRef) -> bool,
-{
     poll_fn(|ctx| {
         let logger = unsafe { BufferLogger::get_logger().as_mut().unwrap() };
         match logger.buffer() {
@@ -110,10 +111,7 @@ async fn read_key(stream: &mut TcpStream) -> Result<String> {
     Ok(String::from_utf8(buffer).unwrap())
 }
 
-async fn handle_connection(
+async fn handle_connection(stream: &mut TcpStream, pull_id: Rc<RefCell<u32>>, cfg: Rc<Config>) -> Result<()> {
-    stream: &mut TcpStream,
-    pull_id: Rc<RefCell<u32>>,
-    cfg: Rc<Config>) -> Result<()> {
     if !expect(&stream, b"ARTIQ management\n").await? {
         return Err(Error::UnexpectedPattern);
     }
@@ -161,7 +159,7 @@ async fn handle_connection(
                         logger.set_buffer_log_level(LevelFilter::Trace);
                     }
                 }
-            },
+            }
             Request::SetLogFilter => {
                 let lvl = read_log_level_filter(stream).await?;
                 info!("Changing log level to {}", lvl);
@@ -172,10 +170,7 @@ async fn handle_connection(
                 let lvl = read_log_level_filter(stream).await?;
                 info!("Changing UART log level to {}", lvl);
                 unsafe {
-                    BufferLogger::get_logger()
+                    BufferLogger::get_logger().as_ref().unwrap().set_uart_log_level(lvl);
-                        .as_ref()
-                        .unwrap()
-                        .set_uart_log_level(lvl);
                 }
                 write_i8(stream, Reply::Success as i8).await?;
             }
@@ -191,16 +186,12 @@ async fn handle_connection(
                     warn!("read error: no such key");
                     write_i8(stream, Reply::Error as i8).await?;
                 }
-            },
+            }
             Request::ConfigWrite => {
                 let key = read_key(stream).await?;
                 debug!("write key: {}", key);
                 let len = read_i32(stream).await?;
-                let len = if len <= 0 {
+                let len = if len <= 0 { 0 } else { len as usize };
-                    0
-                } else {
-                    len as usize
-                };
                 let mut buffer = Vec::with_capacity(len);
                 unsafe {
                     buffer.set_len(len);
@@ -215,7 +206,7 @@ async fn handle_connection(
                     error!("failed to write: {:?}", value);
                     write_i8(stream, Reply::Error as i8).await?;
                 }
-            },
+            }
             Request::ConfigRemove => {
                 let key = read_key(stream).await?;
                 debug!("erase key: {}", key);
@@ -228,6 +219,12 @@ async fn handle_connection(
                     write_i8(stream, Reply::Error as i8).await?;
                 }
             }
+            Request::Reboot => {
+                info!("rebooting");
+                write_i8(stream, Reply::RebootImminent as i8).await?;
+                stream.flush().await?;
+                slcr::reboot();
+            }
         }
     }
 }

src/runtime/src/moninj.rs

@@ -1,26 +1,23 @@
-use core::{fmt, cell::RefCell};
 use alloc::{collections::BTreeMap, rc::Rc};
-use log::{debug, info, warn};
+use core::{cell::RefCell, fmt};
-use void::Void;
 
+use futures::{pin_mut, select_biased, FutureExt};
+use libasync::{block_async, nb, smoltcp::TcpStream, task};
 use libboard_artiq::drtio_routing;
-
+use libboard_zynq::{smoltcp, time::Milliseconds, timer::GlobalTimer};
-use libboard_zynq::{smoltcp, timer::GlobalTimer, time::Milliseconds};
-use libasync::{task, smoltcp::TcpStream, block_async, nb};
 use libcortex_a9::mutex::Mutex;
-
+use log::{debug, info, warn};
 use num_derive::{FromPrimitive, ToPrimitive};
 use num_traits::{FromPrimitive, ToPrimitive};
-use futures::{pin_mut, select_biased, FutureExt};
+use void::Void;
 
 use crate::proto_async::*;
 
-
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub enum Error {
     NetworkError(smoltcp::Error),
     UnexpectedPattern,
-    UnrecognizedPacket
+    UnrecognizedPacket,
 }
 
 pub type Result<T> = core::result::Result<T, Error>;
@@ -29,8 +26,8 @@ impl fmt::Display for Error {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         match self {
             &Error::NetworkError(error) => write!(f, "network error: {}", error),
-            &Error::UnexpectedPattern   => write!(f, "unexpected pattern"),
+            &Error::UnexpectedPattern => write!(f, "unexpected pattern"),
-            &Error::UnrecognizedPacket  => write!(f, "unrecognized packet"),
+            &Error::UnrecognizedPacket => write!(f, "unrecognized packet"),
         }
     }
 }
@@ -46,60 +43,107 @@ enum HostMessage {
     MonitorProbe = 0,
     MonitorInjection = 3,
     Inject = 1,
-    GetInjectionStatus = 2
+    GetInjectionStatus = 2,
 }
 
 #[derive(Debug, FromPrimitive, ToPrimitive)]
 enum DeviceMessage {
     MonitorStatus = 0,
-    InjectionStatus = 1
+    InjectionStatus = 1,
 }
 
 #[cfg(has_drtio)]
 mod remote_moninj {
-    use super::*;
     use libboard_artiq::drtioaux_async;
-    use crate::rtio_mgt::drtio;
     use log::error;
 
-    pub async fn read_probe(aux_mutex: &Rc<Mutex<bool>>, timer: GlobalTimer, linkno: u8, destination: u8, channel: i32, probe: i8) -> i64 {
+    use super::*;
-        let reply = drtio::aux_transact(aux_mutex, linkno, &drtioaux_async::Packet::MonitorRequest { 
+    use crate::rtio_mgt::{drtio, drtio::Error as DrtioError};
-            destination: destination,
+
-            channel: channel as _,
+    pub async fn read_probe(
-            probe: probe as _},
+        aux_mutex: &Rc<Mutex<bool>>,
-            timer).await;
+        routing_table: &drtio_routing::RoutingTable,
+        timer: GlobalTimer,
+        linkno: u8,
+        destination: u8,
+        channel: i32,
+        probe: i8,
+    ) -> i64 {
+        let reply = drtio::aux_transact(
+            aux_mutex,
+            linkno,
+            routing_table,
+            &drtioaux_async::Packet::MonitorRequest {
+                destination: destination,
+                channel: channel as _,
+                probe: probe as _,
+            },
+            timer,
+        )
+        .await;
         match reply {
             Ok(drtioaux_async::Packet::MonitorReply { value }) => return value as i64,
             Ok(packet) => error!("received unexpected aux packet: {:?}", packet),
-            Err("link went down") => { debug!("link is down"); },
+            Err(DrtioError::LinkDown) => {
-            Err(e) => error!("aux packet error ({})", e)
+                warn!("link is down");
+            }
+            Err(e) => error!("aux packet error ({})", e),
         }
         0
     }
 
-    pub async fn inject(aux_mutex: &Rc<Mutex<bool>>, _timer: GlobalTimer, linkno: u8, destination: u8, channel: i32, overrd: i8, value: i8) {
+    pub async fn inject(
+        aux_mutex: &Rc<Mutex<bool>>,
+        _routing_table: &drtio_routing::RoutingTable,
+        _timer: GlobalTimer,
+        linkno: u8,
+        destination: u8,
+        channel: i32,
+        overrd: i8,
+        value: i8,
+    ) {
         let _lock = aux_mutex.lock();
-        drtioaux_async::send(linkno, &drtioaux_async::Packet::InjectionRequest {
+        drtioaux_async::send(
-            destination: destination,
+            linkno,
-            channel: channel as _,
+            &drtioaux_async::Packet::InjectionRequest {
-            overrd: overrd as _,
+                destination: destination,
-            value: value as _
+                channel: channel as _,
-        }).await.unwrap();
+                overrd: overrd as _,
+                value: value as _,
+            },
+        )
+        .await
+        .unwrap();
     }
 
-    pub async fn read_injection_status(aux_mutex: &Rc<Mutex<bool>>, timer: GlobalTimer, linkno: u8, destination: u8, channel: i32, overrd: i8) -> i8 {
+    pub async fn read_injection_status(
-        let reply = drtio::aux_transact(aux_mutex, 
+        aux_mutex: &Rc<Mutex<bool>>,
+        routing_table: &drtio_routing::RoutingTable,
+        timer: GlobalTimer,
+        linkno: u8,
+        destination: u8,
+        channel: i32,
+        overrd: i8,
+    ) -> i8 {
+        let reply = drtio::aux_transact(
+            aux_mutex,
             linkno,
+            routing_table,
             &drtioaux_async::Packet::InjectionStatusRequest {
                 destination: destination,
                 channel: channel as _,
-                overrd: overrd as _},
+                overrd: overrd as _,
-            timer).await;
+            },
+            timer,
+        )
+        .await;
         match reply {
             Ok(drtioaux_async::Packet::InjectionStatusReply { value }) => return value as i8,
             Ok(packet) => error!("received unexpected aux packet: {:?}", packet),
-            Err("link went down") => { debug!("link is down"); },
+            Err(DrtioError::LinkDown) => {
-            Err(e) => error!("aux packet error ({})", e)
+                warn!("link is down");
+            }
+            Err(e) => error!("aux packet error ({})", e),
         }
         0
     }
@@ -144,7 +188,7 @@ macro_rules! dispatch {
             local_moninj::$func(channel.into(), $($param, )*)
         } else {
             let linkno = hop - 1 as u8;
-            remote_moninj::$func($aux_mutex, $timer, linkno, destination, channel, $($param, )*).await
+            remote_moninj::$func($aux_mutex, $routing_table, $timer, linkno, destination, channel, $($param, )*).await
         }
     }}
 }
@@ -157,8 +201,12 @@ macro_rules! dispatch {
     }}
 }
 
-async fn handle_connection(stream: &TcpStream, timer: GlobalTimer, 
+async fn handle_connection(
-        _aux_mutex: &Rc<Mutex<bool>>, _routing_table: &drtio_routing::RoutingTable) -> Result<()> {
+    stream: &TcpStream,
+    timer: GlobalTimer,
+    _aux_mutex: &Rc<Mutex<bool>>,
+    _routing_table: &drtio_routing::RoutingTable,
+) -> Result<()> {
     if !expect(&stream, b"ARTIQ moninj\n").await? {
         return Err(Error::UnexpectedPattern);
     }
@@ -255,7 +303,13 @@ async fn handle_connection(stream: &TcpStream, timer: GlobalTimer,
     }
 }
 
-pub fn start(timer: GlobalTimer, aux_mutex: Rc<Mutex<bool>>, routing_table: Rc<RefCell<drtio_routing::RoutingTable>>) {
+pub fn start(
+    timer: GlobalTimer,
+    aux_mutex: &Rc<Mutex<bool>>,
+    routing_table: &Rc<RefCell<drtio_routing::RoutingTable>>,
+) {
+    let aux_mutex = aux_mutex.clone();
+    let routing_table = routing_table.clone();
     task::spawn(async move {
         loop {
             let aux_mutex = aux_mutex.clone();

src/runtime/src/panic.rs

@@ -1,22 +1,22 @@
-use libboard_zynq::{print, println};
+#[cfg(feature = "target_kasli_soc")]
-use libregister::RegisterR;
+use libboard_zynq::error_led::ErrorLED;
+use libboard_zynq::{print, println, timer::GlobalTimer};
+use libconfig::Config;
 use libcortex_a9::regs::MPIDR;
+use libregister::RegisterR;
+use log::error;
 use unwind::backtrace;
 
+use crate::comms::soft_panic_main;
+
 static mut PANICKED: [bool; 2] = [false; 2];
+static mut SOFT_PANICKED: bool = false;
 
 #[panic_handler]
 fn panic(info: &core::panic::PanicInfo) -> ! {
     let id = MPIDR.read().cpu_id() as usize;
-    print!("Core {} ", id);
+    let soft_panicked = unsafe { SOFT_PANICKED };
-    unsafe {
+    print!("Core {} panic at ", id);
-        if PANICKED[id] {
-            println!("nested panic!");
-            loop {}
-        }
-        PANICKED[id] = true;
-    }
-    print!("panic at ");
     if let Some(location) = info.location() {
         print!("{}:{}:{}", location.file(), location.line(), location.column());
     } else {
@@ -27,6 +27,20 @@ fn panic(info: &core::panic::PanicInfo) -> ! {
     } else {
         println!("");
     }
+    unsafe {
+        // soft panics only allowed for core 0
+        if PANICKED[id] && (SOFT_PANICKED || id == 1) {
+            println!("nested panic!");
+            loop {}
+        }
+        SOFT_PANICKED = true;
+        PANICKED[id] = true;
+    }
+    #[cfg(feature = "target_kasli_soc")]
+    {
+        let mut err_led = ErrorLED::error_led();
+        err_led.toggle(true);
+    }
     println!("Backtrace: ");
     let _ = backtrace(|ip| {
         // Backtrace gives us the return address, i.e. the address after the delay slot,
@@ -34,6 +48,26 @@ fn panic(info: &core::panic::PanicInfo) -> ! {
         print!("{:#08x} ", ip - 2 * 4);
     });
     println!("\nEnd backtrace");
-
+    if !soft_panicked && id == 0 {
+        soft_panic(info);
+    }
     loop {}
 }
+
+fn soft_panic(info: &core::panic::PanicInfo) -> ! {
+    // write panic info to log, so coremgmt can also read it
+    if let Some(location) = info.location() {
+        error!("panic at {}:{}:{}", location.file(), location.line(), location.column());
+    } else {
+        error!("panic at unknown location");
+    }
+    if let Some(message) = info.message() {
+        error!("panic message: {}", message);
+    }
+    let timer = GlobalTimer::start();
+    let cfg = match Config::new() {
+        Ok(cfg) => cfg,
+        Err(_) => Config::new_dummy(),
+    };
+    soft_panic_main(timer, cfg);
+}

src/runtime/src/proto_async.rs

@@ -1,8 +1,7 @@
-use core::cmp::min;
+use core::{cell::RefCell, cmp::min};
-use core::cell::RefCell;
 
-use libboard_zynq::smoltcp;
 use libasync::smoltcp::TcpStream;
+use libboard_zynq::smoltcp;
 
 type Result<T> = core::result::Result<T, smoltcp::Error>;
 
@@ -14,25 +13,27 @@ enum RecvState<T> {
 pub async fn expect(stream: &TcpStream, pattern: &[u8]) -> Result<bool> {
     let mut state = RecvState::NeedsMore(0, true);
     loop {
-        state = stream.recv(|buf| {
+        state = stream
-            let mut consumed = 0;
+            .recv(|buf| {
-            if let RecvState::NeedsMore(mut cur_index, _) = state {
+                let mut consumed = 0;
-                for b in buf.iter() {
+                if let RecvState::NeedsMore(mut cur_index, _) = state {
-                    consumed += 1;
+                    for b in buf.iter() {
-                    if *b == pattern[cur_index] {
+                        consumed += 1;
-                        if cur_index + 1 == pattern.len() {
+                        if *b == pattern[cur_index] {
-                            return (consumed, RecvState::Completed(true));
+                            if cur_index + 1 == pattern.len() {
+                                return (consumed, RecvState::Completed(true));
+                            }
+                        } else {
+                            return (consumed, RecvState::Completed(false));
                         }
-                    } else {
+                        cur_index += 1;
-                        return (consumed, RecvState::Completed(false));
                     }
-                    cur_index += 1;
+                    (consumed, RecvState::NeedsMore(cur_index, true))
+                } else {
+                    unreachable!();
                 }
-                (consumed, RecvState::NeedsMore(cur_index, true))
+            })
-            } else {
+            .await?;
-                unreachable!();
-            }
-        }).await?;
         if let RecvState::Completed(result) = state {
             return Ok(result);
         }
@@ -40,15 +41,11 @@ pub async fn expect(stream: &TcpStream, pattern: &[u8]) -> Result<bool> {
 }
 
 pub async fn read_bool(stream: &TcpStream) -> Result<bool> {
-    Ok(stream.recv(|buf| {
+    Ok(stream.recv(|buf| (1, buf[0] != 0)).await?)
-        (1, buf[0] != 0)
-    }).await?)
 }
 
 pub async fn read_i8(stream: &TcpStream) -> Result<i8> {
-    Ok(stream.recv(|buf| {
+    Ok(stream.recv(|buf| (1, buf[0] as i8)).await?)
-        (1, buf[0] as i8)
-    }).await?)
 }
 
 pub async fn read_i32(stream: &TcpStream) -> Result<i32> {
@@ -68,12 +65,14 @@ pub async fn read_chunk(stream: &TcpStream, destination: &mut [u8]) -> Result<()
     let destination = RefCell::new(destination);
     let mut done = 0;
     while done < total {
-        let count = stream.recv(|buf| {
+        let count = stream
-            let mut destination = destination.borrow_mut();
+            .recv(|buf| {
-            let count = min(total - done, buf.len());
+                let mut destination = destination.borrow_mut();
-            destination[done..done + count].copy_from_slice(&buf[..count]);
+                let count = min(total - done, buf.len());
-            (count, count)
+                destination[done..done + count].copy_from_slice(&buf[..count]);
-        }).await?;
+                (count, count)
+            })
+            .await?;
         done += count;
     }
     Ok(())

src/runtime/src/rpc_async.rs

@@ -0,0 +1,197 @@
+use alloc::boxed::Box;
+use core::future::Future;
+
+use async_recursion::async_recursion;
+use byteorder::{ByteOrder, NativeEndian};
+use cslice::CMutSlice;
+use ksupport::rpc::{tag::{Tag, TagIterator},
+                    *};
+use libasync::smoltcp::TcpStream;
+use libboard_zynq::smoltcp;
+use log::trace;
+
+use crate::proto_async;
+
+/// Reads (deserializes) `length` array or list elements of type `tag` from `stream`,
+/// writing them into the buffer given by `storage`.
+///
+/// `alloc` is used for nested allocations (if elements themselves contain
+/// lists/arrays), see [recv_value].
+#[async_recursion(?Send)]
+async unsafe fn recv_elements<F>(
+    stream: &TcpStream,
+    elt_tag: Tag<'async_recursion>,
+    length: usize,
+    storage: *mut (),
+    alloc: &(impl Fn(usize) -> F + 'async_recursion),
+) -> Result<(), smoltcp::Error>
+where
+    F: Future<Output = *mut ()>,
+{
+    // List of simple types are special-cased in the protocol for performance.
+    match elt_tag {
+        Tag::Bool => {
+            let dest = core::slice::from_raw_parts_mut(storage as *mut u8, length);
+            proto_async::read_chunk(stream, dest).await?;
+        }
+        Tag::Int32 => {
+            let ptr = storage as *mut u32;
+            let dest = core::slice::from_raw_parts_mut(ptr as *mut u8, length * 4);
+            proto_async::read_chunk(stream, dest).await?;
+            drop(dest);
+            let dest = core::slice::from_raw_parts_mut(ptr, length);
+            NativeEndian::from_slice_u32(dest);
+        }
+        Tag::Int64 | Tag::Float64 => {
+            let ptr = storage as *mut u64;
+            let dest = core::slice::from_raw_parts_mut(ptr as *mut u8, length * 8);
+            proto_async::read_chunk(stream, dest).await?;
+            drop(dest);
+            let dest = core::slice::from_raw_parts_mut(ptr, length);
+            NativeEndian::from_slice_u64(dest);
+        }
+        _ => {
+            let mut data = storage;
+            for _ in 0..length {
+                recv_value(stream, elt_tag, &mut data, alloc).await?
+            }
+        }
+    }
+    Ok(())
+}
+
+/// Reads (deserializes) a value of type `tag` from `stream`, writing the results to
+/// the kernel-side buffer `data` (the passed pointer to which is incremented to point
+/// past the just-received data). For nested allocations (lists/arrays), `alloc` is
+/// invoked any number of times with the size of the required allocation as a parameter
+/// (which is assumed to be correctly aligned for all payload types).
+#[async_recursion(?Send)]
+async unsafe fn recv_value<F>(
+    stream: &TcpStream,
+    tag: Tag<'async_recursion>,
+    data: &mut *mut (),
+    alloc: &(impl Fn(usize) -> F + 'async_recursion),
+) -> Result<(), smoltcp::Error>
+where
+    F: Future<Output = *mut ()>,
+{
+    macro_rules! consume_value {
+        ($ty:ty, | $ptr:ident | $map:expr) => {{
+            let $ptr = align_ptr_mut::<$ty>(*data);
+            *data = $ptr.offset(1) as *mut ();
+            $map
+        }};
+    }
+
+    match tag {
+        Tag::None => Ok(()),
+        Tag::Bool => consume_value!(i8, |ptr| {
+            *ptr = proto_async::read_i8(stream).await?;
+            Ok(())
+        }),
+        Tag::Int32 => consume_value!(i32, |ptr| {
+            *ptr = proto_async::read_i32(stream).await?;
+            Ok(())
+        }),
+        Tag::Int64 | Tag::Float64 => consume_value!(i64, |ptr| {
+            *ptr = proto_async::read_i64(stream).await?;
+            Ok(())
+        }),
+        Tag::String | Tag::Bytes | Tag::ByteArray => {
+            consume_value!(CMutSlice<u8>, |ptr| {
+                let length = proto_async::read_i32(stream).await? as usize;
+                *ptr = CMutSlice::new(alloc(length).await as *mut u8, length);
+                proto_async::read_chunk(stream, (*ptr).as_mut()).await?;
+                Ok(())
+            })
+        }
+        Tag::Tuple(it, arity) => {
+            let alignment = tag.alignment();
+            *data = round_up_mut(*data, alignment);
+            let mut it = it.clone();
+            for _ in 0..arity {
+                let tag = it.next().expect("truncated tag");
+                recv_value(stream, tag, data, alloc).await?
+            }
+            // Take into account any tail padding (if element(s) with largest alignment
+            // are not at the end).
+            *data = round_up_mut(*data, alignment);
+            Ok(())
+        }
+        Tag::List(it) => {
+            #[repr(C)]
+            struct List {
+                elements: *mut (),
+                length: usize,
+            }
+            consume_value!(*mut List, |ptr_to_list| {
+                let tag = it.clone().next().expect("truncated tag");
+                let length = proto_async::read_i32(stream).await? as usize;
+
+                // To avoid multiple kernel CPU roundtrips, use a single allocation for
+                // both the pointer/length List (slice) and the backing storage for the
+                // elements. We can assume that alloc() is aligned suitably, so just
+                // need to take into account any extra padding required.
+                // (Note: At the time of writing, there will never actually be any types
+                // with alignment larger than 8 bytes, so storage_offset == 0 always.)
+                let list_size = 4 + 4;
+                let storage_offset = round_up(list_size, tag.alignment());
+                let storage_size = tag.size() * length;
+
+                let allocation = alloc(storage_offset + storage_size).await as *mut u8;
+                *ptr_to_list = allocation as *mut List;
+                let storage = allocation.offset(storage_offset as isize) as *mut ();
+
+                (**ptr_to_list).length = length;
+                (**ptr_to_list).elements = storage;
+                recv_elements(stream, tag, length, storage, alloc).await
+            })
+        }
+        Tag::Array(it, num_dims) => {
+            consume_value!(*mut (), |buffer| {
+                // Deserialize length along each dimension and compute total number of
+                // elements.
+                let mut total_len: usize = 1;
+                for _ in 0..num_dims {
+                    let len = proto_async::read_i32(stream).await? as usize;
+                    total_len *= len;
+                    consume_value!(usize, |ptr| *ptr = len)
+                }
+
+                // Allocate backing storage for elements; deserialize them.
+                let elt_tag = it.clone().next().expect("truncated tag");
+                *buffer = alloc(elt_tag.size() * total_len).await;
+                recv_elements(stream, elt_tag, total_len, *buffer, alloc).await
+            })
+        }
+        Tag::Range(it) => {
+            *data = round_up_mut(*data, tag.alignment());
+            let tag = it.clone().next().expect("truncated tag");
+            recv_value(stream, tag, data, alloc).await?;
+            recv_value(stream, tag, data, alloc).await?;
+            recv_value(stream, tag, data, alloc).await?;
+            Ok(())
+        }
+        Tag::Keyword(_) => unreachable!(),
+        Tag::Object => unreachable!(),
+    }
+}
+
+pub async fn recv_return<F>(
+    stream: &TcpStream,
+    tag_bytes: &[u8],
+    data: *mut (),
+    alloc: &impl Fn(usize) -> F,
+) -> Result<(), smoltcp::Error>
+where
+    F: Future<Output = *mut ()>,
+{
+    let mut it = TagIterator::new(tag_bytes);
+    trace!("recv ...->{}", it);
+
+    let tag = it.next().expect("truncated tag");
+    let mut data = data;
+    unsafe { recv_value(stream, tag, &mut data, alloc).await? };
+
+    Ok(())
+}

src/runtime/src/rtio_clocking.rs

@@ -1,14 +1,16 @@
-use log::{info, warn, error};
-use libboard_zynq::timer::GlobalTimer;
 use embedded_hal::blocking::delay::DelayMs;
-use libconfig::Config;
+#[cfg(has_si5324)]
+use ksupport::i2c;
 use libboard_artiq::pl;
 #[cfg(has_si5324)]
-use libboard_zynq::i2c::I2c;
-#[cfg(has_si5324)]
-use crate::i2c;
-#[cfg(has_si5324)]
 use libboard_artiq::si5324;
+#[cfg(has_si549)]
+use libboard_artiq::si549;
+#[cfg(has_si5324)]
+use libboard_zynq::i2c::I2c;
+use libboard_zynq::timer::GlobalTimer;
+use libconfig::Config;
+use log::{info, warn};
 
 #[derive(Debug, PartialEq, Copy, Clone)]
 #[allow(non_camel_case_types)]
@@ -19,6 +21,7 @@ pub enum RtioClock {
     Int_150,
     Ext0_Bypass,
     Ext0_Synth0_10to125,
+    Ext0_Synth0_80to125,
     Ext0_Synth0_100to125,
     Ext0_Synth0_125to125,
 }
@@ -35,6 +38,7 @@ fn get_rtio_clock_cfg(cfg: &Config) -> RtioClock {
             "ext0_bypass_125" => RtioClock::Ext0_Bypass,
             "ext0_bypass_100" => RtioClock::Ext0_Bypass,
             "ext0_synth0_10to125" => RtioClock::Ext0_Synth0_10to125,
+            "ext0_synth0_80to125" => RtioClock::Ext0_Synth0_80to125,
             "ext0_synth0_100to125" => RtioClock::Ext0_Synth0_100to125,
             "ext0_synth0_125to125" => RtioClock::Ext0_Synth0_125to125,
             _ => {
@@ -42,17 +46,16 @@ fn get_rtio_clock_cfg(cfg: &Config) -> RtioClock {
                 RtioClock::Default
             }
         };
-    }
+    } else {
-    else {
         warn!("error reading configuration. Falling back to default.");
     }
     if res == RtioClock::Default {
-        #[cfg(rtio_frequency="100.0")]
+        #[cfg(rtio_frequency = "100.0")]
         {
             warn!("Using default configuration - internal 100MHz RTIO clock.");
             return RtioClock::Int_100;
         }
-        #[cfg(rtio_frequency="125.0")]
+        #[cfg(rtio_frequency = "125.0")]
         {
             warn!("Using default configuration - internal 125MHz RTIO clock.");
             return RtioClock::Int_125;
@@ -66,189 +69,380 @@ fn get_rtio_clock_cfg(cfg: &Config) -> RtioClock {
     res
 }
 
-
+#[cfg(not(has_drtio))]
-fn init_rtio(timer: &mut GlobalTimer, _clk: RtioClock) {
+fn init_rtio(timer: &mut GlobalTimer) {
-    #[cfg(has_rtio_crg_clock_sel)]
+    info!("Switching SYS clocks...");
-    let clock_sel = match _clk {
-        RtioClock::Ext0_Bypass => { 
-            info!("Using bypassed external clock");
-            1
-        },
-        RtioClock::Int_125 => {
-            info!("Using internal RTIO clock");
-            0
-        },
-        _ => {
-            warn!("rtio_clock setting '{:?}' is not supported. Using default internal RTIO clock instead", _clk);
-            0
-        }
-    };
-
     unsafe {
-        pl::csr::rtio_crg::pll_reset_write(1);
+        pl::csr::sys_crg::clock_switch_write(1);
-        #[cfg(has_rtio_crg_clock_sel)]
-        pl::csr::rtio_crg::clock_sel_write(clock_sel);
-        pl::csr::rtio_crg::pll_reset_write(0);
-    }
-    timer.delay_ms(1);
-    let locked = unsafe { pl::csr::rtio_crg::pll_locked_read() != 0 };
-    if locked {
-        info!("RTIO PLL locked");
-    } else {
-        error!("RTIO PLL failed to lock");
     }
+    // if it's not locked, it will hang at the CSR.
 
+    timer.delay_ms(50); // wait for CPLL/QPLL/SYS PLL lock
+    let clk = unsafe { pl::csr::sys_crg::current_clock_read() };
+    if clk == 1 {
+        info!("SYS CLK switched successfully");
+    } else {
+        panic!("SYS CLK did not switch");
+    }
     unsafe {
         pl::csr::rtio_core::reset_phy_write(1);
     }
+    info!("SYS PLL locked");
 }
 
 #[cfg(has_drtio)]
-fn init_drtio(timer: &mut GlobalTimer)
+fn init_drtio(timer: &mut GlobalTimer) {
-{
     unsafe {
-        pl::csr::drtio_transceiver::stable_clkin_write(1);
+        pl::csr::gt_drtio::stable_clkin_write(1);
+    }
+
+    timer.delay_ms(50); // wait for CPLL/QPLL/SYS PLL lock
+    let clk = unsafe { pl::csr::sys_crg::current_clock_read() };
+    if clk == 1 {
+        info!("SYS CLK switched successfully");
+    } else {
+        panic!("SYS CLK did not switch");
     }
-    timer.delay_ms(2); // wait for CPLL/QPLL lock
     unsafe {
-        pl::csr::drtio_transceiver::txenable_write(0xffffffffu32 as _);
+        pl::csr::rtio_core::reset_phy_write(1);
+        pl::csr::gt_drtio::txenable_write(0xffffffffu32 as _);
+
+        #[cfg(has_drtio_eem)]
+        pl::csr::eem_transceiver::txenable_write(0xffffffffu32 as _);
     }
 }
 
+// Si5324 input to select for locking to an external clock.
+#[cfg(has_si5324)]
+const SI5324_EXT_INPUT: si5324::Input = si5324::Input::Ckin1;
+
 #[cfg(has_si5324)]
 fn setup_si5324(i2c: &mut I2c, timer: &mut GlobalTimer, clk: RtioClock) {
     let (si5324_settings, si5324_ref_input) = match clk {
-        RtioClock::Ext0_Synth0_10to125 => { // 125 MHz output from 10 MHz CLKINx reference, 504 Hz BW
+        RtioClock::Ext0_Synth0_10to125 => {
+            // 125 MHz output from 10 MHz CLKINx reference, 504 Hz BW
             info!("using 10MHz reference to make 125MHz RTIO clock with PLL");
             (
                 si5324::FrequencySettings {
-                    n1_hs  : 10,
+                    n1_hs: 10,
-                    nc1_ls : 4,
+                    nc1_ls: 4,
-                    n2_hs  : 10,
+                    n2_hs: 10,
-                    n2_ls  : 300,
+                    n2_ls: 300,
-                    n31    : 6,
+                    n31: 6,
-                    n32    : 6,
+                    n32: 6,
-                    bwsel  : 4,
+                    bwsel: 4,
-                    crystal_ref: false
+                    crystal_as_ckin2: false,
                 },
-                si5324::Input::Ckin1
+                SI5324_EXT_INPUT,
             )
-        },
+        }
-        RtioClock::Ext0_Synth0_100to125 => { // 125MHz output, from 100MHz CLKINx reference, 586 Hz loop bandwidth
+        RtioClock::Ext0_Synth0_80to125 => {
+            // 125 MHz output from 80 MHz CLKINx reference, 611 Hz BW
+            info!("using 80MHz reference to make 125MHz RTIO clock with PLL");
+            (
+                si5324::FrequencySettings {
+                    n1_hs: 4,
+                    nc1_ls: 10,
+                    n2_hs: 10,
+                    n2_ls: 250,
+                    n31: 40,
+                    n32: 40,
+                    bwsel: 4,
+                    crystal_as_ckin2: false,
+                },
+                SI5324_EXT_INPUT,
+            )
+        }
+        RtioClock::Ext0_Synth0_100to125 => {
+            // 125MHz output, from 100MHz CLKINx reference, 586 Hz loop bandwidth
             info!("using 100MHz reference to make 125MHz RTIO clock with PLL");
             (
                 si5324::FrequencySettings {
-                    n1_hs  : 10,
+                    n1_hs: 10,
-                    nc1_ls : 4,
+                    nc1_ls: 4,
-                    n2_hs  : 10,
+                    n2_hs: 10,
-                    n2_ls  : 260,
+                    n2_ls: 260,
-                    n31    : 52,
+                    n31: 52,
-                    n32    : 52,
+                    n32: 52,
-                    bwsel  : 4,
+                    bwsel: 4,
-                    crystal_ref: false
+                    crystal_as_ckin2: false,
                 },
-                si5324::Input::Ckin1
+                SI5324_EXT_INPUT,
             )
-        },
+        }
-        RtioClock::Ext0_Synth0_125to125 => { // 125MHz output, from 125MHz CLKINx reference, 606 Hz loop bandwidth
+        RtioClock::Ext0_Synth0_125to125 => {
+            // 125MHz output, from 125MHz CLKINx reference, 606 Hz loop bandwidth
             info!("using 125MHz reference to make 125MHz RTIO clock with PLL");
             (
                 si5324::FrequencySettings {
-                    n1_hs  : 5,
+                    n1_hs: 5,
-                    nc1_ls : 8,
+                    nc1_ls: 8,
-                    n2_hs  : 7,
+                    n2_hs: 7,
-                    n2_ls  : 360,
+                    n2_ls: 360,
-                    n31    : 63,
+                    n31: 63,
-                    n32    : 63,
+                    n32: 63,
-                    bwsel  : 4,
+                    bwsel: 4,
-                    crystal_ref: false
+                    crystal_as_ckin2: false,
                 },
-                si5324::Input::Ckin1
+                SI5324_EXT_INPUT,
             )
-        },
+        }
-        RtioClock::Int_150 => { // 150MHz output, from crystal
+        RtioClock::Int_150 => {
+            // 150MHz output, from crystal
             info!("using internal 150MHz RTIO clock");
             (
                 si5324::FrequencySettings {
-                    n1_hs  : 9,
+                    n1_hs: 9,
-                    nc1_ls : 4,
+                    nc1_ls: 4,
-                    n2_hs  : 10,
+                    n2_hs: 10,
-                    n2_ls  : 33732,
+                    n2_ls: 33732,
-                    n31    : 7139,
+                    n31: 7139,
-                    n32    : 7139,
+                    n32: 7139,
-                    bwsel  : 3,
+                    bwsel: 3,
-                    crystal_ref: true
+                    crystal_as_ckin2: true,
                 },
-                si5324::Input::Ckin2
+                si5324::Input::Ckin2,
             )
-        },
+        }
-        RtioClock::Int_100 => { // 100MHz output, from crystal.
+        RtioClock::Int_100 => {
+            // 100MHz output, from crystal
             info!("using internal 100MHz RTIO clock");
             (
                 si5324::FrequencySettings {
-                    n1_hs  : 9,
+                    n1_hs: 9,
-                    nc1_ls : 6,
+                    nc1_ls: 6,
-                    n2_hs  : 10,
+                    n2_hs: 10,
-                    n2_ls  : 33732,
+                    n2_ls: 33732,
-                    n31    : 7139,
+                    n31: 7139,
-                    n32    : 7139,
+                    n32: 7139,
-                    bwsel  : 3,
+                    bwsel: 3,
-                    crystal_ref: true
+                    crystal_as_ckin2: true,
                 },
-                si5324::Input::Ckin2
+                si5324::Input::Ckin2,
             )
-        },
+        }
-        RtioClock::Int_125 => { // 125MHz output, from crystal, 7 Hz
+        RtioClock::Int_125 => {
+            // 125MHz output, from crystal, 7 Hz
             info!("using internal 125MHz RTIO clock");
             (
                 si5324::FrequencySettings {
-                    n1_hs  : 10,
+                    n1_hs: 10,
-                    nc1_ls : 4,
+                    nc1_ls: 4,
-                    n2_hs  : 10,
+                    n2_hs: 10,
-                    n2_ls  : 19972,
+                    n2_ls: 19972,
-                    n31    : 4565,
+                    n31: 4565,
-                    n32    : 4565,
+                    n32: 4565,
-                    bwsel  : 4,
+                    bwsel: 4,
-                    crystal_ref: true
+                    crystal_as_ckin2: true,
                 },
-                si5324::Input::Ckin2
+                si5324::Input::Ckin2,
             )
-        },
+        }
-        _ => { // same setting  as Int_125, but fallback to default
+        _ => {
-            warn!("rtio_clock setting '{:?}' is unsupported. Falling back to default internal 125MHz RTIO clock.", clk);
+            // same setting  as Int_125, but fallback to default
+            warn!(
+                "rtio_clock setting '{:?}' is unsupported. Falling back to default internal 125MHz RTIO clock.",
+                clk
+            );
             (
                 si5324::FrequencySettings {
-                    n1_hs  : 10,
+                    n1_hs: 10,
-                    nc1_ls : 4,
+                    nc1_ls: 4,
-                    n2_hs  : 10,
+                    n2_hs: 10,
-                    n2_ls  : 19972,
+                    n2_ls: 19972,
-                    n31    : 4565,
+                    n31: 4565,
-                    n32    : 4565,
+                    n32: 4565,
-                    bwsel  : 4,
+                    bwsel: 4,
-                    crystal_ref: true
+                    crystal_as_ckin2: true,
                 },
-                si5324::Input::Ckin2
+                si5324::Input::Ckin2,
             )
         }
     };
     si5324::setup(i2c, &si5324_settings, si5324_ref_input, timer).expect("cannot initialize Si5324");
 }
 
-pub fn init(timer: &mut GlobalTimer, cfg: &Config) {
+#[cfg(all(has_si549, has_wrpll))]
+fn wrpll_setup(timer: &mut GlobalTimer, clk: RtioClock, si549_settings: &si549::FrequencySetting) {
+    // register values are directly copied from preconfigured mmcm
+    let (mmcm_setting, mmcm_bypass) = match clk {
+        RtioClock::Ext0_Synth0_10to125 => (
+            si549::wrpll_refclk::MmcmSetting {
+                // CLKFBOUT_MULT = 62.5, DIVCLK_DIVIDE = 1 , CLKOUT0_DIVIDE = 5
+                clkout0_reg1: 0x1083,
+                clkout0_reg2: 0x0080,
+                clkfbout_reg1: 0x179e,
+                clkfbout_reg2: 0x4c00,
+                div_reg: 0x1041,
+                lock_reg1: 0x00fa,
+                lock_reg2: 0x7c01,
+                lock_reg3: 0xffe9,
+                power_reg: 0x9900,
+                filt_reg1: 0x1008,
+                filt_reg2: 0x8800,
+            },
+            false,
+        ),
+        RtioClock::Ext0_Synth0_80to125 => (
+            si549::wrpll_refclk::MmcmSetting {
+                // CLKFBOUT_MULT = 15.625, DIVCLK_DIVIDE = 1 , CLKOUT0_DIVIDE = 10
+                clkout0_reg1: 0x1145,
+                clkout0_reg2: 0x0000,
+                clkfbout_reg1: 0x11c7,
+                clkfbout_reg2: 0x5880,
+                div_reg: 0x1041,
+                lock_reg1: 0x028a,
+                lock_reg2: 0x7c01,
+                lock_reg3: 0xffe9,
+                power_reg: 0x9900,
+                filt_reg1: 0x9908,
+                filt_reg2: 0x8100,
+            },
+            false,
+        ),
+        RtioClock::Ext0_Synth0_100to125 => (
+            si549::wrpll_refclk::MmcmSetting {
+                // CLKFBOUT_MULT = 12.5, DIVCLK_DIVIDE = 1 , CLKOUT0_DIVIDE = 10
+                clkout0_reg1: 0x1145,
+                clkout0_reg2: 0x0000,
+                clkfbout_reg1: 0x1145,
+                clkfbout_reg2: 0x4c00,
+                div_reg: 0x1041,
+                lock_reg1: 0x0339,
+                lock_reg2: 0x7c01,
+                lock_reg3: 0xffe9,
+                power_reg: 0x9900,
+                filt_reg1: 0x9108,
+                filt_reg2: 0x0100,
+            },
+            false,
+        ),
+        RtioClock::Ext0_Synth0_125to125 => (
+            si549::wrpll_refclk::MmcmSetting {
+                // CLKFBOUT_MULT = 10, DIVCLK_DIVIDE = 1 , CLKOUT0_DIVIDE = 10
+                clkout0_reg1: 0x1145,
+                clkout0_reg2: 0x0000,
+                clkfbout_reg1: 0x1145,
+                clkfbout_reg2: 0x0000,
+                div_reg: 0x1041,
+                lock_reg1: 0x03e8,
+                lock_reg2: 0x7001,
+                lock_reg3: 0xf3e9,
+                power_reg: 0x0100,
+                filt_reg1: 0x9908,
+                filt_reg2: 0x1100,
+            },
+            true,
+        ),
+        _ => unreachable!(),
+    };
 
+    si549::helper_setup(timer, &si549_settings).expect("cannot initialize helper Si549");
+    si549::wrpll_refclk::setup(timer, mmcm_setting, mmcm_bypass).expect("cannot initialize ref clk for wrpll");
+    si549::wrpll::select_recovered_clock(true, timer);
+}
+
+#[cfg(has_si549)]
+fn get_si549_setting(clk: RtioClock) -> si549::FrequencySetting {
+    match clk {
+        RtioClock::Ext0_Synth0_10to125 => {
+            info!("using 10MHz reference to make 125MHz RTIO clock with WRPLL");
+        }
+        RtioClock::Ext0_Synth0_80to125 => {
+            info!("using 80MHz reference to make 125MHz RTIO clock with WRPLL");
+        }
+        RtioClock::Ext0_Synth0_100to125 => {
+            info!("using 100MHz reference to make 125MHz RTIO clock with WRPLL");
+        }
+        RtioClock::Ext0_Synth0_125to125 => {
+            info!("using 125MHz reference to make 125MHz RTIO clock with WRPLL");
+        }
+        RtioClock::Int_100 => {
+            info!("using internal 100MHz RTIO clock");
+        }
+        RtioClock::Int_125 => {
+            info!("using internal 125MHz RTIO clock");
+        }
+        _ => {
+            warn!(
+                "rtio_clock setting '{:?}' is unsupported. Falling back to default internal 125MHz RTIO clock.",
+                clk
+            );
+        }
+    };
+
+    match clk {
+        RtioClock::Int_100 => {
+            si549::FrequencySetting {
+                main: si549::DividerConfig {
+                    hsdiv: 0x06C,
+                    lsdiv: 0,
+                    fbdiv: 0x046C5F49797,
+                },
+                helper: si549::DividerConfig {
+                    // 100MHz*32767/32768
+                    hsdiv: 0x06C,
+                    lsdiv: 0,
+                    fbdiv: 0x046C5670BBD,
+                },
+            }
+        }
+        _ => {
+            // Everything else use 125MHz
+            si549::FrequencySetting {
+                main: si549::DividerConfig {
+                    hsdiv: 0x058,
+                    lsdiv: 0,
+                    fbdiv: 0x04815791F25,
+                },
+                helper: si549::DividerConfig {
+                    // 125MHz*32767/32768
+                    hsdiv: 0x058,
+                    lsdiv: 0,
+                    fbdiv: 0x04814E8F442,
+                },
+            }
+        }
+    }
+}
+
+pub fn init(timer: &mut GlobalTimer, cfg: &Config) {
     let clk = get_rtio_clock_cfg(cfg);
     #[cfg(has_si5324)]
     {
         let i2c = unsafe { (&mut i2c::I2C_BUS).as_mut().unwrap() };
-        let si5324_ext_input = si5324::Input::Ckin1;
         match clk {
-            RtioClock::Ext0_Bypass => si5324::bypass(i2c, si5324_ext_input, timer).expect("cannot bypass Si5324"),
+            RtioClock::Ext0_Bypass => {
+                info!("bypassing the PLL for RTIO clock");
+                si5324::bypass(i2c, SI5324_EXT_INPUT, timer).expect("cannot bypass Si5324")
+            }
             _ => setup_si5324(i2c, timer, clk),
         }
     }
+
+    #[cfg(has_si549)]
+    let si549_settings = get_si549_setting(clk);
+
+    #[cfg(has_si549)]
+    si549::main_setup(timer, &si549_settings).expect("cannot initialize main Si549");
+
     #[cfg(has_drtio)]
     init_drtio(timer);
 
-    init_rtio(timer, clk);
+    #[cfg(not(has_drtio))]
+    init_rtio(timer);
 
+    #[cfg(all(has_si549, has_wrpll))]
+    {
+        // SYS CLK switch will reset CSRs that are used by WRPLL
+        match clk {
+            RtioClock::Ext0_Synth0_10to125
+            | RtioClock::Ext0_Synth0_80to125
+            | RtioClock::Ext0_Synth0_100to125
+            | RtioClock::Ext0_Synth0_125to125 => {
+                wrpll_setup(timer, clk, &si549_settings);
+            }
+            _ => {}
+        }
+    }
 }

src/runtime/src/rtio_csr.rs

@@ -1,213 +0,0 @@
-use core::ptr::{read_volatile, write_volatile};
-use cslice::CSlice;
-
-use crate::artiq_raise;
-
-use crate::pl::csr;
-
-pub const RTIO_O_STATUS_WAIT:                      u8 = 1;
-pub const RTIO_O_STATUS_UNDERFLOW:                 u8 = 2;
-pub const RTIO_O_STATUS_DESTINATION_UNREACHABLE:   u8 = 4;
-pub const RTIO_I_STATUS_WAIT_EVENT:                u8 = 1;
-pub const RTIO_I_STATUS_OVERFLOW:                  u8 = 2;
-pub const RTIO_I_STATUS_WAIT_STATUS:               u8 = 4;
-pub const RTIO_I_STATUS_DESTINATION_UNREACHABLE:   u8 = 8;
-
-#[repr(C)]
-pub struct TimestampedData {
-    timestamp: i64,
-    data: i32,
-}
-
-pub extern fn init() {
-    unsafe {
-        csr::rtio_core::reset_write(1);
-    }
-}
-
-pub extern fn get_counter() -> i64 {
-    unsafe {
-        csr::rtio::counter_update_write(1);
-        csr::rtio::counter_read() as i64
-    }
-}
-
-pub extern fn now_mu() -> i64 {
-    unsafe {
-        csr::rtio::now_read() as i64
-    }
-}
-
-pub extern fn at_mu(t: i64) {
-    unsafe {
-        csr::rtio::now_write(t as u64);
-    }
-}
-
-pub extern fn delay_mu(dt: i64) {
-    unsafe {
-        csr::rtio::now_write(csr::rtio::now_read() + dt as u64);
-    }
-}
-
-// writing the LSB of o_data (offset=0) triggers the RTIO write
-#[inline(always)]
-pub unsafe fn rtio_o_data_write(offset: usize, data: u32) {
-    write_volatile(
-        csr::rtio::O_DATA_ADDR.offset((csr::rtio::O_DATA_SIZE - 1 - offset) as isize),
-        data);
-}
-
-#[inline(always)]
-pub unsafe fn rtio_i_data_read(offset: usize) -> u32 {
-    read_volatile(
-        csr::rtio::I_DATA_ADDR.offset((csr::rtio::I_DATA_SIZE - 1 - offset) as isize))
-}
-
-#[inline(never)]
-unsafe fn process_exceptional_status(channel: i32, status: u8) {
-    let timestamp = csr::rtio::now_read() as i64;
-    if status & RTIO_O_STATUS_WAIT != 0 {
-        while csr::rtio::o_status_read() & RTIO_O_STATUS_WAIT != 0 {}
-    }
-    if status & RTIO_O_STATUS_UNDERFLOW != 0 {
-        artiq_raise!("RTIOUnderflow",
-            "RTIO underflow at {0} mu, channel {1}, slack {2} mu",
-            timestamp, channel as i64, timestamp - get_counter());
-    }
-    if status & RTIO_O_STATUS_DESTINATION_UNREACHABLE != 0 {
-        artiq_raise!("RTIODestinationUnreachable",
-            "RTIO destination unreachable, output, at {0} mu, channel {1}",
-            timestamp, channel as i64, 0);
-    }
-}
-
-pub extern fn output(target: i32, data: i32) {
-    unsafe {
-        csr::rtio::target_write(target as u32);
-        // writing target clears o_data
-        rtio_o_data_write(0, data as _);
-        let status = csr::rtio::o_status_read();
-        if status != 0 {
-            process_exceptional_status(target >> 8, status);
-        }
-    }
-}
-
-pub extern fn output_wide(target: i32, data: &CSlice<i32>) {
-    unsafe {
-        csr::rtio::target_write(target as u32);
-        // writing target clears o_data
-        for i in (0..data.len()).rev() {
-            rtio_o_data_write(i, data[i] as _)
-        }
-        let status = csr::rtio::o_status_read();
-        if status != 0 {
-            process_exceptional_status(target >> 8, status);
-        }
-    }
-}
-
-pub extern fn input_timestamp(timeout: i64, channel: i32) -> i64 {
-    unsafe {
-        csr::rtio::target_write((channel as u32) << 8);
-        csr::rtio::i_timeout_write(timeout as u64);
-
-        let mut status = RTIO_I_STATUS_WAIT_STATUS;
-        while status & RTIO_I_STATUS_WAIT_STATUS != 0 {
-            status = csr::rtio::i_status_read();
-        }
-
-        if status & RTIO_I_STATUS_OVERFLOW != 0 {
-            artiq_raise!("RTIOOverflow",
-                         "RTIO input overflow on channel {0}",
-                         channel as i64, 0, 0);
-        }
-        if status & RTIO_I_STATUS_WAIT_EVENT != 0 {
-            return -1
-        }
-        if status & RTIO_I_STATUS_DESTINATION_UNREACHABLE != 0 {
-            artiq_raise!("RTIODestinationUnreachable",
-                         "RTIO destination unreachable, input, on channel {0}",
-                         channel as i64, 0, 0);
-        }
-
-        csr::rtio::i_timestamp_read() as i64
-    }
-}
-
-pub extern fn input_data(channel: i32) -> i32 {
-    unsafe {
-        csr::rtio::target_write((channel as u32) << 8);
-        csr::rtio::i_timeout_write(0xffffffff_ffffffff);
-
-        let mut status = RTIO_I_STATUS_WAIT_STATUS;
-        while status & RTIO_I_STATUS_WAIT_STATUS != 0 {
-            status = csr::rtio::i_status_read();
-        }
-
-        if status & RTIO_I_STATUS_OVERFLOW != 0 {
-            artiq_raise!("RTIOOverflow",
-                         "RTIO input overflow on channel {0}",
-                         channel as i64, 0, 0);
-        }
-        if status & RTIO_I_STATUS_DESTINATION_UNREACHABLE != 0 {
-            artiq_raise!("RTIODestinationUnreachable",
-                         "RTIO destination unreachable, input, on channel {0}",
-                         channel as i64, 0, 0);
-        }
-
-        rtio_i_data_read(0) as i32
-    }
-}
-
-pub extern fn input_timestamped_data(timeout: i64, channel: i32) -> TimestampedData {
-    unsafe {
-        csr::rtio::target_write((channel as u32) << 8);
-        csr::rtio::i_timeout_write(timeout as u64);
-
-        let mut status = RTIO_I_STATUS_WAIT_STATUS;
-        while status & RTIO_I_STATUS_WAIT_STATUS != 0 {
-            status = csr::rtio::i_status_read();
-        }
-
-        if status & RTIO_I_STATUS_OVERFLOW != 0 {
-            artiq_raise!("RTIOOverflow",
-                         "RTIO input overflow on channel {0}",
-                         channel as i64, 0, 0);
-        }
-        if status & RTIO_I_STATUS_WAIT_EVENT != 0 {
-            return TimestampedData { timestamp: -1, data: 0 }
-        }
-        if status & RTIO_I_STATUS_DESTINATION_UNREACHABLE != 0 {
-            artiq_raise!("RTIODestinationUnreachable",
-                         "RTIO destination unreachable, input, on channel {0}",
-                         channel as i64, 0, 0);
-        }
-
-        TimestampedData {
-            timestamp: csr::rtio::i_timestamp_read() as i64,
-            data: rtio_i_data_read(0) as i32
-        }
-    }
-}
-
-pub fn write_log(data: &[i8]) {
-    unsafe {
-        csr::rtio::target_write(csr::CONFIG_RTIO_LOG_CHANNEL << 8);
-
-        let mut word: u32 = 0;
-        for i in 0..data.len() {
-            word <<= 8;
-            word |= data[i] as u32;
-            if i % 4 == 3 {
-                rtio_o_data_write(0, word);
-                word = 0;
-            }
-        }
-
-        if word != 0 {
-            rtio_o_data_write(0, word);
-        }
-    }
-}

src/runtime/src/rtio_dma.rs

@@ -0,0 +1,361 @@
+use alloc::{collections::BTreeMap, rc::Rc, string::String, vec::Vec};
+#[cfg(has_drtio)]
+use core::mem;
+
+use ksupport::kernel::DmaRecorder;
+#[cfg(has_drtio)]
+use libasync::task;
+use libboard_artiq::drtio_routing::RoutingTable;
+use libboard_zynq::timer::GlobalTimer;
+use libcortex_a9::{cache::dcci_slice, mutex::Mutex};
+
+const ALIGNMENT: usize = 16 * 8;
+
+static DMA_RECORD_STORE: Mutex<BTreeMap<String, (u32, Vec<u8>, i64)>> = Mutex::new(BTreeMap::new());
+
+#[cfg(has_drtio)]
+pub mod remote_dma {
+    use libboard_zynq::time::Milliseconds;
+    use log::error;
+
+    use super::*;
+    use crate::rtio_mgt::drtio;
+
+    #[derive(Debug, PartialEq, Clone)]
+    pub enum RemoteState {
+        NotLoaded,
+        Loaded,
+        PlaybackEnded { error: u8, channel: u32, timestamp: u64 },
+    }
+    #[derive(Debug, Clone)]
+    struct RemoteTrace {
+        trace: Vec<u8>,
+        pub state: RemoteState,
+    }
+
+    impl From<Vec<u8>> for RemoteTrace {
+        fn from(trace: Vec<u8>) -> Self {
+            RemoteTrace {
+                trace: trace,
+                state: RemoteState::NotLoaded,
+            }
+        }
+    }
+
+    impl RemoteTrace {
+        pub fn get_trace(&self) -> &Vec<u8> {
+            &self.trace
+        }
+    }
+
+    // represents all traces for a given ID
+    struct TraceSet {
+        id: u32,
+        done_count: Mutex<usize>,
+        traces: Mutex<BTreeMap<u8, RemoteTrace>>,
+    }
+
+    impl TraceSet {
+        pub fn new(id: u32, traces: BTreeMap<u8, Vec<u8>>) -> TraceSet {
+            let mut trace_map: BTreeMap<u8, RemoteTrace> = BTreeMap::new();
+            for (destination, trace) in traces {
+                trace_map.insert(destination, trace.into());
+            }
+            TraceSet {
+                id: id,
+                done_count: Mutex::new(0),
+                traces: Mutex::new(trace_map),
+            }
+        }
+
+        pub async fn await_done(&self, timeout: Option<u64>, timer: GlobalTimer) -> Result<RemoteState, &'static str> {
+            let timeout_ms = Milliseconds(timeout.unwrap_or(10_000));
+            let limit = timer.get_time() + timeout_ms;
+            while (timer.get_time() < limit)
+                & (*(self.done_count.async_lock().await) < self.traces.async_lock().await.len())
+            {
+                task::r#yield().await;
+            }
+            if timer.get_time() >= limit {
+                error!("Remote DMA await done timed out");
+                return Err("Timed out waiting for results.");
+            }
+            let mut playback_state: RemoteState = RemoteState::PlaybackEnded {
+                error: 0,
+                channel: 0,
+                timestamp: 0,
+            };
+            let mut lock = self.traces.async_lock().await;
+            let trace_iter = lock.iter_mut();
+            for (_dest, trace) in trace_iter {
+                match trace.state {
+                    RemoteState::PlaybackEnded {
+                        error: e,
+                        channel: _c,
+                        timestamp: _ts,
+                    } => {
+                        if e != 0 {
+                            playback_state = trace.state.clone();
+                        }
+                    }
+                    _ => (),
+                }
+                trace.state = RemoteState::Loaded;
+            }
+            Ok(playback_state)
+        }
+
+        pub async fn upload_traces(
+            &mut self,
+            aux_mutex: &Rc<Mutex<bool>>,
+            routing_table: &RoutingTable,
+            timer: GlobalTimer,
+        ) {
+            let mut lock = self.traces.async_lock().await;
+            let trace_iter = lock.iter_mut();
+            for (destination, trace) in trace_iter {
+                match drtio::ddma_upload_trace(
+                    aux_mutex,
+                    routing_table,
+                    timer,
+                    self.id,
+                    *destination,
+                    trace.get_trace(),
+                )
+                .await
+                {
+                    Ok(_) => trace.state = RemoteState::Loaded,
+                    Err(e) => error!("Error adding DMA trace on destination {}: {}", destination, e),
+                }
+            }
+            *(self.done_count.async_lock().await) = 0;
+        }
+
+        pub async fn erase(&mut self, aux_mutex: &Rc<Mutex<bool>>, routing_table: &RoutingTable, timer: GlobalTimer) {
+            let lock = self.traces.async_lock().await;
+            let trace_iter = lock.keys();
+            for destination in trace_iter {
+                match drtio::ddma_send_erase(aux_mutex, routing_table, timer, self.id, *destination).await {
+                    Ok(_) => (),
+                    Err(e) => error!("Error adding DMA trace on destination {}: {}", destination, e),
+                }
+            }
+        }
+
+        pub async fn playback_done(&mut self, source: u8, error: u8, channel: u32, timestamp: u64) {
+            let mut traces_locked = self.traces.async_lock().await;
+            let mut trace = traces_locked.get_mut(&source).unwrap();
+            trace.state = RemoteState::PlaybackEnded {
+                error: error,
+                channel: channel,
+                timestamp: timestamp,
+            };
+            *(self.done_count.async_lock().await) += 1;
+        }
+
+        pub async fn playback(
+            &self,
+            aux_mutex: &Rc<Mutex<bool>>,
+            routing_table: &RoutingTable,
+            timer: GlobalTimer,
+            timestamp: u64,
+        ) {
+            let mut dest_list: Vec<u8> = Vec::new();
+            {
+                let lock = self.traces.async_lock().await;
+                let trace_iter = lock.iter();
+                for (dest, trace) in trace_iter {
+                    if trace.state != RemoteState::Loaded {
+                        error!("Destination {} not ready for DMA, state: {:?}", dest, trace.state);
+                        continue;
+                    }
+                    dest_list.push(dest.clone());
+                }
+            }
+            // mutex lock must be dropped before sending a playback request to avoid a deadlock,
+            // if PlaybackStatus is sent from another satellite and the state must be updated.
+            for destination in dest_list {
+                match drtio::ddma_send_playback(aux_mutex, routing_table, timer, self.id, destination, timestamp).await
+                {
+                    Ok(_) => (),
+                    Err(e) => error!("Error during remote DMA playback: {}", e),
+                }
+            }
+        }
+
+        pub async fn destination_changed(
+            &mut self,
+            aux_mutex: &Rc<Mutex<bool>>,
+            routing_table: &RoutingTable,
+            timer: GlobalTimer,
+            destination: u8,
+            up: bool,
+        ) {
+            // update state of the destination, resend traces if it's up
+            if let Some(trace) = self.traces.async_lock().await.get_mut(&destination) {
+                if up {
+                    match drtio::ddma_upload_trace(
+                        aux_mutex,
+                        routing_table,
+                        timer,
+                        self.id,
+                        destination,
+                        trace.get_trace(),
+                    )
+                    .await
+                    {
+                        Ok(_) => trace.state = RemoteState::Loaded,
+                        Err(e) => error!("Error adding DMA trace on destination {}: {}", destination, e),
+                    }
+                } else {
+                    trace.state = RemoteState::NotLoaded;
+                }
+            }
+        }
+
+        pub async fn is_empty(&self) -> bool {
+            self.traces.async_lock().await.is_empty()
+        }
+    }
+
+    static mut TRACES: BTreeMap<u32, TraceSet> = BTreeMap::new();
+
+    pub fn add_traces(id: u32, traces: BTreeMap<u8, Vec<u8>>) {
+        unsafe { TRACES.insert(id, TraceSet::new(id, traces)) };
+    }
+
+    pub async fn await_done(id: u32, timeout: Option<u64>, timer: GlobalTimer) -> Result<RemoteState, &'static str> {
+        let trace_set = unsafe { TRACES.get_mut(&id).unwrap() };
+        trace_set.await_done(timeout, timer).await
+    }
+
+    pub async fn erase(aux_mutex: &Rc<Mutex<bool>>, routing_table: &RoutingTable, timer: GlobalTimer, id: u32) {
+        let trace_set = unsafe { TRACES.get_mut(&id).unwrap() };
+        trace_set.erase(aux_mutex, routing_table, timer).await;
+        unsafe {
+            TRACES.remove(&id);
+        }
+    }
+
+    pub async fn upload_traces(aux_mutex: &Rc<Mutex<bool>>, routing_table: &RoutingTable, timer: GlobalTimer, id: u32) {
+        let trace_set = unsafe { TRACES.get_mut(&id).unwrap() };
+        trace_set.upload_traces(aux_mutex, routing_table, timer).await;
+    }
+
+    pub async fn playback(
+        aux_mutex: &Rc<Mutex<bool>>,
+        routing_table: &RoutingTable,
+        timer: GlobalTimer,
+        id: u32,
+        timestamp: u64,
+    ) {
+        let trace_set = unsafe { TRACES.get_mut(&id).unwrap() };
+        trace_set.playback(aux_mutex, routing_table, timer, timestamp).await;
+    }
+
+    pub async fn playback_done(id: u32, destination: u8, error: u8, channel: u32, timestamp: u64) {
+        let trace_set = unsafe { TRACES.get_mut(&id).unwrap() };
+        trace_set.playback_done(destination, error, channel, timestamp).await;
+    }
+
+    pub async fn destination_changed(
+        aux_mutex: &Rc<Mutex<bool>>,
+        routing_table: &RoutingTable,
+        timer: GlobalTimer,
+        destination: u8,
+        up: bool,
+    ) {
+        let trace_iter = unsafe { TRACES.values_mut() };
+        for trace_set in trace_iter {
+            trace_set
+                .destination_changed(aux_mutex, routing_table, timer, destination, up)
+                .await;
+        }
+    }
+
+    pub async fn has_remote_traces(id: u32) -> bool {
+        let trace_set = unsafe { TRACES.get_mut(&id).unwrap() };
+        !(trace_set.is_empty().await)
+    }
+}
+
+pub async fn put_record(
+    _aux_mutex: &Rc<Mutex<bool>>,
+    _routing_table: &RoutingTable,
+    _timer: GlobalTimer,
+    mut recorder: DmaRecorder,
+) -> u32 {
+    #[cfg(has_drtio)]
+    let mut remote_traces: BTreeMap<u8, Vec<u8>> = BTreeMap::new();
+
+    #[cfg(has_drtio)]
+    if recorder.enable_ddma {
+        let mut local_trace: Vec<u8> = Vec::new();
+        // analyze each entry and put in proper buckets, as the kernel core
+        // sends whole chunks, to limit comms/kernel CPU communication,
+        // and as only comms core has access to varios DMA buffers.
+        let mut ptr = 0;
+        recorder.buffer.push(0);
+        while recorder.buffer[ptr] != 0 {
+            // ptr + 3 = tgt >> 24 (destination)
+            let len = recorder.buffer[ptr] as usize;
+            let destination = recorder.buffer[ptr + 3];
+            if destination == 0 {
+                local_trace.extend(&recorder.buffer[ptr..ptr + len]);
+            } else {
+                if let Some(remote_trace) = remote_traces.get_mut(&destination) {
+                    remote_trace.extend(&recorder.buffer[ptr..ptr + len]);
+                } else {
+                    remote_traces.insert(destination, recorder.buffer[ptr..ptr + len].to_vec());
+                }
+            }
+            // and jump to the next event
+            ptr += len;
+        }
+        mem::swap(&mut recorder.buffer, &mut local_trace);
+    }
+    // trailing zero to indicate end of buffer
+    recorder.buffer.push(0);
+    recorder.buffer.reserve(ALIGNMENT - 1);
+    let original_length = recorder.buffer.len();
+    let padding = ALIGNMENT - recorder.buffer.as_ptr() as usize % ALIGNMENT;
+    let padding = if padding == ALIGNMENT { 0 } else { padding };
+    for _ in 0..padding {
+        recorder.buffer.push(0);
+    }
+    recorder.buffer.copy_within(0..original_length, padding);
+    dcci_slice(&recorder.buffer);
+
+    let ptr = recorder.buffer[padding..].as_ptr() as u32;
+
+    let _old_record = DMA_RECORD_STORE
+        .lock()
+        .insert(recorder.name, (ptr, recorder.buffer, recorder.duration));
+
+    #[cfg(has_drtio)]
+    {
+        if let Some((old_id, _v, _d)) = _old_record {
+            remote_dma::erase(_aux_mutex, _routing_table, _timer, old_id).await;
+        }
+        remote_dma::add_traces(ptr, remote_traces);
+    }
+
+    ptr
+}
+
+pub async fn erase(name: String, _aux_mutex: &Rc<Mutex<bool>>, _routing_table: &RoutingTable, _timer: GlobalTimer) {
+    let _entry = DMA_RECORD_STORE.lock().remove(&name);
+    #[cfg(has_drtio)]
+    if let Some((id, _v, _d)) = _entry {
+        remote_dma::erase(_aux_mutex, _routing_table, _timer, id).await;
+    }
+}
+
+pub async fn retrieve(name: String) -> Option<(i32, i64, bool)> {
+    let (ptr, _v, duration) = DMA_RECORD_STORE.lock().get(&name)?.clone();
+    #[cfg(has_drtio)]
+    let uses_ddma = remote_dma::has_remote_traces(ptr).await;
+    #[cfg(not(has_drtio))]
+    let uses_ddma = false;
+    Some((ptr as i32, duration, uses_ddma))
+}

src/runtime/src/rtio_mgt.rs

@@ -1,26 +1,70 @@
-use core::cell::RefCell;
 use alloc::rc::Rc;
-use libboard_zynq::timer::GlobalTimer;
+use core::cell::RefCell;
-use libboard_artiq::{pl::csr, drtio_routing};
-use libcortex_a9::mutex::Mutex;
 
+use libboard_artiq::{drtio_routing, drtio_routing::RoutingTable, pl::csr};
+use libboard_zynq::timer::GlobalTimer;
+use libcortex_a9::mutex::Mutex;
 
 #[cfg(has_drtio)]
 pub mod drtio {
-    use super::*;
+    use alloc::vec::Vec;
-    use crate::{SEEN_ASYNC_ERRORS, ASYNC_ERROR_BUSY, ASYNC_ERROR_SEQUENCE_ERROR, ASYNC_ERROR_COLLISION};
+    use core::fmt;
-    use libboard_artiq::drtioaux_async;
-    use libboard_artiq::drtioaux_async::Packet;
-    use libboard_artiq::drtioaux::Error;
-    use log::{warn, error, info};
-    use embedded_hal::blocking::delay::DelayMs;
-    use libasync::{task, delay};
-    use libboard_zynq::time::Milliseconds;
 
-    pub fn startup(aux_mutex: &Rc<Mutex<bool>>,
+    use embedded_hal::blocking::delay::DelayMs;
-            routing_table: &Rc<RefCell<drtio_routing::RoutingTable>>,
+    use ksupport::{resolve_channel_name, ASYNC_ERROR_BUSY, ASYNC_ERROR_COLLISION, ASYNC_ERROR_SEQUENCE_ERROR,
-            up_destinations: &Rc<RefCell<[bool; drtio_routing::DEST_COUNT]>>,
+                   SEEN_ASYNC_ERRORS};
-            timer: GlobalTimer) {
+    use libasync::{delay, task};
+    use libboard_artiq::{drtioaux::Error as DrtioError,
+                         drtioaux_async,
+                         drtioaux_async::Packet,
+                         drtioaux_proto::{PayloadStatus, MASTER_PAYLOAD_MAX_SIZE}};
+    use libboard_zynq::time::Milliseconds;
+    use log::{error, info, warn};
+
+    use super::*;
+    use crate::{analyzer::remote_analyzer::RemoteBuffer, rtio_dma::remote_dma, subkernel};
+
+    #[derive(Debug, PartialEq, Eq, Clone, Copy)]
+    pub enum Error {
+        Timeout,
+        AuxError,
+        LinkDown,
+        UnexpectedReply,
+        DmaAddTraceFail(u8),
+        DmaEraseFail(u8),
+        DmaPlaybackFail(u8),
+        SubkernelAddFail(u8),
+        SubkernelRunFail(u8),
+    }
+
+    impl fmt::Display for Error {
+        fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+            match self {
+                Error::Timeout => write!(f, "timed out"),
+                Error::AuxError => write!(f, "aux packet error"),
+                Error::LinkDown => write!(f, "link down"),
+                Error::UnexpectedReply => write!(f, "unexpected reply"),
+                Error::DmaAddTraceFail(dest) => write!(f, "error adding DMA trace on satellite #{}", dest),
+                Error::DmaEraseFail(dest) => write!(f, "error erasing DMA trace on satellite #{}", dest),
+                Error::DmaPlaybackFail(dest) => write!(f, "error playing back DMA trace on satellite #{}", dest),
+                Error::SubkernelAddFail(dest) => write!(f, "error adding subkernel on satellite #{}", dest),
+                Error::SubkernelRunFail(dest) => write!(f, "error on subkernel run request on satellite #{}", dest),
+            }
+        }
+    }
+
+    impl From<DrtioError> for Error {
+        fn from(_error: DrtioError) -> Self {
+            Error::AuxError
+        }
+    }
+
+    pub fn startup(
+        aux_mutex: &Rc<Mutex<bool>>,
+        routing_table: &Rc<RefCell<RoutingTable>>,
+        up_destinations: &Rc<RefCell<[bool; drtio_routing::DEST_COUNT]>>,
+        timer: GlobalTimer,
+    ) {
         let aux_mutex = aux_mutex.clone();
         let routing_table = routing_table.clone();
         let up_destinations = up_destinations.clone();
@@ -32,30 +76,107 @@ pub mod drtio {
 
     async fn link_rx_up(linkno: u8) -> bool {
         let linkno = linkno as usize;
-        unsafe {
+        unsafe { (csr::DRTIO[linkno].rx_up_read)() == 1 }
-            (csr::DRTIO[linkno].rx_up_read)() == 1
+    }
+
+    async fn process_async_packets(linkno: u8, routing_table: &RoutingTable, packet: Packet) -> Option<Packet> {
+        match packet {
+            Packet::DmaPlaybackStatus {
+                id,
+                source,
+                destination: 0,
+                error,
+                channel,
+                timestamp,
+            } => {
+                remote_dma::playback_done(id, source, error, channel, timestamp).await;
+                None
+            }
+            Packet::SubkernelFinished {
+                id,
+                destination: 0,
+                with_exception,
+                exception_src,
+            } => {
+                subkernel::subkernel_finished(id, with_exception, exception_src).await;
+                None
+            }
+            Packet::SubkernelMessage {
+                id,
+                source,
+                destination: 0,
+                status,
+                length,
+                data,
+            } => {
+                subkernel::message_handle_incoming(id, status, length as usize, &data).await;
+                // acknowledge receiving part of the message
+                drtioaux_async::send(linkno, &Packet::SubkernelMessageAck { destination: source })
+                    .await
+                    .unwrap();
+                None
+            }
+            // routable packets
+            Packet::DmaAddTraceRequest { destination, .. }
+            | Packet::DmaAddTraceReply { destination, .. }
+            | Packet::DmaRemoveTraceRequest { destination, .. }
+            | Packet::DmaRemoveTraceReply { destination, .. }
+            | Packet::DmaPlaybackRequest { destination, .. }
+            | Packet::DmaPlaybackReply { destination, .. }
+            | Packet::SubkernelLoadRunRequest { destination, .. }
+            | Packet::SubkernelLoadRunReply { destination, .. }
+            | Packet::SubkernelMessage { destination, .. }
+            | Packet::SubkernelMessageAck { destination, .. }
+            | Packet::DmaPlaybackStatus { destination, .. }
+            | Packet::SubkernelFinished { destination, .. } => {
+                if destination == 0 {
+                    Some(packet)
+                } else {
+                    let dest_link = routing_table.0[destination as usize][0] - 1;
+                    if dest_link == linkno {
+                        warn!(
+                            "[LINK#{}] Re-routed packet would return to the same link, dropping: {:?}",
+                            linkno, packet
+                        );
+                    } else {
+                        drtioaux_async::send(dest_link, &packet).await.unwrap();
+                    }
+                    None
+                }
+            }
+            other => Some(other),
         }
     }
 
-    async fn recv_aux_timeout(linkno: u8, timeout: u64, timer: GlobalTimer) -> Result<Packet, &'static str> {
+    async fn recv_aux_timeout(linkno: u8, timeout: u64, timer: GlobalTimer) -> Result<Packet, Error> {
         if !link_rx_up(linkno).await {
-            return Err("link went down");
+            return Err(Error::LinkDown);
         }
         match drtioaux_async::recv_timeout(linkno, Some(timeout), timer).await {
             Ok(packet) => return Ok(packet),
-            Err(Error::TimedOut) => return Err("timed out"),
+            Err(DrtioError::TimedOut) => return Err(Error::Timeout),
-            Err(_) => return Err("aux packet error"),
+            Err(_) => return Err(Error::AuxError),
         }
     }
 
-    pub async fn aux_transact(aux_mutex: &Mutex<bool>, linkno: u8, request: &Packet,
+    pub async fn aux_transact(
-            timer: GlobalTimer) -> Result<Packet, &'static str> {
+        aux_mutex: &Mutex<bool>,
+        linkno: u8,
+        routing_table: &RoutingTable,
+        request: &Packet,
+        timer: GlobalTimer,
+    ) -> Result<Packet, Error> {
         if !link_rx_up(linkno).await {
-            return Err("link went down");
+            return Err(Error::LinkDown);
         }
         let _lock = aux_mutex.async_lock().await;
         drtioaux_async::send(linkno, request).await.unwrap();
-        recv_aux_timeout(linkno, 200, timer).await
+        loop {
+            let packet = recv_aux_timeout(linkno, 200, timer).await?;
+            if let Some(packet) = process_async_packets(linkno, routing_table, packet).await {
+                return Ok(packet);
+            }
+        }
     }
 
     async fn drain_buffer(linkno: u8, draining_time: Milliseconds, timer: GlobalTimer) {
@@ -63,22 +184,27 @@ pub mod drtio {
         loop {
             if timer.get_time() > max_time {
                 return;
-            } //could this be cut short?
+            }
             let _ = drtioaux_async::recv(linkno).await;
         }
     }
 
-    async fn ping_remote(aux_mutex: &Rc<Mutex<bool>>, linkno: u8, timer: GlobalTimer) -> u32 {
+    async fn ping_remote(
+        aux_mutex: &Rc<Mutex<bool>>,
+        linkno: u8,
+        routing_table: &RoutingTable,
+        timer: GlobalTimer,
+    ) -> u32 {
         let mut count = 0;
         loop {
             if !link_rx_up(linkno).await {
-                return 0
+                return 0;
             }
             count += 1;
             if count > 100 {
                 return 0;
             }
-            let reply = aux_transact(aux_mutex, linkno, &Packet::EchoRequest, timer).await;
+            let reply = aux_transact(aux_mutex, linkno, routing_table, &Packet::EchoRequest, timer).await;
             match reply {
                 Ok(Packet::EchoReply) => {
                     // make sure receive buffer is drained
@@ -91,7 +217,7 @@ pub mod drtio {
         }
     }
 
-    async fn sync_tsc(aux_mutex: &Rc<Mutex<bool>>, linkno: u8, timer: GlobalTimer) -> Result<(), &'static str> {
+    async fn sync_tsc(aux_mutex: &Rc<Mutex<bool>>, linkno: u8, timer: GlobalTimer) -> Result<(), Error> {
         let _lock = aux_mutex.async_lock().await;
 
         unsafe {
@@ -102,34 +228,56 @@ pub mod drtio {
         // by the satellite, in response to a TSC set on the RT link.
         let reply = recv_aux_timeout(linkno, 10000, timer).await?;
         if reply == Packet::TSCAck {
-            return Ok(());
+            Ok(())
         } else {
-            return Err("unexpected reply");
+            Err(Error::UnexpectedReply)
         }
     }
 
-    async fn load_routing_table(aux_mutex: &Rc<Mutex<bool>>, linkno: u8, routing_table: &drtio_routing::RoutingTable,
+    async fn load_routing_table(
-        timer: GlobalTimer) -> Result<(), &'static str> {
+        aux_mutex: &Rc<Mutex<bool>>,
+        linkno: u8,
+        routing_table: &RoutingTable,
+        timer: GlobalTimer,
+    ) -> Result<(), Error> {
         for i in 0..drtio_routing::DEST_COUNT {
-            let reply = aux_transact(aux_mutex, linkno, &Packet::RoutingSetPath {
+            let reply = aux_transact(
-                destination: i as u8,
+                aux_mutex,
-                hops: routing_table.0[i]
+                linkno,
-            }, timer).await?;
+                routing_table,
+                &Packet::RoutingSetPath {
+                    destination: i as u8,
+                    hops: routing_table.0[i],
+                },
+                timer,
+            )
+            .await?;
             if reply != Packet::RoutingAck {
-                return Err("unexpected reply");
+                return Err(Error::UnexpectedReply);
             }
         }
         Ok(())
     }
 
-    async fn set_rank(aux_mutex: &Rc<Mutex<bool>>, linkno: u8, rank: u8, timer: GlobalTimer) -> Result<(), &'static str> {
+    async fn set_rank(
-        let reply = aux_transact(aux_mutex, linkno, &Packet::RoutingSetRank {
+        aux_mutex: &Rc<Mutex<bool>>,
-            rank: rank
+        linkno: u8,
-        }, timer).await?;
+        rank: u8,
-        if reply != Packet::RoutingAck {
+        routing_table: &RoutingTable,
-            return Err("unexpected reply");
+        timer: GlobalTimer,
+    ) -> Result<(), Error> {
+        let reply = aux_transact(
+            aux_mutex,
+            linkno,
+            routing_table,
+            &Packet::RoutingSetRank { rank: rank },
+            timer,
+        )
+        .await?;
+        match reply {
+            Packet::RoutingAck => Ok(()),
+            _ => Err(Error::UnexpectedReply),
         }
-        Ok(())
     }
 
     async fn init_buffer_space(destination: u8, linkno: u8) {
@@ -139,18 +287,25 @@ pub mod drtio {
             (csr::DRTIO[linkno].force_destination_write)(1);
             (csr::DRTIO[linkno].o_get_buffer_space_write)(1);
             while (csr::DRTIO[linkno].o_wait_read)() == 1 {}
-            info!("[DEST#{}] buffer space is {}",
+            info!(
-                destination, (csr::DRTIO[linkno].o_dbg_buffer_space_read)());
+                "[DEST#{}] buffer space is {}",
+                destination,
+                (csr::DRTIO[linkno].o_dbg_buffer_space_read)()
+            );
             (csr::DRTIO[linkno].force_destination_write)(0);
         }
     }
 
-    async fn process_unsolicited_aux(aux_mutex: &Rc<Mutex<bool>>, linkno: u8) {
+    async fn process_unsolicited_aux(aux_mutex: &Mutex<bool>, linkno: u8, routing_table: &RoutingTable) {
         let _lock = aux_mutex.async_lock().await;
         match drtioaux_async::recv(linkno).await {
-            Ok(Some(packet)) => warn!("[LINK#{}] unsolicited aux packet: {:?}", linkno, packet),
+            Ok(Some(packet)) => {
+                if let Some(packet) = process_async_packets(linkno, routing_table, packet).await {
+                    warn!("[LINK#{}] unsolicited aux packet: {:?}", linkno, packet);
+                }
+            }
             Ok(None) => (),
-            Err(_) => warn!("[LINK#{}] aux packet error", linkno)
+            Err(_) => warn!("[LINK#{}] aux packet error", linkno),
         }
     }
 
@@ -175,9 +330,12 @@ pub mod drtio {
         }
     }
 
-    async fn destination_set_up(routing_table: &drtio_routing::RoutingTable,
+    async fn destination_set_up(
-            up_destinations: &Rc<RefCell<[bool; drtio_routing::DEST_COUNT]>>,
+        routing_table: &RoutingTable,
-            destination: u8, up: bool) {
+        up_destinations: &Rc<RefCell<[bool; drtio_routing::DEST_COUNT]>>,
+        destination: u8,
+        up: bool,
+    ) {
         let mut up_destinations = up_destinations.borrow_mut();
         up_destinations[destination as usize] = up;
         if up {
@@ -194,10 +352,13 @@ pub mod drtio {
         up_destinations[destination as usize]
     }
 
-    async fn destination_survey(aux_mutex: &Rc<Mutex<bool>>, routing_table: &drtio_routing::RoutingTable,
+    async fn destination_survey(
-            up_links: &[bool],
+        aux_mutex: &Rc<Mutex<bool>>,
-            up_destinations: &Rc<RefCell<[bool; drtio_routing::DEST_COUNT]>>,
+        routing_table: &RoutingTable,
-            timer: GlobalTimer) {
+        up_links: &[bool],
+        up_destinations: &Rc<RefCell<[bool; drtio_routing::DEST_COUNT]>>,
+        timer: GlobalTimer,
+    ) {
         for destination in 0..drtio_routing::DEST_COUNT {
             let hop = routing_table.0[destination][0];
             let destination = destination as u8;
@@ -211,44 +372,84 @@ pub mod drtio {
                 let linkno = hop - 1;
                 if destination_up(up_destinations, destination).await {
                     if up_links[linkno as usize] {
-                        let reply = aux_transact(aux_mutex, linkno, &Packet::DestinationStatusRequest {
+                        let reply = aux_transact(
-                            destination: destination
+                            aux_mutex,
-                        }, timer).await;
+                            linkno,
+                            routing_table,
+                            &Packet::DestinationStatusRequest {
+                                destination: destination,
+                            },
+                            timer,
+                        )
+                        .await;
                         match reply {
-                            Ok(Packet::DestinationDownReply) =>
+                            Ok(Packet::DestinationDownReply) => {
-                                destination_set_up(routing_table, up_destinations, destination, false).await,
+                                destination_set_up(routing_table, up_destinations, destination, false).await;
+                                remote_dma::destination_changed(aux_mutex, routing_table, timer, destination, false)
+                                    .await;
+                                subkernel::destination_changed(aux_mutex, routing_table, timer, destination, false)
+                                    .await;
+                            }
                             Ok(Packet::DestinationOkReply) => (),
-                            Ok(Packet::DestinationSequenceErrorReply { channel }) =>{
+                            Ok(Packet::DestinationSequenceErrorReply { channel }) => {
-                                error!("[DEST#{}] RTIO sequence error involving channel 0x{:04x}", destination, channel);
+                                error!(
+                                    "[DEST#{}] RTIO sequence error involving channel 0x{:04x}:{}",
+                                    destination,
+                                    channel,
+                                    resolve_channel_name(channel as u32)
+                                );
                                 unsafe { SEEN_ASYNC_ERRORS |= ASYNC_ERROR_SEQUENCE_ERROR };
                             }
-                            Ok(Packet::DestinationCollisionReply { channel }) =>{
+                            Ok(Packet::DestinationCollisionReply { channel }) => {
-                                error!("[DEST#{}] RTIO collision involving channel 0x{:04x}", destination, channel);
+                                error!(
+                                    "[DEST#{}] RTIO collision involving channel 0x{:04x}:{}",
+                                    destination,
+                                    channel,
+                                    resolve_channel_name(channel as u32)
+                                );
                                 unsafe { SEEN_ASYNC_ERRORS |= ASYNC_ERROR_COLLISION };
                             }
-                            Ok(Packet::DestinationBusyReply { channel }) =>{
+                            Ok(Packet::DestinationBusyReply { channel }) => {
-                                error!("[DEST#{}] RTIO busy error involving channel 0x{:04x}", destination, channel);
+                                error!(
+                                    "[DEST#{}] RTIO busy error involving channel 0x{:04x}:{}",
+                                    destination,
+                                    channel,
+                                    resolve_channel_name(channel as u32)
+                                );
                                 unsafe { SEEN_ASYNC_ERRORS |= ASYNC_ERROR_BUSY };
                             }
                             Ok(packet) => error!("[DEST#{}] received unexpected aux packet: {:?}", destination, packet),
-                            Err(e) => error!("[DEST#{}] communication failed ({})", destination, e)
+                            Err(e) => error!("[DEST#{}] communication failed ({})", destination, e),
                         }
                     } else {
                         destination_set_up(routing_table, up_destinations, destination, false).await;
+                        remote_dma::destination_changed(aux_mutex, routing_table, timer, destination, false).await;
+                        subkernel::destination_changed(aux_mutex, routing_table, timer, destination, false).await;
                     }
                 } else {
                     if up_links[linkno as usize] {
-                        let reply = aux_transact(aux_mutex, linkno, &Packet::DestinationStatusRequest {
+                        let reply = aux_transact(
-                            destination: destination
+                            aux_mutex,
-                        }, timer).await;
+                            linkno,
+                            routing_table,
+                            &Packet::DestinationStatusRequest {
+                                destination: destination,
+                            },
+                            timer,
+                        )
+                        .await;
                         match reply {
                             Ok(Packet::DestinationDownReply) => (),
                             Ok(Packet::DestinationOkReply) => {
                                 destination_set_up(routing_table, up_destinations, destination, true).await;
                                 init_buffer_space(destination as u8, linkno).await;
-                            },
+                                remote_dma::destination_changed(aux_mutex, routing_table, timer, destination, true)
+                                    .await;
+                                subkernel::destination_changed(aux_mutex, routing_table, timer, destination, true)
+                                    .await;
+                            }
                             Ok(packet) => error!("[DEST#{}] received unexpected aux packet: {:?}", destination, packet),
-                            Err(e) => error!("[DEST#{}] communication failed ({})", destination, e)
+                            Err(e) => error!("[DEST#{}] communication failed ({})", destination, e),
                         }
                     }
                 }
@@ -256,10 +457,12 @@ pub mod drtio {
         }
     }
 
-    pub async fn link_task(aux_mutex: &Rc<Mutex<bool>>,
+    pub async fn link_task(
-            routing_table: &drtio_routing::RoutingTable,
+        aux_mutex: &Rc<Mutex<bool>>,
-            up_destinations: &Rc<RefCell<[bool; drtio_routing::DEST_COUNT]>>,
+        routing_table: &RoutingTable,
-            timer: GlobalTimer) {
+        up_destinations: &Rc<RefCell<[bool; drtio_routing::DEST_COUNT]>>,
+        timer: GlobalTimer,
+    ) {
         let mut up_links = [false; csr::DRTIO.len()];
         loop {
             for linkno in 0..csr::DRTIO.len() {
@@ -267,7 +470,7 @@ pub mod drtio {
                 if up_links[linkno as usize] {
                     /* link was previously up */
                     if link_rx_up(linkno).await {
-                        process_unsolicited_aux(aux_mutex, linkno).await;
+                        process_unsolicited_aux(aux_mutex, linkno, routing_table).await;
                         process_local_errors(linkno).await;
                     } else {
                         info!("[LINK#{}] link is down", linkno);
@@ -277,7 +480,7 @@ pub mod drtio {
                     /* link was previously down */
                     if link_rx_up(linkno).await {
                         info!("[LINK#{}] link RX became up, pinging", linkno);
-                        let ping_count = ping_remote(aux_mutex, linkno, timer).await;
+                        let ping_count = ping_remote(aux_mutex, linkno, routing_table, timer).await;
                         if ping_count > 0 {
                             info!("[LINK#{}] remote replied after {} packets", linkno, ping_count);
                             up_links[linkno as usize] = true;
@@ -287,7 +490,7 @@ pub mod drtio {
                             if let Err(e) = load_routing_table(aux_mutex, linkno, routing_table, timer).await {
                                 error!("[LINK#{}] failed to load routing table ({})", linkno, e);
                             }
-                            if let Err(e) = set_rank(aux_mutex, linkno, 1 as u8, timer).await {
+                            if let Err(e) = set_rank(aux_mutex, linkno, 1 as u8, routing_table, timer).await {
                                 error!("[LINK#{}] failed to set rank ({})", linkno, e);
                             }
                             info!("[LINK#{}] link initialization completed", linkno);
@@ -304,7 +507,7 @@ pub mod drtio {
     }
 
     #[allow(dead_code)]
-    pub fn reset(aux_mutex: Rc<Mutex<bool>>, mut timer: GlobalTimer) {
+    pub fn reset(aux_mutex: Rc<Mutex<bool>>, routing_table: &RoutingTable, mut timer: GlobalTimer) {
         for linkno in 0..csr::DRTIO.len() {
             unsafe {
                 (csr::DRTIO[linkno].reset_write)(1);
@@ -320,33 +523,387 @@ pub mod drtio {
         for linkno in 0..csr::DRTIO.len() {
             let linkno = linkno as u8;
             if task::block_on(link_rx_up(linkno)) {
-                let reply = task::block_on(aux_transact(&aux_mutex, linkno,
+                let reply = task::block_on(aux_transact(
-                    &Packet::ResetRequest, timer));
+                    &aux_mutex,
+                    linkno,
+                    routing_table,
+                    &Packet::ResetRequest,
+                    timer,
+                ));
                 match reply {
                     Ok(Packet::ResetAck) => (),
                     Ok(_) => error!("[LINK#{}] reset failed, received unexpected aux packet", linkno),
-                    Err(e) => error!("[LINK#{}] reset failed, aux packet error ({})", linkno, e)
+                    Err(e) => error!("[LINK#{}] reset failed, aux packet error ({})", linkno, e),
                 }
             }
         }
     }
+
+    async fn partition_data<PacketF, HandlerF>(
+        linkno: u8,
+        aux_mutex: &Rc<Mutex<bool>>,
+        routing_table: &RoutingTable,
+        timer: GlobalTimer,
+        data: &[u8],
+        packet_f: PacketF,
+        reply_handler_f: HandlerF,
+    ) -> Result<(), Error>
+    where
+        PacketF: Fn(&[u8; MASTER_PAYLOAD_MAX_SIZE], PayloadStatus, usize) -> Packet,
+        HandlerF: Fn(&Packet) -> Result<(), Error>,
+    {
+        let mut i = 0;
+        while i < data.len() {
+            let mut slice: [u8; MASTER_PAYLOAD_MAX_SIZE] = [0; MASTER_PAYLOAD_MAX_SIZE];
+            let len: usize = if i + MASTER_PAYLOAD_MAX_SIZE < data.len() {
+                MASTER_PAYLOAD_MAX_SIZE
+            } else {
+                data.len() - i
+            } as usize;
+            let first = i == 0;
+            let last = i + len == data.len();
+            slice[..len].clone_from_slice(&data[i..i + len]);
+            i += len;
+            let status = PayloadStatus::from_status(first, last);
+            let packet = packet_f(&slice, status, len);
+            let reply = aux_transact(aux_mutex, linkno, routing_table, &packet, timer).await?;
+            reply_handler_f(&reply)?;
+        }
+        Ok(())
+    }
+
+    pub async fn ddma_upload_trace(
+        aux_mutex: &Rc<Mutex<bool>>,
+        routing_table: &RoutingTable,
+        timer: GlobalTimer,
+        id: u32,
+        destination: u8,
+        trace: &Vec<u8>,
+    ) -> Result<(), Error> {
+        let linkno = routing_table.0[destination as usize][0] - 1;
+        partition_data(
+            linkno,
+            aux_mutex,
+            routing_table,
+            timer,
+            trace,
+            |slice, status, len| Packet::DmaAddTraceRequest {
+                id: id,
+                source: 0,
+                destination: destination,
+                status: status,
+                length: len as u16,
+                trace: *slice,
+            },
+            |reply| match reply {
+                Packet::DmaAddTraceReply {
+                    destination: 0,
+                    succeeded: true,
+                    ..
+                } => Ok(()),
+                Packet::DmaAddTraceReply {
+                    destination: 0,
+                    succeeded: false,
+                    ..
+                } => Err(Error::DmaAddTraceFail(destination)),
+                _ => Err(Error::UnexpectedReply),
+            },
+        )
+        .await
+    }
+
+    pub async fn ddma_send_erase(
+        aux_mutex: &Rc<Mutex<bool>>,
+        routing_table: &RoutingTable,
+        timer: GlobalTimer,
+        id: u32,
+        destination: u8,
+    ) -> Result<(), Error> {
+        let linkno = routing_table.0[destination as usize][0] - 1;
+        let reply = aux_transact(
+            aux_mutex,
+            linkno,
+            routing_table,
+            &Packet::DmaRemoveTraceRequest {
+                id: id,
+                source: 0,
+                destination: destination,
+            },
+            timer,
+        )
+        .await?;
+        match reply {
+            Packet::DmaRemoveTraceReply {
+                destination: 0,
+                succeeded: true,
+            } => Ok(()),
+            Packet::DmaRemoveTraceReply {
+                destination: 0,
+                succeeded: false,
+            } => Err(Error::DmaEraseFail(destination)),
+            _ => Err(Error::UnexpectedReply),
+        }
+    }
+
+    pub async fn ddma_send_playback(
+        aux_mutex: &Rc<Mutex<bool>>,
+        routing_table: &RoutingTable,
+        timer: GlobalTimer,
+        id: u32,
+        destination: u8,
+        timestamp: u64,
+    ) -> Result<(), Error> {
+        let linkno = routing_table.0[destination as usize][0] - 1;
+        let reply = aux_transact(
+            aux_mutex,
+            linkno,
+            routing_table,
+            &Packet::DmaPlaybackRequest {
+                id: id,
+                source: 0,
+                destination: destination,
+                timestamp: timestamp,
+            },
+            timer,
+        )
+        .await?;
+        match reply {
+            Packet::DmaPlaybackReply {
+                destination: 0,
+                succeeded: true,
+            } => Ok(()),
+            Packet::DmaPlaybackReply {
+                destination: 0,
+                succeeded: false,
+            } => Err(Error::DmaPlaybackFail(destination)),
+            _ => Err(Error::UnexpectedReply),
+        }
+    }
+
+    async fn analyzer_get_data(
+        aux_mutex: &Rc<Mutex<bool>>,
+        routing_table: &RoutingTable,
+        timer: GlobalTimer,
+        destination: u8,
+    ) -> Result<RemoteBuffer, Error> {
+        let linkno = routing_table.0[destination as usize][0] - 1;
+        let reply = aux_transact(
+            aux_mutex,
+            linkno,
+            routing_table,
+            &Packet::AnalyzerHeaderRequest {
+                destination: destination,
+            },
+            timer,
+        )
+        .await?;
+        let (sent, total, overflow) = match reply {
+            Packet::AnalyzerHeader {
+                sent_bytes,
+                total_byte_count,
+                overflow_occurred,
+            } => (sent_bytes, total_byte_count, overflow_occurred),
+            _ => return Err(Error::UnexpectedReply),
+        };
+
+        let mut remote_data: Vec<u8> = Vec::new();
+        if sent > 0 {
+            let mut last_packet = false;
+            while !last_packet {
+                let reply = aux_transact(
+                    aux_mutex,
+                    linkno,
+                    routing_table,
+                    &Packet::AnalyzerDataRequest {
+                        destination: destination,
+                    },
+                    timer,
+                )
+                .await?;
+                match reply {
+                    Packet::AnalyzerData { last, length, data } => {
+                        last_packet = last;
+                        remote_data.extend(&data[0..length as usize]);
+                    }
+                    _ => return Err(Error::UnexpectedReply),
+                }
+            }
+        }
+
+        Ok(RemoteBuffer {
+            sent_bytes: sent,
+            total_byte_count: total,
+            error: overflow,
+            data: remote_data,
+        })
+    }
+
+    pub async fn analyzer_query(
+        aux_mutex: &Rc<Mutex<bool>>,
+        routing_table: &RoutingTable,
+        up_destinations: &Rc<RefCell<[bool; drtio_routing::DEST_COUNT]>>,
+        timer: GlobalTimer,
+    ) -> Result<Vec<RemoteBuffer>, Error> {
+        let mut remote_buffers: Vec<RemoteBuffer> = Vec::new();
+        for i in 1..drtio_routing::DEST_COUNT {
+            if destination_up(up_destinations, i as u8).await {
+                remote_buffers.push(analyzer_get_data(aux_mutex, routing_table, timer, i as u8).await?);
+            }
+        }
+        Ok(remote_buffers)
+    }
+
+    pub async fn subkernel_upload(
+        aux_mutex: &Rc<Mutex<bool>>,
+        routing_table: &RoutingTable,
+        timer: GlobalTimer,
+        id: u32,
+        destination: u8,
+        data: &Vec<u8>,
+    ) -> Result<(), Error> {
+        let linkno = routing_table.0[destination as usize][0] - 1;
+        partition_data(
+            linkno,
+            aux_mutex,
+            routing_table,
+            timer,
+            data,
+            |slice, status, len| Packet::SubkernelAddDataRequest {
+                id: id,
+                destination: destination,
+                status: status,
+                length: len as u16,
+                data: *slice,
+            },
+            |reply| match reply {
+                Packet::SubkernelAddDataReply { succeeded: true } => Ok(()),
+                Packet::SubkernelAddDataReply { succeeded: false } => Err(Error::SubkernelAddFail(destination)),
+                _ => Err(Error::UnexpectedReply),
+            },
+        )
+        .await
+    }
+
+    pub async fn subkernel_load(
+        aux_mutex: &Rc<Mutex<bool>>,
+        routing_table: &RoutingTable,
+        timer: GlobalTimer,
+        id: u32,
+        destination: u8,
+        run: bool,
+    ) -> Result<(), Error> {
+        let linkno = routing_table.0[destination as usize][0] - 1;
+        let reply = aux_transact(
+            aux_mutex,
+            linkno,
+            routing_table,
+            &Packet::SubkernelLoadRunRequest {
+                id: id,
+                source: 0,
+                destination: destination,
+                run: run,
+            },
+            timer,
+        )
+        .await?;
+        match reply {
+            Packet::SubkernelLoadRunReply {
+                destination: 0,
+                succeeded: true,
+            } => return Ok(()),
+            Packet::SubkernelLoadRunReply {
+                destination: 0,
+                succeeded: false,
+            } => return Err(Error::SubkernelRunFail(destination)),
+            _ => Err(Error::UnexpectedReply),
+        }
+    }
+
+    pub async fn subkernel_retrieve_exception(
+        aux_mutex: &Rc<Mutex<bool>>,
+        routing_table: &RoutingTable,
+        timer: GlobalTimer,
+        destination: u8,
+    ) -> Result<Vec<u8>, Error> {
+        let linkno = routing_table.0[destination as usize][0] - 1;
+        let mut remote_data: Vec<u8> = Vec::new();
+        loop {
+            let reply = aux_transact(
+                aux_mutex,
+                linkno,
+                routing_table,
+                &Packet::SubkernelExceptionRequest {
+                    destination: destination,
+                },
+                timer,
+            )
+            .await?;
+            match reply {
+                Packet::SubkernelException { last, length, data } => {
+                    remote_data.extend(&data[0..length as usize]);
+                    if last {
+                        return Ok(remote_data);
+                    }
+                }
+                _ => return Err(Error::UnexpectedReply),
+            }
+        }
+    }
+
+    pub async fn subkernel_send_message(
+        aux_mutex: &Rc<Mutex<bool>>,
+        routing_table: &RoutingTable,
+        timer: GlobalTimer,
+        id: u32,
+        destination: u8,
+        message: &[u8],
+    ) -> Result<(), Error> {
+        let linkno = routing_table.0[destination as usize][0] - 1;
+        partition_data(
+            linkno,
+            aux_mutex,
+            routing_table,
+            timer,
+            message,
+            |slice, status, len| Packet::SubkernelMessage {
+                source: 0,
+                destination: destination,
+                id: id,
+                status: status,
+                length: len as u16,
+                data: *slice,
+            },
+            |reply| match reply {
+                Packet::SubkernelMessageAck { .. } => Ok(()),
+                _ => Err(Error::UnexpectedReply),
+            },
+        )
+        .await
+    }
 }
 
 #[cfg(not(has_drtio))]
 pub mod drtio {
     use super::*;
 
-    pub fn startup(_aux_mutex: &Rc<Mutex<bool>>, _routing_table: &Rc<RefCell<drtio_routing::RoutingTable>>,
+    pub fn startup(
-        _up_destinations: &Rc<RefCell<[bool; drtio_routing::DEST_COUNT]>>, _timer: GlobalTimer) {}
+        _aux_mutex: &Rc<Mutex<bool>>,
+        _routing_table: &Rc<RefCell<RoutingTable>>,
+        _up_destinations: &Rc<RefCell<[bool; drtio_routing::DEST_COUNT]>>,
+        _timer: GlobalTimer,
+    ) {
+    }
 
     #[allow(dead_code)]
-    pub fn reset(_aux_mutex: Rc<Mutex<bool>>, mut _timer: GlobalTimer) {}
+    pub fn reset(_aux_mutex: Rc<Mutex<bool>>, _routing_table: &RoutingTable, mut _timer: GlobalTimer) {}
 }
 
-pub fn startup(aux_mutex: &Rc<Mutex<bool>>,
+pub fn startup(
-        routing_table: &Rc<RefCell<drtio_routing::RoutingTable>>,
+    aux_mutex: &Rc<Mutex<bool>>,
-        up_destinations: &Rc<RefCell<[bool; drtio_routing::DEST_COUNT]>>, 
+    routing_table: &Rc<RefCell<RoutingTable>>,
-        timer: GlobalTimer) {
+    up_destinations: &Rc<RefCell<[bool; drtio_routing::DEST_COUNT]>>,
+    timer: GlobalTimer,
+) {
     drtio::startup(aux_mutex, routing_table, up_destinations, timer);
     unsafe {
         csr::rtio_core::reset_phy_write(1);
@@ -354,9 +911,9 @@ pub fn startup(aux_mutex: &Rc<Mutex<bool>>,
 }
 
 #[allow(dead_code)]
-pub fn reset(aux_mutex: Rc<Mutex<bool>>, timer: GlobalTimer) {
+pub fn reset(aux_mutex: Rc<Mutex<bool>>, routing_table: &RoutingTable, timer: GlobalTimer) {
     unsafe {
         csr::rtio_core::reset_write(1);
     }
-    drtio::reset(aux_mutex, timer)
+    drtio::reset(aux_mutex, routing_table, timer)
 }

src/runtime/src/subkernel.rs

@@ -0,0 +1,325 @@
+use alloc::{collections::BTreeMap, rc::Rc, vec::Vec};
+
+use libasync::task;
+use libboard_artiq::{drtio_routing::RoutingTable,
+                     drtioaux_proto::{PayloadStatus, MASTER_PAYLOAD_MAX_SIZE}};
+use libboard_zynq::{time::Milliseconds, timer::GlobalTimer};
+use libcortex_a9::mutex::Mutex;
+use log::{error, warn};
+
+use crate::rtio_mgt::{drtio, drtio::Error as DrtioError};
+
+#[derive(Debug, PartialEq, Clone, Copy)]
+pub enum FinishStatus {
+    Ok,
+    CommLost,
+    Exception(u8), // exception source
+}
+
+#[derive(Debug, PartialEq, Clone, Copy)]
+pub enum SubkernelState {
+    NotLoaded,
+    Uploaded,
+    Running,
+    Finished { status: FinishStatus },
+}
+
+#[derive(Debug, PartialEq, Eq, Clone, Copy)]
+pub enum Error {
+    Timeout,
+    IncorrectState,
+    SubkernelNotFound,
+    SubkernelException,
+    CommLost,
+    DrtioError(DrtioError),
+}
+
+impl From<DrtioError> for Error {
+    fn from(value: DrtioError) -> Error {
+        Error::DrtioError(value)
+    }
+}
+
+pub struct SubkernelFinished {
+    pub id: u32,
+    pub status: FinishStatus,
+    pub exception: Option<Vec<u8>>,
+}
+
+struct Subkernel {
+    pub destination: u8,
+    pub data: Vec<u8>,
+    pub state: SubkernelState,
+}
+
+impl Subkernel {
+    pub fn new(destination: u8, data: Vec<u8>) -> Self {
+        Subkernel {
+            destination: destination,
+            data: data,
+            state: SubkernelState::NotLoaded,
+        }
+    }
+}
+
+static SUBKERNELS: Mutex<BTreeMap<u32, Subkernel>> = Mutex::new(BTreeMap::new());
+
+pub async fn add_subkernel(id: u32, destination: u8, kernel: Vec<u8>) {
+    SUBKERNELS
+        .async_lock()
+        .await
+        .insert(id, Subkernel::new(destination, kernel));
+}
+
+pub async fn upload(
+    aux_mutex: &Rc<Mutex<bool>>,
+    routing_table: &RoutingTable,
+    timer: GlobalTimer,
+    id: u32,
+) -> Result<(), Error> {
+    if let Some(subkernel) = SUBKERNELS.async_lock().await.get_mut(&id) {
+        drtio::subkernel_upload(
+            aux_mutex,
+            routing_table,
+            timer,
+            id,
+            subkernel.destination,
+            &subkernel.data,
+        )
+        .await?;
+        subkernel.state = SubkernelState::Uploaded;
+        Ok(())
+    } else {
+        Err(Error::SubkernelNotFound)
+    }
+}
+
+pub async fn load(
+    aux_mutex: &Rc<Mutex<bool>>,
+    routing_table: &RoutingTable,
+    timer: GlobalTimer,
+    id: u32,
+    run: bool,
+) -> Result<(), Error> {
+    if let Some(subkernel) = SUBKERNELS.async_lock().await.get_mut(&id) {
+        if subkernel.state != SubkernelState::Uploaded {
+            return Err(Error::IncorrectState);
+        }
+        drtio::subkernel_load(aux_mutex, routing_table, timer, id, subkernel.destination, run).await?;
+        if run {
+            subkernel.state = SubkernelState::Running;
+        }
+        Ok(())
+    } else {
+        Err(Error::SubkernelNotFound)
+    }
+}
+
+pub async fn clear_subkernels() {
+    SUBKERNELS.async_lock().await.clear();
+    MESSAGE_QUEUE.async_lock().await.clear();
+    CURRENT_MESSAGES.async_lock().await.clear();
+}
+
+pub async fn subkernel_finished(id: u32, with_exception: bool, exception_src: u8) {
+    // called upon receiving DRTIO SubkernelRunDone
+    // may be None if session ends and is cleared
+    if let Some(subkernel) = SUBKERNELS.async_lock().await.get_mut(&id) {
+        if subkernel.state == SubkernelState::Running {
+            subkernel.state = SubkernelState::Finished {
+                status: match with_exception {
+                    true => FinishStatus::Exception(exception_src),
+                    false => FinishStatus::Ok,
+                },
+            }
+        }
+    }
+}
+
+pub async fn destination_changed(
+    aux_mutex: &Rc<Mutex<bool>>,
+    routing_table: &RoutingTable,
+    timer: GlobalTimer,
+    destination: u8,
+    up: bool,
+) {
+    let mut locked_subkernels = SUBKERNELS.async_lock().await;
+    for (id, subkernel) in locked_subkernels.iter_mut() {
+        if subkernel.destination == destination {
+            if up {
+                match drtio::subkernel_upload(aux_mutex, routing_table, timer, *id, destination, &subkernel.data).await
+                {
+                    Ok(_) => subkernel.state = SubkernelState::Uploaded,
+                    Err(e) => error!("Error adding subkernel on destination {}: {}", destination, e),
+                }
+            } else {
+                subkernel.state = match subkernel.state {
+                    SubkernelState::Running => SubkernelState::Finished {
+                        status: FinishStatus::CommLost,
+                    },
+                    _ => SubkernelState::NotLoaded,
+                }
+            }
+        }
+    }
+}
+
+pub async fn await_finish(
+    aux_mutex: &Rc<Mutex<bool>>,
+    routing_table: &RoutingTable,
+    timer: GlobalTimer,
+    id: u32,
+    timeout: i64,
+) -> Result<SubkernelFinished, Error> {
+    match SUBKERNELS.async_lock().await.get(&id).unwrap().state {
+        SubkernelState::Running | SubkernelState::Finished { .. } => (),
+        _ => return Err(Error::IncorrectState),
+    }
+    if timeout > 0 {
+        let max_time = timer.get_time() + Milliseconds(timeout as u64);
+        while timer.get_time() < max_time {
+            match SUBKERNELS.async_lock().await.get(&id).unwrap().state {
+                SubkernelState::Finished { .. } => break,
+                _ => (),
+            };
+            task::r#yield().await;
+        }
+        if timer.get_time() >= max_time {
+            error!("Remote subkernel finish await timed out");
+            return Err(Error::Timeout);
+        }
+    } else {
+        // no timeout, wait forever
+        loop {
+            match SUBKERNELS.async_lock().await.get(&id).unwrap().state {
+                SubkernelState::Finished { .. } => break,
+                _ => (),
+            };
+            task::r#yield().await;
+        }
+    }
+    if let Some(subkernel) = SUBKERNELS.async_lock().await.get_mut(&id) {
+        match subkernel.state {
+            SubkernelState::Finished { status } => {
+                subkernel.state = SubkernelState::Uploaded;
+                Ok(SubkernelFinished {
+                    id: id,
+                    status: status,
+                    exception: if let FinishStatus::Exception(dest) = status {
+                        Some(drtio::subkernel_retrieve_exception(aux_mutex, routing_table, timer, dest).await?)
+                    } else {
+                        None
+                    },
+                })
+            }
+            _ => Err(Error::IncorrectState),
+        }
+    } else {
+        Err(Error::SubkernelNotFound)
+    }
+}
+
+pub struct Message {
+    from_id: u32,
+    pub count: u8,
+    pub data: Vec<u8>,
+}
+
+// FIFO queue of messages
+static MESSAGE_QUEUE: Mutex<Vec<Message>> = Mutex::new(Vec::new());
+// currently under construction message(s) (can be from multiple sources)
+static CURRENT_MESSAGES: Mutex<BTreeMap<u32, Message>> = Mutex::new(BTreeMap::new());
+
+pub async fn message_handle_incoming(
+    id: u32,
+    status: PayloadStatus,
+    length: usize,
+    data: &[u8; MASTER_PAYLOAD_MAX_SIZE],
+) {
+    // called when receiving a message from satellite
+    {
+        let subkernel_lock = SUBKERNELS.async_lock().await;
+        let subkernel = subkernel_lock.get(&id);
+        if subkernel.is_some() && subkernel.unwrap().state != SubkernelState::Running {
+            // do not add messages for non-running or deleted subkernels
+            warn!("received a message for a non-running subkernel #{}", id);
+            return;
+        }
+    }
+    let mut current_messages = CURRENT_MESSAGES.async_lock().await;
+
+    if status.is_first() {
+        current_messages.remove(&id);
+    }
+
+    match current_messages.get_mut(&id) {
+        Some(message) => message.data.extend(&data[..length]),
+        None => {
+            current_messages.insert(
+                id,
+                Message {
+                    from_id: id,
+                    count: data[0],
+                    data: data[1..length].to_vec(),
+                },
+            );
+        }
+    };
+    if status.is_last() {
+        // when done, remove from working queue
+        MESSAGE_QUEUE
+            .async_lock()
+            .await
+            .push(current_messages.remove(&id).unwrap());
+    }
+}
+
+pub async fn message_await(id: u32, timeout: i64, timer: GlobalTimer) -> Result<Message, Error> {
+    let is_subkernel = SUBKERNELS.async_lock().await.get(&id).is_some();
+    if is_subkernel {
+        match SUBKERNELS.async_lock().await.get(&id).unwrap().state {
+            SubkernelState::Finished {
+                status: FinishStatus::CommLost,
+            } => return Err(Error::CommLost),
+            SubkernelState::Running | SubkernelState::Finished { .. } => (),
+            _ => return Err(Error::IncorrectState),
+        }
+    }
+    let max_time = timer.get_time() + Milliseconds(timeout as u64);
+    while timeout < 0 || (timeout > 0 && timer.get_time() < max_time) {
+        {
+            let mut message_queue = MESSAGE_QUEUE.async_lock().await;
+            for i in 0..message_queue.len() {
+                let msg = &message_queue[i];
+                if msg.from_id == id {
+                    let message = message_queue.remove(i);
+                    return Ok(message);
+                }
+            }
+        }
+        if is_subkernel {
+            match SUBKERNELS.async_lock().await.get(&id).unwrap().state {
+                SubkernelState::Finished {
+                    status: FinishStatus::CommLost,
+                } => return Err(Error::CommLost),
+                SubkernelState::Finished {
+                    status: FinishStatus::Exception(_),
+                } => return Err(Error::SubkernelException),
+                _ => (),
+            }
+        }
+        task::r#yield().await;
+    }
+    Err(Error::Timeout)
+}
+
+pub async fn message_send<'a>(
+    aux_mutex: &Rc<Mutex<bool>>,
+    routing_table: &RoutingTable,
+    timer: GlobalTimer,
+    id: u32,
+    destination: u8,
+    message: Vec<u8>,
+) -> Result<(), Error> {
+    Ok(drtio::subkernel_send_message(aux_mutex, routing_table, timer, id, destination, &message).await?)
+}

src/rustfmt.toml

@@ -0,0 +1,68 @@
+max_width = 120
+hard_tabs = false
+tab_spaces = 4
+newline_style = "Auto"
+use_small_heuristics = "Default"
+indent_style = "Block"
+wrap_comments = false
+format_code_in_doc_comments = false
+comment_width = 100
+normalize_comments = false
+normalize_doc_attributes = false
+license_template_path = ""
+format_strings = true
+format_macro_matchers = true
+format_macro_bodies = true
+empty_item_single_line = true
+struct_lit_single_line = true
+fn_single_line = false
+where_single_line = true
+imports_indent = "Visual"
+imports_layout = "Mixed"
+merge_imports = true
+group_imports = "StdExternalCrate"
+reorder_imports = true
+reorder_modules = true
+reorder_impl_items = false
+type_punctuation_density = "Wide"
+space_before_colon = false
+space_after_colon = true
+spaces_around_ranges = false
+binop_separator = "Front"
+remove_nested_parens = true
+combine_control_expr = true
+overflow_delimited_expr = false
+struct_field_align_threshold = 0
+enum_discrim_align_threshold = 0
+match_arm_blocks = true
+match_arm_leading_pipes = "Never"
+force_multiline_blocks = false
+fn_args_layout = "Tall"
+brace_style = "SameLineWhere"
+control_brace_style = "AlwaysSameLine"
+trailing_semicolon = true
+trailing_comma = "Vertical"
+match_block_trailing_comma = false
+blank_lines_upper_bound = 1
+blank_lines_lower_bound = 0
+edition = "2018"
+version = "Two"
+inline_attribute_width = 0
+merge_derives = true
+use_try_shorthand = false
+use_field_init_shorthand = false
+force_explicit_abi = true
+condense_wildcard_suffixes = false
+color = "Auto"
+required_version = "1.4.32"
+unstable_features = false
+disable_all_formatting = false
+skip_children = false
+hide_parse_errors = false
+error_on_line_overflow = false
+error_on_unformatted = false
+report_todo = "Never"
+report_fixme = "Never"
+ignore = []
+emit_mode = "Files"
+make_backup = false

src/satman/Cargo.toml

@@ -1,28 +0,0 @@
-[package]
-authors = ["M-Labs"]
-name = "satman"
-version = "0.0.0"
-build = "build.rs"
-
-[features]
-target_zc706 = ["libboard_zynq/target_zc706", "libsupport_zynq/target_zc706", "libconfig/target_zc706", "libboard_artiq/target_zc706"]
-target_kasli_soc = ["libboard_zynq/target_kasli_soc", "libsupport_zynq/target_kasli_soc", "libconfig/target_kasli_soc", "libboard_artiq/target_kasli_soc"]
-default = ["target_zc706", ]
-
-[build-dependencies]
-build_zynq = { path = "../libbuild_zynq" }
-
-[dependencies]
-log = { version = "0.4", default-features = false }
-embedded-hal = "0.2"
-
-libboard_zynq = { git = "https://git.m-labs.hk/M-Labs/zynq-rs.git", features = ["ipv6"]}
-libsupport_zynq = { default-features = false, features = ["alloc_core"], git = "https://git.m-labs.hk/M-Labs/zynq-rs.git" }
-libcortex_a9 = { git = "https://git.m-labs.hk/M-Labs/zynq-rs.git" }
-libasync = { git = "https://git.m-labs.hk/M-Labs/zynq-rs.git" }
-libregister = { git = "https://git.m-labs.hk/M-Labs/zynq-rs.git" }
-libconfig = { git = "https://git.m-labs.hk/M-Labs/zynq-rs.git", features = ["fat_lfn", "ipv6"] }
-
-libboard_artiq = { path = "../libboard_artiq" }
-unwind = { path = "../libunwind" }
-libc = { path = "../libc" }

src/satman/Cargo.toml.tpl

@@ -0,0 +1,33 @@
+[package]
+authors = ["M-Labs"]
+name = "satman"
+version = "0.0.0"
+build = "build.rs"
+
+[features]
+target_zc706 = ["libboard_zynq/target_zc706", "libsupport_zynq/target_zc706", "libconfig/target_zc706", "libboard_artiq/target_zc706"]
+target_kasli_soc = ["libboard_zynq/target_kasli_soc", "libsupport_zynq/target_kasli_soc", "libconfig/target_kasli_soc", "libboard_artiq/target_kasli_soc"]
+calibrate_wrpll_skew = ["libboard_artiq/calibrate_wrpll_skew"]
+default = ["target_zc706", ]
+
+[build-dependencies]
+build_zynq = { path = "../libbuild_zynq" }
+
+[dependencies]
+log = { version = "0.4", default-features = false }
+core_io = { version = "0.1", features = ["collections"] }
+cslice = "0.3"
+embedded-hal = "0.2"
+
+libboard_zynq = { path = "@@ZYNQ_RS@@/libboard_zynq", features = ["ipv6"]}
+libsupport_zynq = { path = "@@ZYNQ_RS@@/libsupport_zynq", default-features = false, features = ["alloc_core"] }
+libcortex_a9 = { path = "@@ZYNQ_RS@@/libcortex_a9" }
+libasync = { path = "@@ZYNQ_RS@@/libasync" }
+libregister = { path = "@@ZYNQ_RS@@/libregister" }
+libconfig = { path = "@@ZYNQ_RS@@/libconfig", features = ["fat_lfn", "ipv6"] }
+
+libboard_artiq = { path = "../libboard_artiq" }
+unwind = { path = "../libunwind" }
+libc = { path = "../libc" }
+io = { path = "../libio", features = ["alloc"] }
+ksupport = { path = "../libksupport" }

src/satman/src/analyzer.rs

@@ -0,0 +1,126 @@
+use core::cmp::min;
+
+use libboard_artiq::{drtioaux_proto::SAT_PAYLOAD_MAX_SIZE, pl::csr};
+use libcortex_a9::cache;
+
+const BUFFER_SIZE: usize = 512 * 1024;
+
+#[repr(align(64))]
+struct Buffer {
+    data: [u8; BUFFER_SIZE],
+}
+
+static mut BUFFER: Buffer = Buffer { data: [0; BUFFER_SIZE] };
+
+fn arm() {
+    unsafe {
+        let base_addr = &mut BUFFER.data[0] as *mut _ as usize;
+        let last_addr = &mut BUFFER.data[BUFFER_SIZE - 1] as *mut _ as usize;
+        csr::rtio_analyzer::dma_base_address_write(base_addr as u32);
+        csr::rtio_analyzer::message_encoder_overflow_reset_write(1);
+        csr::rtio_analyzer::dma_last_address_write(last_addr as u32);
+        csr::rtio_analyzer::dma_reset_write(1);
+        csr::rtio_analyzer::enable_write(1);
+    }
+}
+
+fn disarm() {
+    unsafe {
+        csr::rtio_analyzer::enable_write(0);
+        while csr::rtio_analyzer::busy_read() != 0 {}
+        cache::dcci_slice(&BUFFER.data);
+    }
+}
+
+pub struct Analyzer {
+    // necessary for keeping track of sent data
+    data_len: usize,
+    sent_bytes: usize,
+    data_pointer: usize,
+}
+
+pub struct Header {
+    pub total_byte_count: u64,
+    pub sent_bytes: u32,
+    pub error: bool,
+}
+
+pub struct AnalyzerSliceMeta {
+    pub len: u16,
+    pub last: bool,
+}
+
+impl Drop for Analyzer {
+    fn drop(&mut self) {
+        disarm();
+    }
+}
+
+impl Analyzer {
+    pub fn new() -> Analyzer {
+        // create and arm new Analyzer
+        arm();
+        Analyzer {
+            data_len: 0,
+            sent_bytes: 0,
+            data_pointer: 0,
+        }
+    }
+
+    pub fn get_header(&mut self) -> Header {
+        disarm();
+
+        let overflow = unsafe { csr::rtio_analyzer::message_encoder_overflow_read() != 0 };
+        let bus_err = unsafe { csr::rtio_analyzer::dma_bus_error_read() != 0 };
+        let total_byte_count = unsafe { csr::rtio_analyzer::dma_byte_count_read() as u64 };
+        let wraparound = total_byte_count >= BUFFER_SIZE as u64;
+        self.data_len = if wraparound {
+            BUFFER_SIZE
+        } else {
+            total_byte_count as usize
+        };
+        self.data_pointer = if wraparound {
+            (total_byte_count % BUFFER_SIZE as u64) as usize
+        } else {
+            0
+        };
+        self.sent_bytes = 0;
+
+        if overflow {
+            warn!("overflow occured");
+        }
+        if bus_err {
+            warn!("bus error occured");
+        }
+
+        Header {
+            total_byte_count: total_byte_count,
+            sent_bytes: self.data_len as u32,
+            error: overflow | bus_err,
+        }
+    }
+
+    pub fn get_data(&mut self, data_slice: &mut [u8; SAT_PAYLOAD_MAX_SIZE]) -> AnalyzerSliceMeta {
+        let data = unsafe { &BUFFER.data[..] };
+        let i = (self.data_pointer + self.sent_bytes) % BUFFER_SIZE;
+        let len = min(SAT_PAYLOAD_MAX_SIZE, self.data_len - self.sent_bytes);
+        let last = self.sent_bytes + len == self.data_len;
+
+        if i + len >= BUFFER_SIZE {
+            data_slice[..(BUFFER_SIZE - i)].clone_from_slice(&data[i..BUFFER_SIZE]);
+            data_slice[(BUFFER_SIZE - i)..len].clone_from_slice(&data[..(i + len) % BUFFER_SIZE]);
+        } else {
+            data_slice[..len].clone_from_slice(&data[i..i + len]);
+        }
+        self.sent_bytes += len;
+
+        if last {
+            arm();
+        }
+
+        AnalyzerSliceMeta {
+            len: len as u16,
+            last: last,
+        }
+    }
+}

src/satman/src/dma.rs

@@ -0,0 +1,563 @@
+use alloc::{collections::btree_map::BTreeMap, string::String, vec::Vec};
+use core::mem;
+
+use ksupport::kernel::DmaRecorder;
+use libboard_artiq::{drtio_routing::RoutingTable,
+                     drtioaux_proto::{Packet, PayloadStatus, MASTER_PAYLOAD_MAX_SIZE},
+                     pl::csr};
+use libcortex_a9::cache::dcci_slice;
+use routing::{Router, Sliceable};
+use subkernel::Manager as KernelManager;
+
+const ALIGNMENT: usize = 64;
+
+#[derive(Debug, PartialEq)]
+enum ManagerState {
+    Idle,
+    Playback,
+}
+
+pub struct RtioStatus {
+    pub source: u8,
+    pub id: u32,
+    pub error: u8,
+    pub channel: u32,
+    pub timestamp: u64,
+}
+
+#[derive(Debug)]
+pub enum Error {
+    IdNotFound,
+    PlaybackInProgress,
+    EntryNotComplete,
+    MasterDmaFound,
+    UploadFail,
+}
+
+#[derive(Debug)]
+struct Entry {
+    trace: Vec<u8>,
+    padding_len: usize,
+    complete: bool,
+    duration: i64, // relevant for local DMA
+}
+
+impl Entry {
+    pub fn from_vec(data: Vec<u8>, duration: i64) -> Entry {
+        let mut entry = Entry {
+            trace: data,
+            padding_len: 0,
+            complete: true,
+            duration: duration,
+        };
+        entry.realign();
+        entry
+    }
+
+    pub fn id(&self) -> u32 {
+        self.trace[self.padding_len..].as_ptr() as u32
+    }
+
+    pub fn realign(&mut self) {
+        self.trace.push(0);
+        let data_len = self.trace.len();
+
+        self.trace.reserve(ALIGNMENT - 1);
+        let padding = ALIGNMENT - self.trace.as_ptr() as usize % ALIGNMENT;
+        let padding = if padding == ALIGNMENT { 0 } else { padding };
+        for _ in 0..padding {
+            // Vec guarantees that this will not reallocate
+            self.trace.push(0)
+        }
+        for i in 1..data_len + 1 {
+            self.trace[data_len + padding - i] = self.trace[data_len - i]
+        }
+        self.complete = true;
+        self.padding_len = padding;
+
+        dcci_slice(&self.trace);
+    }
+}
+
+#[derive(Debug)]
+enum RemoteTraceState {
+    Unsent,
+    Sending(usize),
+    Ready,
+    Running(usize),
+}
+
+#[derive(Debug)]
+struct RemoteTraces {
+    remote_traces: BTreeMap<u8, Sliceable>,
+    state: RemoteTraceState,
+}
+
+impl RemoteTraces {
+    pub fn new(traces: BTreeMap<u8, Sliceable>) -> RemoteTraces {
+        RemoteTraces {
+            remote_traces: traces,
+            state: RemoteTraceState::Unsent,
+        }
+    }
+
+    // on subkernel request
+    pub fn upload_traces(
+        &mut self,
+        id: u32,
+        router: &mut Router,
+        rank: u8,
+        self_destination: u8,
+        routing_table: &RoutingTable,
+    ) -> usize {
+        let len = self.remote_traces.len();
+        if len > 0 {
+            self.state = RemoteTraceState::Sending(self.remote_traces.len());
+            for (dest, trace) in self.remote_traces.iter_mut() {
+                // queue up the first packet for all destinations, rest will be sent after first ACK
+                let mut data_slice: [u8; MASTER_PAYLOAD_MAX_SIZE] = [0; MASTER_PAYLOAD_MAX_SIZE];
+                let meta = trace.get_slice_master(&mut data_slice);
+                router.route(
+                    Packet::DmaAddTraceRequest {
+                        source: self_destination,
+                        destination: *dest,
+                        id: id,
+                        status: meta.status,
+                        length: meta.len,
+                        trace: data_slice,
+                    },
+                    routing_table,
+                    rank,
+                    self_destination,
+                );
+            }
+        }
+        len
+    }
+
+    // on incoming Packet::DmaAddTraceReply
+    pub fn ack_upload(
+        &mut self,
+        kernel_manager: &mut KernelManager,
+        source: u8,
+        id: u32,
+        succeeded: bool,
+        router: &mut Router,
+        rank: u8,
+        self_destination: u8,
+        routing_table: &RoutingTable,
+    ) {
+        if let RemoteTraceState::Sending(count) = self.state {
+            if let Some(trace) = self.remote_traces.get_mut(&source) {
+                if trace.at_end() {
+                    if count - 1 == 0 {
+                        self.state = RemoteTraceState::Ready;
+                        if let Some((id, timestamp)) = kernel_manager.ddma_remote_uploaded(succeeded) {
+                            self.playback(id, timestamp, router, rank, self_destination, routing_table);
+                        }
+                    } else {
+                        self.state = RemoteTraceState::Sending(count - 1);
+                    }
+                } else {
+                    // send next slice
+                    let mut data_slice: [u8; MASTER_PAYLOAD_MAX_SIZE] = [0; MASTER_PAYLOAD_MAX_SIZE];
+                    let meta = trace.get_slice_master(&mut data_slice);
+                    router.route(
+                        Packet::DmaAddTraceRequest {
+                            source: self_destination,
+                            destination: meta.destination,
+                            id: id,
+                            status: meta.status,
+                            length: meta.len,
+                            trace: data_slice,
+                        },
+                        routing_table,
+                        rank,
+                        self_destination,
+                    );
+                }
+            }
+        }
+    }
+
+    // on subkernel request
+    pub fn playback(
+        &mut self,
+        id: u32,
+        timestamp: u64,
+        router: &mut Router,
+        rank: u8,
+        self_destination: u8,
+        routing_table: &RoutingTable,
+    ) {
+        // route all the playback requests
+        // remote traces (local trace runs on core1 unlike mainline firmware)
+        self.state = RemoteTraceState::Running(self.remote_traces.len());
+        for (dest, _) in self.remote_traces.iter() {
+            router.route(
+                Packet::DmaPlaybackRequest {
+                    source: self_destination,
+                    destination: *dest,
+                    id: id,
+                    timestamp: timestamp,
+                },
+                routing_table,
+                rank,
+                self_destination,
+            );
+            // response will be ignored (succeeded = false handled by the main thread)
+        }
+    }
+
+    // on incoming Packet::DmaPlaybackDone
+    pub fn remote_finished(&mut self, kernel_manager: &mut KernelManager, error: u8, channel: u32, timestamp: u64) {
+        if let RemoteTraceState::Running(count) = self.state {
+            if error != 0 || count - 1 == 0 {
+                // notify the kernel about a DDMA error or finish
+                kernel_manager.ddma_finished(error, channel, timestamp);
+                self.state = RemoteTraceState::Ready;
+                // further messages will be ignored (if there was an error)
+            } else {
+                // no error and not the last one awaited
+                self.state = RemoteTraceState::Running(count - 1);
+            }
+        }
+    }
+
+    pub fn erase(
+        &mut self,
+        id: u32,
+        router: &mut Router,
+        rank: u8,
+        self_destination: u8,
+        routing_table: &RoutingTable,
+    ) {
+        for (dest, _) in self.remote_traces.iter() {
+            router.route(
+                Packet::DmaRemoveTraceRequest {
+                    source: self_destination,
+                    destination: *dest,
+                    id: id,
+                },
+                routing_table,
+                rank,
+                self_destination,
+            );
+            // response will be ignored as this object will stop existing too
+        }
+    }
+
+    pub fn has_remote_traces(&self) -> bool {
+        self.remote_traces.len() > 0
+    }
+}
+
+#[derive(Debug)]
+pub struct Manager {
+    entries: BTreeMap<(u8, u32), Entry>,
+    state: ManagerState,
+    current_id: u32,
+    current_source: u8,
+
+    remote_entries: BTreeMap<u32, RemoteTraces>,
+    name_map: BTreeMap<String, u32>,
+}
+
+impl Manager {
+    pub fn new() -> Manager {
+        // in case Manager is created during a DMA in progress
+        // wait for it to end
+        unsafe { while csr::rtio_dma::enable_read() != 0 {} }
+        Manager {
+            entries: BTreeMap::new(),
+            current_id: 0,
+            current_source: 0,
+            state: ManagerState::Idle,
+            remote_entries: BTreeMap::new(),
+            name_map: BTreeMap::new(),
+        }
+    }
+
+    pub fn add(
+        &mut self,
+        source: u8,
+        id: u32,
+        status: PayloadStatus,
+        trace: &[u8],
+        trace_len: usize,
+    ) -> Result<(), Error> {
+        let entry = match self.entries.get_mut(&(source, id)) {
+            Some(entry) => {
+                if entry.complete || status.is_first() {
+                    // replace entry
+                    self.entries.remove(&(source, id));
+                    self.entries.insert(
+                        (source, id),
+                        Entry {
+                            trace: Vec::new(),
+                            padding_len: 0,
+                            complete: false,
+                            duration: 0,
+                        },
+                    );
+                    self.entries.get_mut(&(source, id)).unwrap()
+                } else {
+                    entry
+                }
+            }
+            None => {
+                self.entries.insert(
+                    (source, id),
+                    Entry {
+                        trace: Vec::new(),
+                        padding_len: 0,
+                        complete: false,
+                        duration: 0,
+                    },
+                );
+                self.entries.get_mut(&(source, id)).unwrap()
+            }
+        };
+        entry.trace.extend(&trace[0..trace_len]);
+
+        if status.is_last() {
+            entry.realign();
+        }
+        Ok(())
+    }
+
+    // api for DRTIO
+    pub fn erase(&mut self, source: u8, id: u32) -> Result<(), Error> {
+        match self.entries.remove(&(source, id)) {
+            Some(_) => Ok(()),
+            None => Err(Error::IdNotFound),
+        }
+    }
+
+    // API for subkernel
+    pub fn erase_name(
+        &mut self,
+        name: &str,
+        router: &mut Router,
+        rank: u8,
+        self_destination: u8,
+        routing_table: &RoutingTable,
+    ) {
+        if let Some(id) = self.name_map.get(name) {
+            if let Some(traces) = self.remote_entries.get_mut(&id) {
+                traces.erase(*id, router, rank, self_destination, routing_table);
+                self.remote_entries.remove(&id);
+            }
+            self.entries.remove(&(self_destination, *id));
+            self.name_map.remove(name);
+        }
+    }
+
+    pub fn remote_finished(
+        &mut self,
+        kernel_manager: &mut KernelManager,
+        id: u32,
+        error: u8,
+        channel: u32,
+        timestamp: u64,
+    ) {
+        if let Some(entry) = self.remote_entries.get_mut(&id) {
+            entry.remote_finished(kernel_manager, error, channel, timestamp);
+        }
+    }
+
+    pub fn ack_upload(
+        &mut self,
+        kernel_manager: &mut KernelManager,
+        source: u8,
+        id: u32,
+        succeeded: bool,
+        router: &mut Router,
+        rank: u8,
+        self_destination: u8,
+        routing_table: &RoutingTable,
+    ) {
+        if let Some(entry) = self.remote_entries.get_mut(&id) {
+            entry.ack_upload(
+                kernel_manager,
+                source,
+                id,
+                succeeded,
+                router,
+                rank,
+                self_destination,
+                routing_table,
+            );
+        }
+    }
+
+    // API for subkernel
+    pub fn upload_traces(
+        &mut self,
+        id: u32,
+        router: &mut Router,
+        rank: u8,
+        self_destination: u8,
+        routing_table: &RoutingTable,
+    ) -> Result<usize, Error> {
+        let remote_traces = self.remote_entries.get_mut(&id);
+        let mut len = 0;
+        if let Some(traces) = remote_traces {
+            len = traces.upload_traces(id, router, rank, self_destination, routing_table);
+        }
+        Ok(len)
+    }
+
+    // API for subkernel
+    pub fn playback_remote(
+        &mut self,
+        id: u32,
+        timestamp: u64,
+        router: &mut Router,
+        rank: u8,
+        self_destination: u8,
+        routing_table: &RoutingTable,
+    ) -> Result<(), Error> {
+        if let Some(traces) = self.remote_entries.get_mut(&id) {
+            traces.playback(id, timestamp, router, rank, self_destination, routing_table);
+            Ok(())
+        } else {
+            Err(Error::IdNotFound)
+        }
+    }
+
+    // API for subkernel
+    pub fn cleanup(&mut self, router: &mut Router, rank: u8, self_destination: u8, routing_table: &RoutingTable) {
+        // after subkernel ends, remove all self-generated traces
+        for (_, id) in self.name_map.iter_mut() {
+            if let Some(traces) = self.remote_entries.get_mut(&id) {
+                traces.erase(*id, router, rank, self_destination, routing_table);
+                self.remote_entries.remove(&id);
+            }
+            self.entries.remove(&(self_destination, *id));
+        }
+        self.name_map.clear();
+    }
+
+    // API for subkernel
+    pub fn retrieve(&self, self_destination: u8, name: &String) -> Option<(i32, i64, bool)> {
+        let id = self.name_map.get(name)?;
+        let duration = self.entries.get(&(self_destination, *id))?.duration;
+        let uses_ddma = self.has_remote_traces(*id);
+        Some((*id as i32, duration, uses_ddma))
+    }
+
+    pub fn has_remote_traces(&self, id: u32) -> bool {
+        match self.remote_entries.get(&id) {
+            Some(traces) => traces.has_remote_traces(),
+            _ => false,
+        }
+    }
+
+    pub fn put_record(&mut self, mut recorder: DmaRecorder, self_destination: u8) -> Result<u32, Error> {
+        let mut remote_traces: BTreeMap<u8, Sliceable> = BTreeMap::new();
+
+        let mut local_trace: Vec<u8> = Vec::new();
+        // analyze each entry and put in proper buckets, as the kernel core
+        // sends whole chunks, to limit comms/kernel CPU communication,
+        // and as only comms core has access to varios DMA buffers.
+        let mut ptr = 0;
+        recorder.buffer.push(0);
+        while recorder.buffer[ptr] != 0 {
+            // ptr + 3 = tgt >> 24 (destination)
+            let len = recorder.buffer[ptr] as usize;
+            let destination = recorder.buffer[ptr + 3];
+            if destination == 0 {
+                return Err(Error::MasterDmaFound);
+            } else if destination == self_destination {
+                local_trace.extend(&recorder.buffer[ptr..ptr + len]);
+            } else {
+                if let Some(remote_trace) = remote_traces.get_mut(&destination) {
+                    remote_trace.extend(&recorder.buffer[ptr..ptr + len]);
+                } else {
+                    remote_traces.insert(
+                        destination,
+                        Sliceable::new(destination, recorder.buffer[ptr..ptr + len].to_vec()),
+                    );
+                }
+            }
+            // and jump to the next event
+            ptr += len;
+        }
+        let local_entry = Entry::from_vec(local_trace, recorder.duration);
+
+        let id = local_entry.id();
+        self.entries.insert((self_destination, id), local_entry);
+        self.remote_entries.insert(id, RemoteTraces::new(remote_traces));
+        let mut name = String::new();
+        mem::swap(&mut recorder.name, &mut name);
+        self.name_map.insert(name, id);
+
+        Ok(id)
+    }
+
+    pub fn playback(&mut self, source: u8, id: u32, timestamp: u64) -> Result<(), Error> {
+        if self.state != ManagerState::Idle {
+            return Err(Error::PlaybackInProgress);
+        }
+
+        let entry = match self.entries.get(&(source, id)) {
+            Some(entry) => entry,
+            None => {
+                return Err(Error::IdNotFound);
+            }
+        };
+        if !entry.complete {
+            return Err(Error::EntryNotComplete);
+        }
+        let ptr = entry.trace[entry.padding_len..].as_ptr();
+        assert!(ptr as u32 % 64 == 0);
+
+        self.state = ManagerState::Playback;
+        self.current_id = id;
+        self.current_source = source;
+
+        unsafe {
+            csr::rtio_dma::base_address_write(ptr as u32);
+            csr::rtio_dma::time_offset_write(timestamp as u64);
+
+            csr::cri_con::selected_write(1);
+            csr::rtio_dma::enable_write(1);
+            // playback has begun here, for status call check_state
+        }
+        Ok(())
+    }
+
+    pub fn check_state(&mut self) -> Option<RtioStatus> {
+        if self.state != ManagerState::Playback {
+            // nothing to report
+            return None;
+        }
+        let dma_enable = unsafe { csr::rtio_dma::enable_read() };
+        if dma_enable != 0 {
+            return None;
+        } else {
+            self.state = ManagerState::Idle;
+            unsafe {
+                csr::cri_con::selected_write(0);
+                let error = csr::rtio_dma::error_read();
+                let channel = csr::rtio_dma::error_channel_read();
+                let timestamp = csr::rtio_dma::error_timestamp_read();
+                if error != 0 {
+                    csr::rtio_dma::error_write(1);
+                }
+                return Some(RtioStatus {
+                    source: self.current_source,
+                    id: self.current_id,
+                    error: error,
+                    channel: channel,
+                    timestamp: timestamp,
+                });
+            }
+        }
+    }
+
+    pub fn running(&self) -> bool {
+        self.state == ManagerState::Playback
+    }
+}

src/satman/src/repeater.rs

@@ -1,19 +1,17 @@
-use libboard_artiq::{drtioaux, drtio_routing};
-use libboard_zynq::timer::GlobalTimer;
-
-#[cfg(has_drtio_routing)]
-use libboard_artiq::{pl::csr};
-#[cfg(has_drtio_routing)]
-use libboard_zynq::time::Milliseconds;
 #[cfg(has_drtio_routing)]
 use embedded_hal::prelude::_embedded_hal_blocking_delay_DelayUs;
+#[cfg(has_drtio_routing)]
+use libboard_artiq::pl::csr;
+use libboard_artiq::{drtio_routing, drtioaux};
+#[cfg(has_drtio_routing)]
+use libboard_zynq::time::Milliseconds;
+use libboard_zynq::timer::GlobalTimer;
+use routing::Router;
 
 #[cfg(has_drtio_routing)]
 fn rep_link_rx_up(repno: u8) -> bool {
     let repno = repno as usize;
-    unsafe {
+    unsafe { (csr::DRTIOREP[repno].rx_up_read)() == 1 }
-        (csr::DRTIOREP[repno].rx_up_read)() == 1
-    }
 }
 
 #[cfg(has_drtio_routing)]
@@ -23,12 +21,14 @@ enum RepeaterState {
     SendPing { ping_count: u16 },
     WaitPingReply { ping_count: u16, timeout: Milliseconds },
     Up,
-    Failed
+    Failed,
 }
 
 #[cfg(has_drtio_routing)]
 impl Default for RepeaterState {
-    fn default() -> RepeaterState { RepeaterState::Down }
+    fn default() -> RepeaterState {
+        RepeaterState::Down
+    }
 }
 
 #[cfg(has_drtio_routing)]
@@ -36,7 +36,7 @@ impl Default for RepeaterState {
 pub struct Repeater {
     repno: u8,
     auxno: u8,
-    state: RepeaterState
+    state: RepeaterState,
 }
 
 #[cfg(has_drtio_routing)]
@@ -45,7 +45,7 @@ impl Repeater {
         Repeater {
             repno: repno,
             auxno: repno + 1,
-            state: RepeaterState::Down
+            state: RepeaterState::Down,
         }
     }
 
@@ -54,8 +54,14 @@ impl Repeater {
         self.state == RepeaterState::Up
     }
 
-    pub fn service(&mut self, routing_table: &drtio_routing::RoutingTable, rank: u8,
+    pub fn service(
-        timer: &mut GlobalTimer) {
+        &mut self,
+        routing_table: &drtio_routing::RoutingTable,
+        rank: u8,
+        destination: u8,
+        router: &mut Router,
+        timer: &mut GlobalTimer,
+    ) {
         self.process_local_errors();
 
         match self.state {
@@ -70,7 +76,7 @@ impl Repeater {
                     drtioaux::send(self.auxno, &drtioaux::Packet::EchoRequest).unwrap();
                     self.state = RepeaterState::WaitPingReply {
                         ping_count: ping_count + 1,
-                        timeout: timer.get_time() + Milliseconds(100)
+                        timeout: timer.get_time() + Milliseconds(100),
                     }
                 } else {
                     error!("[REP#{}] link RX went down during ping", self.repno);
@@ -113,7 +119,7 @@ impl Repeater {
                 }
             }
             RepeaterState::Up => {
-                self.process_unsolicited_aux();
+                self.process_unsolicited_aux(routing_table, rank, destination, router);
                 if !rep_link_rx_up(self.repno) {
                     info!("[REP#{}] link is down", self.repno);
                     self.state = RepeaterState::Down;
@@ -128,11 +134,17 @@ impl Repeater {
         }
     }
 
-    fn process_unsolicited_aux(&self) {
+    fn process_unsolicited_aux(
+        &self,
+        routing_table: &drtio_routing::RoutingTable,
+        rank: u8,
+        destination: u8,
+        router: &mut Router,
+    ) {
         match drtioaux::recv(self.auxno) {
-            Ok(Some(packet)) => warn!("[REP#{}] unsolicited aux packet: {:?}", self.repno, packet),
+            Ok(Some(packet)) => router.route(packet, routing_table, rank, destination),
             Ok(None) => (),
-            Err(_) => warn!("[REP#{}] aux packet error", self.repno)
+            Err(_) => warn!("[REP#{}] aux packet error", self.repno),
         }
     }
 
@@ -155,14 +167,20 @@ impl Repeater {
                 cmd = (csr::DRTIOREP[repno].command_missed_cmd_read)();
                 chan_sel = (csr::DRTIOREP[repno].command_missed_chan_sel_read)();
             }
-            error!("[REP#{}] CRI command missed, cmd={}, chan_sel=0x{:06x}", repno, cmd, chan_sel)
+            error!(
+                "[REP#{}] CRI command missed, cmd={}, chan_sel=0x{:06x}",
+                repno, cmd, chan_sel
+            )
         }
         if errors & 8 != 0 {
             let destination;
             unsafe {
                 destination = (csr::DRTIOREP[repno].buffer_space_timeout_dest_read)();
             }
-            error!("[REP#{}] timeout attempting to get remote buffer space, destination=0x{:02x}", repno, destination);
+            error!(
+                "[REP#{}] timeout attempting to get remote buffer space, destination=0x{:02x}",
+                repno, destination
+            );
         }
         unsafe {
             (csr::DRTIOREP[repno].protocol_error_write)(errors);
@@ -181,21 +199,25 @@ impl Repeater {
             match drtioaux::recv(self.auxno) {
                 Ok(Some(packet)) => return Ok(packet),
                 Ok(None) => (),
-                Err(e) => return Err(e)
+                Err(e) => return Err(e),
             }
         }
     }
 
     pub fn aux_forward(&self, request: &drtioaux::Packet, timer: &mut GlobalTimer) -> Result<(), drtioaux::Error> {
-        if self.state != RepeaterState::Up {
+        self.aux_send(request)?;
-            return Err(drtioaux::Error::LinkDown);
-        }
-        drtioaux::send(self.auxno, request).unwrap();
         let reply = self.recv_aux_timeout(200, timer)?;
         drtioaux::send(0, &reply).unwrap();
         Ok(())
     }
 
+    pub fn aux_send(&self, request: &drtioaux::Packet) -> Result<(), drtioaux::Error> {
+        if self.state != RepeaterState::Up {
+            return Err(drtioaux::Error::LinkDown);
+        }
+        drtioaux::send(self.auxno, request)
+    }
+
     pub fn sync_tsc(&self, timer: &mut GlobalTimer) -> Result<(), drtioaux::Error> {
         if self.state != RepeaterState::Up {
             return Ok(());
@@ -217,15 +239,24 @@ impl Repeater {
         }
     }
 
-    pub fn set_path(&self, destination: u8, hops: &[u8; drtio_routing::MAX_HOPS], timer: &mut GlobalTimer) -> Result<(), drtioaux::Error> {
+    pub fn set_path(
+        &self,
+        destination: u8,
+        hops: &[u8; drtio_routing::MAX_HOPS],
+        timer: &mut GlobalTimer,
+    ) -> Result<(), drtioaux::Error> {
         if self.state != RepeaterState::Up {
             return Ok(());
         }
 
-        drtioaux::send(self.auxno, &drtioaux::Packet::RoutingSetPath {
+        drtioaux::send(
-            destination: destination,
+            self.auxno,
-            hops: *hops
+            &drtioaux::Packet::RoutingSetPath {
-        }).unwrap();
+                destination: destination,
+                hops: *hops,
+            },
+        )
+        .unwrap();
         let reply = self.recv_aux_timeout(200, timer)?;
         if reply != drtioaux::Packet::RoutingAck {
             return Err(drtioaux::Error::UnexpectedReply);
@@ -233,7 +264,11 @@ impl Repeater {
         Ok(())
     }
 
-    pub fn load_routing_table(&self, routing_table: &drtio_routing::RoutingTable, timer: &mut GlobalTimer) -> Result<(), drtioaux::Error> {
+    pub fn load_routing_table(
+        &self,
+        routing_table: &drtio_routing::RoutingTable,
+        timer: &mut GlobalTimer,
+    ) -> Result<(), drtioaux::Error> {
         for i in 0..drtio_routing::DEST_COUNT {
             self.set_path(i as u8, &routing_table.0[i], timer)?;
         }
@@ -244,9 +279,7 @@ impl Repeater {
         if self.state != RepeaterState::Up {
             return Ok(());
         }
-        drtioaux::send(self.auxno, &drtioaux::Packet::RoutingSetRank {
+        drtioaux::send(self.auxno, &drtioaux::Packet::RoutingSetRank { rank: rank }).unwrap();
-            rank: rank
-        }).unwrap();
         let reply = self.recv_aux_timeout(200, timer)?;
         if reply != drtioaux::Packet::RoutingAck {
             return Err(drtioaux::Error::UnexpectedReply);
@@ -256,9 +289,13 @@ impl Repeater {
 
     pub fn rtio_reset(&self, timer: &mut GlobalTimer) -> Result<(), drtioaux::Error> {
         let repno = self.repno as usize;
-        unsafe { (csr::DRTIOREP[repno].reset_write)(1); }
+        unsafe {
+            (csr::DRTIOREP[repno].reset_write)(1);
+        }
         timer.delay_us(100);
-        unsafe { (csr::DRTIOREP[repno].reset_write)(0); }
+        unsafe {
+            (csr::DRTIOREP[repno].reset_write)(0);
+        }
 
         if self.state != RepeaterState::Up {
             return Ok(());
@@ -275,16 +312,29 @@ impl Repeater {
 
 #[cfg(not(has_drtio_routing))]
 #[derive(Clone, Copy, Default)]
-pub struct Repeater {
+pub struct Repeater {}
-}
 
 #[cfg(not(has_drtio_routing))]
 impl Repeater {
-    pub fn new(_repno: u8) -> Repeater { Repeater::default() }
+    pub fn new(_repno: u8) -> Repeater {
+        Repeater::default()
+    }
 
-    pub fn service(&self, _routing_table: &drtio_routing::RoutingTable, _rank: u8, _timer: &mut GlobalTimer) { }
+    pub fn service(
+        &self,
+        _routing_table: &drtio_routing::RoutingTable,
+        _rank: u8,
+        _destination: u8,
+        _router: &mut Router,
+        _timer: &mut GlobalTimer,
+    ) {
+    }
 
-    pub fn sync_tsc(&self, _timer: &mut GlobalTimer) -> Result<(), drtioaux::Error> { Ok(()) }
+    pub fn sync_tsc(&self, _timer: &mut GlobalTimer) -> Result<(), drtioaux::Error> {
+        Ok(())
+    }
 
-    pub fn rtio_reset(&self, _timer: &mut GlobalTimer) -> Result<(), drtioaux::Error> { Ok(()) }
+    pub fn rtio_reset(&self, _timer: &mut GlobalTimer) -> Result<(), drtioaux::Error> {
+        Ok(())
+    }
 }

src/satman/src/routing.rs

@@ -0,0 +1,174 @@
+use alloc::{collections::vec_deque::VecDeque, vec::Vec};
+use core::cmp::min;
+
+#[cfg(has_drtio_routing)]
+use libboard_artiq::pl::csr;
+use libboard_artiq::{drtio_routing, drtioaux,
+                     drtioaux_proto::{PayloadStatus, MASTER_PAYLOAD_MAX_SIZE, SAT_PAYLOAD_MAX_SIZE}};
+
+pub struct SliceMeta {
+    pub destination: u8,
+    pub len: u16,
+    pub status: PayloadStatus,
+}
+
+/* represents data that has to be sent to Master */
+#[derive(Debug)]
+pub struct Sliceable {
+    it: usize,
+    data: Vec<u8>,
+    destination: u8,
+}
+
+macro_rules! get_slice_fn {
+    ($name:tt, $size:expr) => {
+        pub fn $name(&mut self, data_slice: &mut [u8; $size]) -> SliceMeta {
+            let first = self.it == 0;
+            let len = min($size, self.data.len() - self.it);
+            let last = self.it + len == self.data.len();
+            let status = PayloadStatus::from_status(first, last);
+
+            data_slice[..len].clone_from_slice(&self.data[self.it..self.it + len]);
+            self.it += len;
+
+            SliceMeta {
+                destination: self.destination,
+                len: len as u16,
+                status: status,
+            }
+        }
+    };
+}
+
+impl Sliceable {
+    pub fn new(destination: u8, data: Vec<u8>) -> Sliceable {
+        Sliceable {
+            it: 0,
+            data: data,
+            destination: destination,
+        }
+    }
+
+    pub fn at_end(&self) -> bool {
+        self.it == self.data.len()
+    }
+
+    pub fn extend(&mut self, data: &[u8]) {
+        self.data.extend(data);
+    }
+
+    get_slice_fn!(get_slice_sat, SAT_PAYLOAD_MAX_SIZE);
+    get_slice_fn!(get_slice_master, MASTER_PAYLOAD_MAX_SIZE);
+}
+
+// Packets from downstream (further satellites) are received and routed appropriately.
+// they're passed as soon as possible downstream (within the subtree), or sent upstream,
+// which is notified about pending packets.
+// for rank 1 (connected to master) satellites, these packets are passed as an answer to DestinationStatusRequest;
+// for higher ranks, after getting a notification, it will transact with downstream to get the pending packets.
+
+// forward! macro is not deprecated, as routable packets are only these that can originate
+// from both master and satellite, e.g. DDMA and Subkernel.
+
+pub struct Router {
+    upstream_queue: VecDeque<drtioaux::Packet>,
+    local_queue: VecDeque<drtioaux::Packet>,
+    #[cfg(has_drtio_routing)]
+    downstream_queue: VecDeque<(usize, drtioaux::Packet)>,
+}
+
+impl Router {
+    pub fn new() -> Router {
+        Router {
+            upstream_queue: VecDeque::new(),
+            local_queue: VecDeque::new(),
+            #[cfg(has_drtio_routing)]
+            downstream_queue: VecDeque::new(),
+        }
+    }
+
+    // Called by local sources (DDMA, kernel) and by repeaters on receiving async data;
+    // messages are always buffered for both upstream and downstream
+    pub fn route(
+        &mut self,
+        packet: drtioaux::Packet,
+        _routing_table: &drtio_routing::RoutingTable,
+        _rank: u8,
+        self_destination: u8,
+    ) {
+        let destination = packet.routable_destination();
+        #[cfg(has_drtio_routing)]
+        {
+            if let Some(destination) = destination {
+                let hop = _routing_table.0[destination as usize][_rank as usize] as usize;
+                if destination == self_destination {
+                    self.local_queue.push_back(packet);
+                } else if hop > 0 && hop < csr::DRTIOREP.len() {
+                    let repno = (hop - 1) as usize;
+                    self.downstream_queue.push_back((repno, packet));
+                } else {
+                    self.upstream_queue.push_back(packet);
+                }
+            } else {
+                error!("Received an unroutable packet: {:?}", packet);
+            }
+        }
+        #[cfg(not(has_drtio_routing))]
+        {
+            if destination == Some(self_destination) {
+                self.local_queue.push_back(packet);
+            } else {
+                self.upstream_queue.push_back(packet);
+            }
+        }
+    }
+
+    // Sends a packet to a required destination, routing if necessary
+    pub fn send(
+        &mut self,
+        packet: drtioaux::Packet,
+        _routing_table: &drtio_routing::RoutingTable,
+        _rank: u8,
+        _destination: u8,
+    ) -> Result<(), drtioaux::Error> {
+        #[cfg(has_drtio_routing)]
+        {
+            let destination = packet.routable_destination();
+            if let Some(destination) = destination {
+                let hop = _routing_table.0[destination as usize][_rank as usize] as usize;
+                if destination == 0 {
+                    // response is needed immediately if master required it
+                    drtioaux::send(0, &packet)?;
+                } else if !(hop > 0 && hop < csr::DRTIOREP.len()) {
+                    // higher rank can wait
+                    self.upstream_queue.push_back(packet);
+                } else {
+                    let repno = (hop - 1) as usize;
+                    // transaction will occur at closest possible opportunity
+                    self.downstream_queue.push_back((repno, packet));
+                }
+                Ok(())
+            } else {
+                // packet not supported in routing, fallback - sent directly
+                drtioaux::send(0, &packet)
+            }
+        }
+        #[cfg(not(has_drtio_routing))]
+        {
+            drtioaux::send(0, &packet)
+        }
+    }
+
+    pub fn get_upstream_packet(&mut self) -> Option<drtioaux::Packet> {
+        self.upstream_queue.pop_front()
+    }
+
+    #[cfg(has_drtio_routing)]
+    pub fn get_downstream_packet(&mut self) -> Option<(usize, drtioaux::Packet)> {
+        self.downstream_queue.pop_front()
+    }
+
+    pub fn get_local_packet(&mut self) -> Option<drtioaux::Packet> {
+        self.local_queue.pop_front()
+    }
+}

src/satman/src/subkernel.rs

@@ -0,0 +1,985 @@
+use alloc::{collections::BTreeMap,
+            format,
+            string::{String, ToString},
+            vec::Vec};
+use core::{option::NoneError, slice, str};
+
+use core_io::{Error as IoError, Write};
+use cslice::AsCSlice;
+use dma::{Error as DmaError, Manager as DmaManager};
+use io::{Cursor, ProtoWrite};
+use ksupport::{eh_artiq, kernel, rpc};
+use libboard_artiq::{drtio_routing::RoutingTable,
+                     drtioaux,
+                     drtioaux_proto::{PayloadStatus, MASTER_PAYLOAD_MAX_SIZE, SAT_PAYLOAD_MAX_SIZE},
+                     pl::csr};
+use libboard_zynq::{time::Milliseconds, timer::GlobalTimer};
+use libcortex_a9::sync_channel::Receiver;
+use log::warn;
+use routing::{Router, SliceMeta, Sliceable};
+
+#[derive(Debug, Clone, PartialEq)]
+enum KernelState {
+    Absent,
+    Loaded,
+    Running,
+    MsgAwait {
+        max_time: Option<Milliseconds>,
+        id: u32,
+        tags: Vec<u8>,
+    },
+    MsgSending,
+    SubkernelAwaitLoad,
+    SubkernelAwaitFinish {
+        max_time: Option<Milliseconds>,
+        id: u32,
+    },
+    DmaUploading,
+    DmaPendingPlayback {
+        id: u32,
+        timestamp: u64,
+    },
+    DmaPendingAwait {
+        id: u32,
+        timestamp: u64,
+        max_time: Milliseconds,
+    },
+    DmaAwait {
+        max_time: Milliseconds,
+    },
+}
+
+#[derive(Debug)]
+pub enum Error {
+    Load(String),
+    KernelNotFound,
+    Unexpected(String),
+    NoMessage,
+    AwaitingMessage,
+    SubkernelIoError,
+    DrtioError,
+    KernelException(Sliceable),
+    DmaError(DmaError),
+}
+
+impl From<NoneError> for Error {
+    fn from(_: NoneError) -> Error {
+        Error::KernelNotFound
+    }
+}
+
+impl From<IoError> for Error {
+    fn from(_value: IoError) -> Error {
+        Error::SubkernelIoError
+    }
+}
+
+impl From<DmaError> for Error {
+    fn from(value: DmaError) -> Error {
+        Error::DmaError(value)
+    }
+}
+
+impl From<()> for Error {
+    fn from(_: ()) -> Error {
+        Error::NoMessage
+    }
+}
+
+impl From<drtioaux::Error> for Error {
+    fn from(_value: drtioaux::Error) -> Error {
+        Error::DrtioError
+    }
+}
+
+macro_rules! unexpected {
+    ($($arg:tt)*) => (return Err(Error::Unexpected(format!($($arg)*))));
+}
+
+/* represents interkernel messages */
+struct Message {
+    count: u8,
+    id: u32,
+    data: Vec<u8>,
+}
+
+#[derive(PartialEq)]
+enum OutMessageState {
+    NoMessage,
+    MessageBeingSent,
+    MessageSent,
+    MessageAcknowledged,
+}
+
+/* for dealing with incoming and outgoing interkernel messages */
+struct MessageManager {
+    out_message: Option<Sliceable>,
+    out_state: OutMessageState,
+    in_queue: Vec<Message>,
+    in_buffer: Option<Message>,
+}
+
+// Per-run state
+struct Session {
+    id: u32,
+    kernel_state: KernelState,
+    last_exception: Option<Sliceable>,
+    messages: MessageManager,
+    source: u8, // which destination requested running the kernel
+    subkernels_finished: Vec<u32>,
+}
+
+impl Session {
+    pub fn new(id: u32) -> Session {
+        Session {
+            id: id,
+            kernel_state: KernelState::Absent,
+            last_exception: None,
+            messages: MessageManager::new(),
+            source: 0,
+            subkernels_finished: Vec::new(),
+        }
+    }
+
+    fn running(&self) -> bool {
+        match self.kernel_state {
+            KernelState::Absent | KernelState::Loaded => false,
+            _ => true,
+        }
+    }
+}
+
+#[derive(Debug)]
+struct KernelLibrary {
+    library: Vec<u8>,
+    complete: bool,
+}
+
+pub struct Manager<'a> {
+    kernels: BTreeMap<u32, KernelLibrary>,
+    session: Session,
+    control: &'a mut kernel::Control,
+    cache: BTreeMap<String, Vec<i32>>,
+    last_finished: Option<SubkernelFinished>,
+}
+
+pub struct SubkernelFinished {
+    pub id: u32,
+    pub with_exception: bool,
+    pub exception_source: u8,
+    pub source: u8,
+}
+
+impl MessageManager {
+    pub fn new() -> MessageManager {
+        MessageManager {
+            out_message: None,
+            out_state: OutMessageState::NoMessage,
+            in_queue: Vec::new(),
+            in_buffer: None,
+        }
+    }
+
+    pub fn handle_incoming(
+        &mut self,
+        status: PayloadStatus,
+        id: u32,
+        length: usize,
+        data: &[u8; MASTER_PAYLOAD_MAX_SIZE],
+    ) {
+        // called when receiving a message from master
+        if status.is_first() {
+            self.in_buffer = None;
+        }
+        match self.in_buffer.as_mut() {
+            Some(message) => message.data.extend(&data[..length]),
+            None => {
+                self.in_buffer = Some(Message {
+                    count: data[0],
+                    id: id,
+                    data: data[1..length].to_vec(),
+                });
+            }
+        };
+        if status.is_last() {
+            // when done, remove from working queue
+            self.in_queue.push(self.in_buffer.take().unwrap());
+        }
+    }
+
+    pub fn was_message_acknowledged(&mut self) -> bool {
+        match self.out_state {
+            OutMessageState::MessageAcknowledged => {
+                self.out_state = OutMessageState::NoMessage;
+                true
+            }
+            _ => false,
+        }
+    }
+
+    pub fn get_outgoing_slice(&mut self, data_slice: &mut [u8; MASTER_PAYLOAD_MAX_SIZE]) -> Option<SliceMeta> {
+        if self.out_state != OutMessageState::MessageBeingSent {
+            return None;
+        }
+        let meta = self.out_message.as_mut()?.get_slice_master(data_slice);
+        if meta.status.is_last() {
+            // clear the message slot
+            self.out_message = None;
+            // notify kernel with a flag that message is sent
+            self.out_state = OutMessageState::MessageSent;
+        }
+        Some(meta)
+    }
+
+    pub fn ack_slice(&mut self) -> bool {
+        // returns whether or not there's more to be sent
+        match self.out_state {
+            OutMessageState::MessageBeingSent => true,
+            OutMessageState::MessageSent => {
+                self.out_state = OutMessageState::MessageAcknowledged;
+                false
+            }
+            _ => {
+                warn!("received unsolicited SubkernelMessageAck");
+                false
+            }
+        }
+    }
+
+    pub fn accept_outgoing(
+        &mut self,
+        id: u32,
+        self_destination: u8,
+        destination: u8,
+        message: Vec<u8>,
+        routing_table: &RoutingTable,
+        rank: u8,
+        router: &mut Router,
+    ) -> Result<(), Error> {
+        self.out_message = Some(Sliceable::new(destination, message));
+
+        let mut data_slice: [u8; MASTER_PAYLOAD_MAX_SIZE] = [0; MASTER_PAYLOAD_MAX_SIZE];
+        self.out_state = OutMessageState::MessageBeingSent;
+        let meta = self.get_outgoing_slice(&mut data_slice).unwrap();
+        router.route(
+            drtioaux::Packet::SubkernelMessage {
+                source: self_destination,
+                destination: destination,
+                id: id,
+                status: meta.status,
+                length: meta.len as u16,
+                data: data_slice,
+            },
+            routing_table,
+            rank,
+            self_destination,
+        );
+        Ok(())
+    }
+
+    pub fn get_incoming(&mut self, id: u32) -> Option<Message> {
+        for i in 0..self.in_queue.len() {
+            if self.in_queue[i].id == id {
+                return Some(self.in_queue.remove(i));
+            }
+        }
+        None
+    }
+}
+
+impl<'a> Manager<'_> {
+    pub fn new(control: &mut kernel::Control) -> Manager {
+        Manager {
+            kernels: BTreeMap::new(),
+            session: Session::new(0),
+            control: control,
+            cache: BTreeMap::new(),
+            last_finished: None,
+        }
+    }
+
+    pub fn add(&mut self, id: u32, status: PayloadStatus, data: &[u8], data_len: usize) -> Result<(), Error> {
+        let kernel = match self.kernels.get_mut(&id) {
+            Some(kernel) => {
+                if kernel.complete || status.is_first() {
+                    // replace entry
+                    self.kernels.remove(&id);
+                    self.kernels.insert(
+                        id,
+                        KernelLibrary {
+                            library: Vec::new(),
+                            complete: false,
+                        },
+                    );
+                    self.kernels.get_mut(&id)?
+                } else {
+                    kernel
+                }
+            }
+            None => {
+                self.kernels.insert(
+                    id,
+                    KernelLibrary {
+                        library: Vec::new(),
+                        complete: false,
+                    },
+                );
+                self.kernels.get_mut(&id)?
+            }
+        };
+        kernel.library.extend(&data[0..data_len]);
+
+        kernel.complete = status.is_last();
+        Ok(())
+    }
+
+    pub fn running(&self) -> bool {
+        self.session.running()
+    }
+
+    pub fn get_current_id(&self) -> Option<u32> {
+        match self.running() {
+            true => Some(self.session.id),
+            false => None,
+        }
+    }
+
+    pub fn run(&mut self, source: u8, id: u32) -> Result<(), Error> {
+        if self.session.kernel_state != KernelState::Loaded || self.session.id != id {
+            self.load(id)?;
+        }
+        self.session.kernel_state = KernelState::Running;
+        self.session.source = source;
+        unsafe {
+            csr::cri_con::selected_write(2);
+        }
+
+        self.control.tx.send(kernel::Message::StartRequest);
+        Ok(())
+    }
+
+    pub fn message_handle_incoming(
+        &mut self,
+        status: PayloadStatus,
+        id: u32,
+        length: usize,
+        slice: &[u8; MASTER_PAYLOAD_MAX_SIZE],
+    ) {
+        if !self.running() {
+            return;
+        }
+        self.session.messages.handle_incoming(status, id, length, slice);
+    }
+
+    pub fn message_get_slice(&mut self, slice: &mut [u8; MASTER_PAYLOAD_MAX_SIZE]) -> Option<SliceMeta> {
+        if !self.running() {
+            return None;
+        }
+        self.session.messages.get_outgoing_slice(slice)
+    }
+
+    pub fn message_ack_slice(&mut self) -> bool {
+        if !self.running() {
+            warn!("received unsolicited SubkernelMessageAck");
+            return false;
+        }
+        self.session.messages.ack_slice()
+    }
+
+    pub fn load(&mut self, id: u32) -> Result<(), Error> {
+        if self.session.id == id && self.session.kernel_state == KernelState::Loaded {
+            return Ok(());
+        }
+        if !self.kernels.get(&id)?.complete {
+            return Err(Error::KernelNotFound);
+        }
+        self.session = Session::new(id);
+        self.control.restart();
+
+        self.control
+            .tx
+            .send(kernel::Message::LoadRequest(self.kernels.get(&id)?.library.clone()));
+        let reply = self.control.rx.recv();
+        match reply {
+            kernel::Message::LoadCompleted => Ok(()),
+            kernel::Message::LoadFailed => Err(Error::Load("kernel load failed".to_string())),
+            _ => Err(Error::Load(format!(
+                "unexpected kernel CPU reply to load request: {:?}",
+                reply
+            ))),
+        }
+    }
+
+    pub fn exception_get_slice(&mut self, data_slice: &mut [u8; SAT_PAYLOAD_MAX_SIZE]) -> SliceMeta {
+        match self.session.last_exception.as_mut() {
+            Some(exception) => exception.get_slice_sat(data_slice),
+            None => SliceMeta {
+                destination: 0,
+                len: 0,
+                status: PayloadStatus::FirstAndLast,
+            },
+        }
+    }
+
+    fn kernel_stop(&mut self) {
+        self.session.kernel_state = KernelState::Absent;
+        unsafe {
+            csr::cri_con::selected_write(0);
+        }
+    }
+
+    fn runtime_exception(&mut self, cause: Error) {
+        let raw_exception: Vec<u8> = Vec::new();
+        let mut writer = Cursor::new(raw_exception);
+        match write_exception(
+            &mut writer,
+            &[Some(eh_artiq::Exception {
+                id: 11, // SubkernelError, defined in ksupport
+                message: format!("in subkernel id {}: {:?}", self.session.id, cause).as_c_slice(),
+                param: [0, 0, 0],
+                file: file!().as_c_slice(),
+                line: line!(),
+                column: column!(),
+                function: format!("subkernel id {}", self.session.id).as_c_slice(),
+            })],
+            &[eh_artiq::StackPointerBacktrace {
+                stack_pointer: 0,
+                initial_backtrace_size: 0,
+                current_backtrace_size: 0,
+            }],
+            &[],
+            0,
+        ) {
+            Ok(_) => self.session.last_exception = Some(Sliceable::new(0, writer.into_inner())),
+            Err(_) => error!("Error writing exception data"),
+        }
+        self.kernel_stop();
+    }
+
+    pub fn ddma_finished(&mut self, error: u8, channel: u32, timestamp: u64) {
+        if let KernelState::DmaAwait { .. } = self.session.kernel_state {
+            self.control.tx.send(kernel::Message::DmaAwaitRemoteReply {
+                timeout: false,
+                error: error,
+                channel: channel,
+                timestamp: timestamp,
+            });
+            self.session.kernel_state = KernelState::Running;
+        }
+    }
+
+    pub fn ddma_nack(&mut self) {
+        // for simplicity treat it as a timeout...
+        if let KernelState::DmaAwait { .. } = self.session.kernel_state {
+            self.control.tx.send(kernel::Message::DmaAwaitRemoteReply {
+                timeout: true,
+                error: 0,
+                channel: 0,
+                timestamp: 0,
+            });
+            self.session.kernel_state = KernelState::Running;
+        }
+    }
+
+    pub fn ddma_remote_uploaded(&mut self, succeeded: bool) -> Option<(u32, u64)> {
+        // returns a tuple of id, timestamp in case a playback needs to be started immediately
+        if !succeeded {
+            self.kernel_stop();
+            self.runtime_exception(Error::DmaError(DmaError::UploadFail));
+        }
+        let res = match self.session.kernel_state {
+            KernelState::DmaPendingPlayback { id, timestamp } => {
+                self.session.kernel_state = KernelState::Running;
+                Some((id, timestamp))
+            }
+            KernelState::DmaPendingAwait {
+                id,
+                timestamp,
+                max_time,
+            } => {
+                self.session.kernel_state = KernelState::DmaAwait { max_time: max_time };
+                Some((id, timestamp))
+            }
+            KernelState::DmaUploading => {
+                self.session.kernel_state = KernelState::Running;
+                None
+            }
+            _ => None,
+        };
+        res
+    }
+
+    pub fn process_kern_requests(
+        &mut self,
+        router: &mut Router,
+        routing_table: &RoutingTable,
+        rank: u8,
+        destination: u8,
+        dma_manager: &mut DmaManager,
+        timer: &GlobalTimer,
+    ) {
+        if let Some(subkernel_finished) = self.last_finished.take() {
+            info!(
+                "subkernel {} finished, with exception: {}",
+                subkernel_finished.id, subkernel_finished.with_exception
+            );
+            router.route(
+                drtioaux::Packet::SubkernelFinished {
+                    destination: subkernel_finished.source,
+                    id: subkernel_finished.id,
+                    with_exception: subkernel_finished.with_exception,
+                    exception_src: subkernel_finished.exception_source,
+                },
+                &routing_table,
+                rank,
+                destination,
+            );
+        }
+
+        if !self.running() {
+            return;
+        }
+
+        match self.process_external_messages(timer) {
+            Ok(()) => (),
+            Err(Error::AwaitingMessage) => return, // kernel still waiting, do not process kernel messages
+            Err(Error::KernelException(exception)) => {
+                self.session.last_exception = Some(exception);
+                self.last_finished = Some(SubkernelFinished {
+                    id: self.session.id,
+                    with_exception: true,
+                    exception_source: destination,
+                    source: self.session.source,
+                });
+            }
+            Err(e) => {
+                error!("Error while running processing external messages: {:?}", e);
+                self.runtime_exception(e);
+                self.last_finished = Some(SubkernelFinished {
+                    id: self.session.id,
+                    with_exception: true,
+                    exception_source: destination,
+                    source: self.session.source,
+                });
+            }
+        }
+
+        match self.process_kern_message(router, routing_table, rank, destination, dma_manager, timer) {
+            Ok(true) => {
+                self.last_finished = Some(SubkernelFinished {
+                    id: self.session.id,
+                    with_exception: false,
+                    exception_source: 0,
+                    source: self.session.source,
+                });
+            }
+            Ok(false) | Err(Error::NoMessage) => (),
+            Err(Error::KernelException(exception)) => {
+                self.session.last_exception = Some(exception);
+                self.last_finished = Some(SubkernelFinished {
+                    id: self.session.id,
+                    with_exception: true,
+                    exception_source: destination,
+                    source: self.session.source,
+                });
+            }
+            Err(e) => {
+                error!("Error while running kernel: {:?}", e);
+                self.runtime_exception(e);
+                self.last_finished = Some(SubkernelFinished {
+                    id: self.session.id,
+                    with_exception: true,
+                    exception_source: destination,
+                    source: self.session.source,
+                });
+            }
+        }
+    }
+
+    pub fn subkernel_load_run_reply(&mut self, succeeded: bool) {
+        if self.session.kernel_state == KernelState::SubkernelAwaitLoad {
+            self.control
+                .tx
+                .send(kernel::Message::SubkernelLoadRunReply { succeeded: succeeded });
+            self.session.kernel_state = KernelState::Running;
+        } else {
+            warn!("received unsolicited SubkernelLoadRunReply");
+        }
+    }
+
+    pub fn remote_subkernel_finished(&mut self, id: u32, with_exception: bool, exception_source: u8) {
+        if with_exception {
+            self.kernel_stop();
+            self.last_finished = Some(SubkernelFinished {
+                source: self.session.source,
+                id: self.session.id,
+                with_exception: true,
+                exception_source: exception_source,
+            })
+        } else {
+            self.session.subkernels_finished.push(id);
+        }
+    }
+
+    fn process_kern_message(
+        &mut self,
+        router: &mut Router,
+        routing_table: &RoutingTable,
+        rank: u8,
+        self_destination: u8,
+        dma_manager: &mut DmaManager,
+        timer: &GlobalTimer,
+    ) -> Result<bool, Error> {
+        let reply = self.control.rx.try_recv()?;
+        match reply {
+            kernel::Message::KernelFinished(_async_errors) => {
+                self.kernel_stop();
+                dma_manager.cleanup(router, rank, self_destination, routing_table);
+                return Ok(true);
+            }
+            kernel::Message::KernelException(exceptions, stack_pointers, backtrace, async_errors) => {
+                error!("exception in kernel");
+                for exception in exceptions {
+                    error!("{:?}", exception.unwrap());
+                }
+                error!("stack pointers: {:?}", stack_pointers);
+                error!("backtrace: {:?}", backtrace);
+                let buf: Vec<u8> = Vec::new();
+                let mut writer = Cursor::new(buf);
+                match write_exception(&mut writer, exceptions, stack_pointers, backtrace, async_errors) {
+                    Ok(()) => (),
+                    Err(_) => error!("Error writing exception data"),
+                }
+                self.kernel_stop();
+                return Err(Error::KernelException(Sliceable::new(0, writer.into_inner())));
+            }
+            kernel::Message::CachePutRequest(key, value) => {
+                self.cache.insert(key, value);
+            }
+            kernel::Message::CacheGetRequest(key) => {
+                const DEFAULT: Vec<i32> = Vec::new();
+                let value = self.cache.get(&key).unwrap_or(&DEFAULT).clone();
+                self.control.tx.send(kernel::Message::CacheGetReply(value));
+            }
+
+            kernel::Message::DmaPutRequest(recorder) => {
+                // ddma is always used on satellites
+                if let Ok(id) = dma_manager.put_record(recorder, self_destination) {
+                    dma_manager.upload_traces(id, router, rank, self_destination, routing_table)?;
+                    self.session.kernel_state = KernelState::DmaUploading;
+                } else {
+                    unexpected!("DMAError: found an unsupported call to RTIO devices on master")
+                }
+            }
+            kernel::Message::DmaEraseRequest(name) => {
+                dma_manager.erase_name(&name, router, rank, self_destination, routing_table);
+            }
+            kernel::Message::DmaGetRequest(name) => {
+                let dma_meta = dma_manager.retrieve(self_destination, &name);
+                self.control.tx.send(kernel::Message::DmaGetReply(dma_meta));
+            }
+            kernel::Message::DmaStartRemoteRequest { id, timestamp } => {
+                if self.session.kernel_state != KernelState::DmaUploading {
+                    dma_manager.playback_remote(
+                        id as u32,
+                        timestamp as u64,
+                        router,
+                        rank,
+                        self_destination,
+                        routing_table,
+                    )?;
+                } else {
+                    self.session.kernel_state = KernelState::DmaPendingPlayback {
+                        id: id as u32,
+                        timestamp: timestamp as u64,
+                    };
+                }
+            }
+            kernel::Message::DmaAwaitRemoteRequest(_id) => {
+                let max_time = timer.get_time() + Milliseconds(10000);
+                self.session.kernel_state = match self.session.kernel_state {
+                    // if we are still waiting for the traces to be uploaded, extend the state by timeout
+                    KernelState::DmaPendingPlayback { id, timestamp } => KernelState::DmaPendingAwait {
+                        id: id,
+                        timestamp: timestamp,
+                        max_time: max_time,
+                    },
+                    _ => KernelState::DmaAwait { max_time: max_time },
+                };
+            }
+
+            kernel::Message::SubkernelMsgSend {
+                id: _id,
+                destination: msg_dest,
+                data,
+            } => {
+                let msg_dest = msg_dest.or(Some(self.session.source)).unwrap();
+                self.session.messages.accept_outgoing(
+                    self.session.id,
+                    self_destination,
+                    msg_dest,
+                    data,
+                    routing_table,
+                    rank,
+                    router,
+                )?;
+                self.session.kernel_state = KernelState::MsgSending;
+            }
+            kernel::Message::SubkernelMsgRecvRequest { id, timeout, tags } => {
+                let id = if id == -1 { self.session.id } else { id as u32 };
+                let max_time = if timeout > 0 {
+                    Some(timer.get_time() + Milliseconds(timeout as u64))
+                } else {
+                    None
+                };
+                self.session.kernel_state = KernelState::MsgAwait {
+                    max_time: max_time,
+                    id: id,
+                    tags: tags,
+                };
+            }
+            kernel::Message::SubkernelLoadRunRequest {
+                id,
+                destination: sk_destination,
+                run,
+            } => {
+                self.session.kernel_state = KernelState::SubkernelAwaitLoad;
+                router.route(
+                    drtioaux::Packet::SubkernelLoadRunRequest {
+                        source: self_destination,
+                        destination: sk_destination,
+                        id: id,
+                        run: run,
+                    },
+                    routing_table,
+                    rank,
+                    self_destination,
+                );
+            }
+
+            kernel::Message::SubkernelAwaitFinishRequest { id, timeout } => {
+                let max_time = if timeout > 0 {
+                    Some(timer.get_time() + Milliseconds(timeout as u64))
+                } else {
+                    None
+                };
+                self.session.kernel_state = KernelState::SubkernelAwaitFinish {
+                    max_time: max_time,
+                    id: id,
+                };
+            }
+            kernel::Message::UpDestinationsRequest(destination) => {
+                self.control.tx.send(kernel::Message::UpDestinationsReply(
+                    destination == (self_destination as i32),
+                ));
+            }
+            _ => {
+                unexpected!("unexpected message from core1 while kernel was running: {:?}", reply);
+            }
+        }
+        Ok(false)
+    }
+
+    fn process_external_messages(&mut self, timer: &GlobalTimer) -> Result<(), Error> {
+        match &self.session.kernel_state {
+            KernelState::MsgAwait { max_time, id, tags } => {
+                if let Some(max_time) = *max_time {
+                    if timer.get_time() > max_time {
+                        self.control.tx.send(kernel::Message::SubkernelMsgRecvReply {
+                            status: kernel::SubkernelStatus::Timeout,
+                            count: 0,
+                        });
+                        self.session.kernel_state = KernelState::Running;
+                        return Ok(());
+                    }
+                }
+                if let Some(message) = self.session.messages.get_incoming(*id) {
+                    self.control.tx.send(kernel::Message::SubkernelMsgRecvReply {
+                        status: kernel::SubkernelStatus::NoError,
+                        count: message.count,
+                    });
+                    let tags = tags.clone();
+                    self.session.kernel_state = KernelState::Running;
+                    self.pass_message_to_kernel(&message, tags, timer)
+                } else {
+                    Err(Error::AwaitingMessage)
+                }
+            }
+            KernelState::MsgSending => {
+                if self.session.messages.was_message_acknowledged() {
+                    self.session.kernel_state = KernelState::Running;
+                    self.control.tx.send(kernel::Message::SubkernelMsgSent);
+                    Ok(())
+                } else {
+                    Err(Error::AwaitingMessage)
+                }
+            }
+            KernelState::SubkernelAwaitFinish { max_time, id } => {
+                if let Some(max_time) = *max_time {
+                    if timer.get_time() > max_time {
+                        self.control.tx.send(kernel::Message::SubkernelAwaitFinishReply {
+                            status: kernel::SubkernelStatus::Timeout,
+                        });
+                        self.session.kernel_state = KernelState::Running;
+                        return Ok(());
+                    }
+                }
+                let mut i = 0;
+                for status in &self.session.subkernels_finished {
+                    if *status == *id {
+                        self.control.tx.send(kernel::Message::SubkernelAwaitFinishReply {
+                            status: kernel::SubkernelStatus::NoError,
+                        });
+                        self.session.kernel_state = KernelState::Running;
+                        self.session.subkernels_finished.swap_remove(i);
+                        break;
+                    }
+                    i += 1;
+                }
+                Ok(())
+            }
+            KernelState::DmaAwait { max_time } | KernelState::DmaPendingAwait { max_time, .. } => {
+                if timer.get_time() > *max_time {
+                    self.control.tx.send(kernel::Message::DmaAwaitRemoteReply {
+                        timeout: true,
+                        error: 0,
+                        channel: 0,
+                        timestamp: 0,
+                    });
+                    self.session.kernel_state = KernelState::Running;
+                }
+                Ok(())
+            }
+            _ => Ok(()),
+        }
+    }
+
+    fn pass_message_to_kernel(&mut self, message: &Message, tags: Vec<u8>, timer: &GlobalTimer) -> Result<(), Error> {
+        let mut reader = Cursor::new(&message.data);
+        let mut current_tags: &[u8] = &tags;
+        let mut i = message.count;
+        loop {
+            let slot = match recv_w_timeout(&mut self.control.rx, timer, 100)? {
+                kernel::Message::RpcRecvRequest(slot) => slot,
+                other => unexpected!("expected root value slot from core1, not {:?}", other),
+            };
+            let mut exception: Option<Sliceable> = None;
+            let mut unexpected: Option<String> = None;
+            let remaining_tags = rpc::recv_return(&mut reader, current_tags, slot, &mut |size| {
+                if size == 0 {
+                    0 as *mut ()
+                } else {
+                    self.control.tx.send(kernel::Message::RpcRecvReply(Ok(size)));
+                    match recv_w_timeout(&mut self.control.rx, timer, 100) {
+                        Ok(kernel::Message::RpcRecvRequest(slot)) => slot,
+                        Ok(kernel::Message::KernelException(exceptions, stack_pointers, backtrace, async_errors)) => {
+                            let buf: Vec<u8> = Vec::new();
+                            let mut writer = Cursor::new(buf);
+                            match write_exception(&mut writer, exceptions, stack_pointers, backtrace, async_errors) {
+                                Ok(()) => {
+                                    exception = Some(Sliceable::new(0, writer.into_inner()));
+                                }
+                                Err(_) => {
+                                    unexpected = Some("Error writing exception data".to_string());
+                                }
+                            };
+                            0 as *mut ()
+                        }
+                        other => {
+                            unexpected = Some(format!("expected nested value slot from kernel CPU, not {:?}", other));
+                            0 as *mut ()
+                        }
+                    }
+                }
+            })?;
+            if let Some(exception) = exception {
+                self.kernel_stop();
+                return Err(Error::KernelException(exception));
+            } else if let Some(unexpected) = unexpected {
+                self.kernel_stop();
+                unexpected!("{}", unexpected);
+            }
+            self.control.tx.send(kernel::Message::RpcRecvReply(Ok(0)));
+            i -= 1;
+            if i == 0 {
+                break;
+            } else {
+                current_tags = remaining_tags;
+            }
+        }
+        Ok(())
+    }
+}
+
+fn write_exception<W>(
+    writer: &mut W,
+    exceptions: &[Option<eh_artiq::Exception>],
+    stack_pointers: &[eh_artiq::StackPointerBacktrace],
+    backtrace: &[(usize, usize)],
+    async_errors: u8,
+) -> Result<(), Error>
+where
+    W: Write + ?Sized,
+{
+    /* header */
+    writer.write_bytes(&[0x5a, 0x5a, 0x5a, 0x5a, /*Reply::KernelException*/ 9])?;
+    writer.write_u32(exceptions.len() as u32)?;
+    for exception in exceptions.iter() {
+        let exception = exception.as_ref().unwrap();
+        writer.write_u32(exception.id)?;
+
+        if exception.message.len() == usize::MAX {
+            // exception with host string
+            writer.write_u32(u32::MAX)?;
+            writer.write_u32(exception.message.as_ptr() as u32)?;
+        } else {
+            let msg =
+                str::from_utf8(unsafe { slice::from_raw_parts(exception.message.as_ptr(), exception.message.len()) })
+                    .unwrap()
+                    .replace(
+                        "{rtio_channel_info:0}",
+                        &format!(
+                            "0x{:04x}:{}",
+                            exception.param[0],
+                            ksupport::resolve_channel_name(exception.param[0] as u32)
+                        ),
+                    );
+            writer.write_string(&msg)?;
+        }
+        writer.write_u64(exception.param[0] as u64)?;
+        writer.write_u64(exception.param[1] as u64)?;
+        writer.write_u64(exception.param[2] as u64)?;
+        writer.write_bytes(exception.file.as_ref())?;
+        writer.write_u32(exception.line)?;
+        writer.write_u32(exception.column)?;
+        writer.write_bytes(exception.function.as_ref())?;
+    }
+
+    for sp in stack_pointers.iter() {
+        writer.write_u32(sp.stack_pointer as u32)?;
+        writer.write_u32(sp.initial_backtrace_size as u32)?;
+        writer.write_u32(sp.current_backtrace_size as u32)?;
+    }
+    writer.write_u32(backtrace.len() as u32)?;
+    for &(addr, sp) in backtrace {
+        writer.write_u32(addr as u32)?;
+        writer.write_u32(sp as u32)?;
+    }
+    writer.write_u8(async_errors as u8)?;
+    Ok(())
+}
+
+fn recv_w_timeout(
+    rx: &mut Receiver<'_, kernel::Message>,
+    timer: &GlobalTimer,
+    timeout: u64,
+) -> Result<kernel::Message, Error> {
+    let max_time = timer.get_time() + Milliseconds(timeout);
+    while timer.get_time() < max_time {
+        match rx.try_recv() {
+            Err(_) => (),
+            Ok(message) => return Ok(message),
+        }
+    }
+    Err(Error::NoMessage)
+}